Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -435,19 +435,20 @@ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/session/test_session.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/test_recover.c \ - $(TOP)/ext/rbu/test_rbu.c + $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions # TESTSRC += \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/test_expert.c \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/basexx.c \ $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/cksumvfs.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/csv.c \ $(TOP)/ext/misc/decimal.c \ @@ -776,10 +777,19 @@ sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . +sqlite3r.h: sqlite3.h + $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h + +sqlite3r.c: sqlite3.c sqlite3r.h + cp $(TOP)/ext/recover/sqlite3recover.c tsrc/ + cp $(TOP)/ext/recover/sqlite3recover.h tsrc/ + cp $(TOP)/ext/recover/dbdata.c tsrc/ + $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) + sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . tclsqlite3.c: sqlite3.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c @@ -1106,10 +1116,13 @@ SHELL_SRC = \ $(TOP)/src/shell.c.in \ $(TOP)/ext/misc/appendvfs.c \ $(TOP)/ext/misc/completion.c \ $(TOP)/ext/misc/decimal.c \ + $(TOP)/ext/misc/basexx.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/shathree.c \ Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1559,10 +1559,11 @@ TESTEXT = \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\test_expert.c \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\appendvfs.c \ + $(TOP)\ext\misc\basexx.c \ $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\cksumvfs.c \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\csv.c \ $(TOP)\ext\misc\decimal.c \ @@ -2229,10 +2230,12 @@ # Source files that go into making shell.c SHELL_SRC = \ $(TOP)\src\shell.c.in \ $(TOP)\ext\misc\appendvfs.c \ $(TOP)\ext\misc\completion.c \ + $(TOP)\ext\misc\base64.c \ + $(TOP)\ext\misc\base85.c \ $(TOP)\ext\misc\decimal.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.40.2 +3.41.0 Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.40.2. +# Generated by GNU Autoconf 2.69 for sqlite 3.41.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # @@ -724,12 +724,12 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.40.2' -PACKAGE_STRING='sqlite 3.40.2' +PACKAGE_VERSION='3.41.0' +PACKAGE_STRING='sqlite 3.41.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ @@ -1466,11 +1466,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.40.2 to adapt to many kinds of systems. +\`configure' configures sqlite 3.41.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1531,11 +1531,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.40.2:";; + short | recursive ) echo "Configuration of sqlite 3.41.0:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1659,11 +1659,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.40.2 +sqlite configure 3.41.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. @@ -2078,11 +2078,11 @@ } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.40.2, which was +It was created by sqlite $as_me 3.41.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF @@ -12388,11 +12388,11 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.40.2, which was +This file was extended by sqlite $as_me 3.41.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -12454,11 +12454,11 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.40.2 +sqlite config.status 3.41.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -5071,11 +5071,11 @@ i64 iLastRowid = 0; /* Initialize a doclist-iterator for each input buffer. Arrange them in ** a linked-list starting at pHead in ascending order of rowid. Avoid ** linking any iterators already at EOF into the linked list at all. */ - assert( nBuf+1<=sizeof(aMerger)/sizeof(aMerger[0]) ); + assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) ); memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1)); pHead = &aMerger[nBuf]; fts5DoclistIterInit(p1, &pHead->iter); for(i=0; ip.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ /* A transaction must be open when this is called. */ - assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); + assert( pTab->ts.eState==1 ); assert( pVtab->zErrMsg==0 ); assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER || sqlite3_value_type(apVal[0])==SQLITE_NULL Index: ext/fts5/test/fts5misc.test ================================================================== --- ext/fts5/test/fts5misc.test +++ ext/fts5/test/fts5misc.test @@ -400,48 +400,7 @@ do_execsql_test 15.4 END do_test 15.4 { list [catch { db2 eval COMMIT } msg] $msg } {0 {}} -#------------------------------------------------------------------------- -reset_db -forcedelete test.db2 -sqlite3 db2 test.db -do_execsql_test 16.0 { - - ATTACH 'test.db2' AS aux; - CREATE TABLE aux.t2(x,y); - INSERT INTO t2 VALUES(1, 2); - CREATE VIRTUAL TABLE x1 USING fts5(a); - BEGIN; - INSERT INTO x1 VALUES('abc'); - INSERT INTO t2 VALUES(3, 4); -} - -do_execsql_test -db db2 16.1 { - ATTACH 'test.db2' AS aux; - BEGIN; - SELECT * FROM t2 -} {1 2} - -do_catchsql_test 16.2 { - COMMIT; -} {1 {database is locked}} - -do_execsql_test 16.3 { - INSERT INTO x1 VALUES('def'); -} - -do_execsql_test -db db2 16.4 { - END -} - -do_execsql_test 16.5 { - COMMIT -} - -do_execsql_test -db db2 16.6 { - SELECT * FROM x1 -} {abc def} - finish_test ADDED ext/misc/base64.c Index: ext/misc/base64.c ================================================================== --- /dev/null +++ ext/misc/base64.c @@ -0,0 +1,268 @@ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters and to +** terminate any encoded blob having non-zero length. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes matching that of USASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s ){ + char c; + while( (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; + } + pOut += nbo; + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big", -1); + return; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (u8*)sqlite3_value_blob(av[0]); + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ ADDED ext/misc/base85.c Index: ext/misc/base85.c ================================================================== --- /dev/null +++ ext/misc/base85.c @@ -0,0 +1,435 @@ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit USASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +# include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +/* Classify c according to interval within USASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static u8 base85DigitValue( char c ){ + u8 dv = (u8)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + +static char * skipNonB85( char *s ){ + char c; + while( (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. + * Do not use the macro form with argument expression having a side-effect.*/ +#if 0 +static char base85Numeral( u8 b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} +#else +# define base85Numeral( dn )\ + ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*'))) +#endif + +static char *putcs(char *pc, char *s){ + char c; + while( (c = *s++)!=0 ) *pc++ = c; + return pc; +} + +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + while( nbIn >= 4 ){ + int nco = 5; + unsigned long qbv = (pIn[0]<<24)|(pIn[1]<<16)|(pIn[2]<<8)|pIn[3]; + while( nco > 0 ){ + unsigned nqv = (unsigned)(qbv/85UL); + unsigned char dv = qbv - 85UL*nqv; + qbv = nqv; + pOut[--nco] = base85Numeral(dv); + } + nbIn -= 4; + pIn += 4; + pOut += 5; + if( pSep && (nCol += 5)>=B85_DARK_MAX ){ + pOut = putcs(pOut, pSep); + nCol = 0; + } + } + if( nbIn > 0 ){ + int nco = nbIn + 1; + unsigned long qv = *pIn++; + int nbe = 1; + while( nbe++ < nbIn ){ + qv = (qv<<8) | *pIn++; + } + nCol += nco; + while( nco > 0 ){ + u8 dv = (u8)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += (nbIn+1); + } + if( pSep && nCol>0 ) pOut = putcs(pOut, pSep); + *pOut = 0; + return pOut; +} + +/* Decode base85 text into a byte buffer. */ +static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + if( nbo==0 ) break; + while( nti>0 ){ + char c = *pIn++; + u8 cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + (c - cdo); + --nti; + } + nbo -= nti; /* Adjust for early (non-digit) end of group. */ + switch( nbo ){ + case 4: + *pOut++ = (qv >> 24)&0xff; + case 3: + *pOut++ = (qv >> 16)&0xff; + case 2: + *pOut++ = (qv >> 8)&0xff; + case 1: + *pOut++ = qv&0xff; + case 0: + break; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); + return; + } +} +# endif + +/* This function does the work for the SQLite base85(x) UDF. */ +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big", -1); + return; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (u8*)sqlite3_value_blob(av[0]); + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + u8 bBuf[4*(B85_DARK_MAX/5)]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif ADDED ext/misc/basexx.c Index: ext/misc/basexx.c ================================================================== --- /dev/null +++ ext/misc/basexx.c @@ -0,0 +1,86 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +** +** Example usages: +** +** - Build a runtime-loadable extension from SQLite checkout directory: +** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c +** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll +** +** - Incorporate as built-in in sqlite3 shell: +** *Nix, OSX with gcc on a like platform: +** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** make sqlite3 "OPTS=$mop1 $mop2" +** Win32 with Microsoft toolset on Windows: +** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** set mops="OPTS=%mop1% %mop2%" +** nmake -f Makefile.msc sqlite3.exe %mops% +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +typedef unsigned char u8; +#define U8_TYPEDEF + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + init_api_ptr(pApi); + int rc1 = BASE64_INIT(db); + int rc2 = BASE85_INIT(db); + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ Index: ext/misc/decimal.c ================================================================== --- ext/misc/decimal.c +++ ext/misc/decimal.c @@ -614,11 +614,11 @@ unsigned int i; (void)pzErrMsg; /* Unused parameter */ SQLITE_EXTENSION_INIT2(pApi); - for(i=0; ipRow; for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ const char *zSql = sqlite3_sql(p); sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0; @@ -269,10 +277,11 @@ */ static int stmtBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ + (void)tab; pIdxInfo->estimatedCost = (double)500; pIdxInfo->estimatedRows = 500; return SQLITE_OK; } Index: ext/misc/zipfile.c ================================================================== --- ext/misc/zipfile.c +++ ext/misc/zipfile.c @@ -350,10 +350,11 @@ int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE; int nFile = 0; const char *zFile = 0; ZipfileTab *pNew = 0; int rc; + (void)pAux; /* If the table name is not "zipfile", require that the argument be ** specified. This stops zipfile tables from being created as: ** ** CREATE VIRTUAL TABLE zzz USING zipfile(); @@ -806,10 +807,11 @@ ZipfileEntry **ppEntry /* OUT: Pointer to new object */ ){ u8 *aRead; char **pzErr = &pTab->base.zErrMsg; int rc = SQLITE_OK; + (void)nBlob; if( aBlob==0 ){ aRead = pTab->aBuffer; rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); }else{ @@ -1251,10 +1253,13 @@ ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; ZipfileCsr *pCsr = (ZipfileCsr*)cur; const char *zFile = 0; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ int bInMemory = 0; /* True for an in-memory zipfile */ + + (void)idxStr; + (void)argc; zipfileResetCursor(pCsr); if( pTab->zFile ){ zFile = pTab->zFile; @@ -1278,11 +1283,11 @@ }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } if( 0==pTab->pWriteFd && 0==bInMemory ){ - pCsr->pFile = zFile ? fopen(zFile, "rb") : 0; + pCsr->pFile = fopen(zFile, "rb"); if( pCsr->pFile==0 ){ zipfileCursorErr(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; }else{ rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); @@ -1312,10 +1317,11 @@ sqlite3_index_info *pIdxInfo ){ int i; int idx = -1; int unusable = 0; + (void)tab; for(i=0; inConstraint; i++){ const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue; if( pCons->usable==0 ){ @@ -1561,10 +1567,12 @@ ZipfileEntry *pOld = 0; ZipfileEntry *pOld2 = 0; int bUpdate = 0; /* True for an update that modifies "name" */ int bIsDir = 0; u32 iCrc32 = 0; + + (void)pRowid; if( pTab->pWriteFd==0 ){ rc = zipfileBegin(pVtab); if( rc!=SQLITE_OK ) return rc; } @@ -1896,10 +1904,11 @@ int nArg, /* Number of SQL function arguments */ const char *zName, /* Name of SQL function */ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ + (void)nArg; if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ *pxFunc = zipfileFunctionCds; *ppArg = (void*)pVtab; return 1; } Index: ext/rbu/rbuB.test ================================================================== --- ext/rbu/rbuB.test +++ ext/rbu/rbuB.test @@ -46,11 +46,11 @@ run_rbu test.db rbu.db } {SQLITE_DONE} do_test 1.3 { set ::errlog -} {SQLITE_NOTICE_RECOVER_WAL SQLITE_INTERNAL} +} {SQLITE_NOTICE_RECOVER_WAL SQLITE_NOTICE_RBU} do_execsql_test 1.4 { SELECT * FROM t1 } {1 2 3 4 5 6 7 8 9} Index: ext/rbu/sqlite3rbu.c ================================================================== --- ext/rbu/sqlite3rbu.c +++ ext/rbu/sqlite3rbu.c @@ -3029,25 +3029,25 @@ ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, ** READ0 and CHECKPOINT locks taken as part of the checkpoint are ** no-ops. These locks will not be released until the connection ** is closed. ** - ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** * Attempting to xSync() the database file causes an SQLITE_NOTICE ** error. ** ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the - ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[] ** array populated with a set of (frame -> page) mappings. Because the ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy ** data from the wal file into the database file according to the ** contents of aFrame[]. */ if( p->rc==SQLITE_OK ){ int rc2; p->eStage = RBU_STAGE_CAPTURE; rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); - if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + if( rc2!=SQLITE_NOTICE ) p->rc = rc2; } if( p->rc==SQLITE_OK && p->nFrame>0 ){ p->eStage = RBU_STAGE_CKPT; p->nStep = (pState ? pState->nRow : 0); @@ -3089,11 +3089,11 @@ const u32 mReq = (1<mLock!=mReq ){ pRbu->rc = SQLITE_BUSY; - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } pRbu->pgsz = iAmt; if( pRbu->nFrame==pRbu->nFrameAlloc ){ int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; @@ -4476,11 +4476,11 @@ ** are recorded. Additionally, successful attempts to obtain exclusive ** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target ** database file are recorded. xShmLock() calls to unlock the same ** locks are no-ops (so that once obtained, these locks are never ** relinquished). Finally, calls to xSync() on the target database -** file fail with SQLITE_INTERNAL errors. +** file fail with SQLITE_NOTICE errors. */ static void rbuUnlockShm(rbu_file *p){ assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); if( p->pRbu ){ @@ -4755,11 +4755,11 @@ */ static int rbuVfsSync(sqlite3_file *pFile, int flags){ rbu_file *p = (rbu_file *)pFile; if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } return SQLITE_OK; } return p->pReal->pMethods->xSync(p->pReal, flags); } Index: ext/recover/dbdata.c ================================================================== --- ext/recover/dbdata.c +++ ext/recover/dbdata.c @@ -162,10 +162,13 @@ char **pzErr ){ DbdataTable *pTab = 0; int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); + (void)argc; + (void)argv; + (void)pzErr; if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ @@ -766,10 +769,12 @@ ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; int rc = SQLITE_OK; const char *zSchema = "main"; + (void)idxStr; + (void)argc; dbdataResetCursor(pCsr); assert( pCsr->iPgno==1 ); if( idxNum & 0x01 ){ zSchema = (const char*)sqlite3_value_text(argv[0]); @@ -934,9 +939,10 @@ sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; return sqlite3DbdataRegister(db); } #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ Index: ext/recover/recover1.test ================================================================== --- ext/recover/recover1.test +++ ext/recover/recover1.test @@ -273,48 +273,46 @@ } {} do_recover_test 15 #------------------------------------------------------------------------- reset_db -if {[wal_is_capable]} { - do_execsql_test 16.1 { - PRAGMA journal_mode = wal; - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(1), (2), (3); - } {wal} - do_test 16.2 { - set R [sqlite3_recover_init db main test.db2] - $R run - $R finish - } {} - do_execsql_test 16.3 { - SELECT * FROM t1; - } {1 2 3} - - do_execsql_test 16.4 { - BEGIN; - SELECT * FROM t1; - } {1 2 3} - do_test 16.5 { - set R [sqlite3_recover_init db main test.db2] - $R run - list [catch { $R finish } msg] $msg - } {1 {cannot start a transaction within a transaction}} - do_execsql_test 16.6 { - SELECT * FROM t1; - } {1 2 3} - do_execsql_test 16.7 { - INSERT INTO t1 VALUES(4); - } - do_test 16.8 { - set R [sqlite3_recover_init db main test.db2] - $R run - list [catch { $R finish } msg] $msg - } {1 {cannot start a transaction within a transaction}} - do_execsql_test 16.9 { - SELECT * FROM t1; - COMMIT; - } {1 2 3 4} -} +do_execsql_test 16.1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1), (2), (3); +} {wal} +do_test 16.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} +do_execsql_test 16.3 { + SELECT * FROM t1; +} {1 2 3} + +do_execsql_test 16.4 { + BEGIN; + SELECT * FROM t1; +} {1 2 3} +do_test 16.5 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {1 {cannot start a transaction within a transaction}} +do_execsql_test 16.6 { + SELECT * FROM t1; +} {1 2 3} +do_execsql_test 16.7 { + INSERT INTO t1 VALUES(4); +} +do_test 16.8 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {1 {cannot start a transaction within a transaction}} +do_execsql_test 16.9 { + SELECT * FROM t1; + COMMIT; +} {1 2 3 4} finish_test Index: ext/recover/sqlite3recover.c ================================================================== --- ext/recover/sqlite3recover.c +++ ext/recover/sqlite3recover.c @@ -759,10 +759,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ const char *zText = (const char*)sqlite3_value_text(argv[0]); + (void)argc; if( zText && zText[0]=='\'' ){ int nText = sqlite3_value_bytes(argv[0]); int i; char zBuf1[20]; char zBuf2[20]; @@ -911,11 +912,11 @@ if( rc!=SQLITE_OK ){ recoverDbError(p, db2); return; } - for(ii=0; iidbIn, "PRAGMA %Q.%s", p->zDb, zPrag); if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ const char *zArg = (const char*)sqlite3_column_text(p1, 0); @@ -989,11 +990,13 @@ if( p->errCode==SQLITE_OK ){ p->errCode = sqlite3_dbdata_init(db, 0, 0); } /* Register the custom user-functions with the output handle. */ - for(ii=0; p->errCode==SQLITE_OK && iierrCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0])); + ii++){ p->errCode = sqlite3_create_function(db, aFunc[ii].zName, aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 ); } @@ -2386,11 +2389,11 @@ recoverPutU32(&aHdr[56], enc); recoverPutU16(&aHdr[105], pgsz-nReserve); if( pgsz==65536 ) pgsz = 1; recoverPutU16(&aHdr[16], pgsz); aHdr[20] = nReserve; - for(ii=0; ii=(4+6*sizeof(GeoCoord)) + && (nByte = sqlite3_value_bytes(pVal))>=(int)(4+6*sizeof(GeoCoord)) ){ const unsigned char *a = sqlite3_value_blob(pVal); int nVertex; if( a==0 ){ if( pCtx ) sqlite3_result_error_nomem(pCtx); @@ -360,10 +360,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3_result_blob(context, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); sqlite3_free(p); } @@ -379,10 +380,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3 *db = sqlite3_context_db_handle(context); sqlite3_str *x = sqlite3_str_new(db); int i; sqlite3_str_append(x, "[", 1); @@ -460,10 +462,11 @@ double D = sqlite3_value_double(argv[4]); double E = sqlite3_value_double(argv[5]); double F = sqlite3_value_double(argv[6]); GeoCoord x1, y1, x0, y0; int ii; + (void)argc; if( p ){ for(ii=0; iinVertex; ii++){ x0 = GeoX(p,ii); y0 = GeoY(p,ii); x1 = (GeoCoord)(A*x0 + B*y0 + E); @@ -510,10 +513,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3_result_double(context, geopolyArea(p)); sqlite3_free(p); } } @@ -535,10 +539,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ if( geopolyArea(p)<0.0 ){ int ii, jj; for(ii=1, jj=p->nVertex-1; ii1000 ) n = 1000; p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) ); if( p==0 ){ @@ -698,10 +704,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + (void)argc; if( p ){ sqlite3_result_blob(context, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); sqlite3_free(p); } @@ -725,10 +732,11 @@ int argc, sqlite3_value **argv ){ RtreeCoord a[4]; int rc = SQLITE_OK; + (void)argc; (void)geopolyBBox(context, argv[0], a, &rc); if( rc==SQLITE_OK ){ GeoBBox *pBBox; pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox)); if( pBBox==0 ) return; @@ -813,10 +821,12 @@ double x0 = sqlite3_value_double(argv[1]); double y0 = sqlite3_value_double(argv[2]); int v = 0; int cnt = 0; int ii; + (void)argc; + if( p1==0 ) return; for(ii=0; iinVertex-1; ii++){ v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), GeoX(p1,ii+1),GeoY(p1,ii+1)); if( v==2 ) break; @@ -852,10 +862,11 @@ int argc, sqlite3_value **argv ){ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; if( p1 && p2 ){ int x = geopolyOverlap(p1, p2); if( x<0 ){ sqlite3_result_error_nomem(context); }else{ @@ -1182,10 +1193,11 @@ int argc, sqlite3_value **argv ){ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; if( p1 && p2 ){ int x = geopolyOverlap(p1, p2); if( x<0 ){ sqlite3_result_error_nomem(context); }else{ @@ -1202,12 +1214,16 @@ static void geopolyDebugFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ + (void)context; + (void)argc; #ifdef GEOPOLY_ENABLE_DEBUG geo_debug = sqlite3_value_int(argv[0]); +#else + (void)argv; #endif } /* ** This function is the implementation of both the xConnect and xCreate @@ -1231,10 +1247,11 @@ sqlite3_int64 nDb; /* Length of string argv[1] */ sqlite3_int64 nName; /* Length of string argv[2] */ sqlite3_str *pSql; char *zSql; int ii; + (void)pAux; sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); @@ -1347,10 +1364,11 @@ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; RtreeNode *pRoot = 0; int rc = SQLITE_OK; int iCell = 0; + (void)idxStr; rtreeReference(pRtree); /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ resetCursor(pCsr); @@ -1473,10 +1491,11 @@ static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int ii; int iRowidTerm = -1; int iFuncTerm = -1; int idxNum = 0; + (void)tab; for(ii=0; iinConstraint; ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; if( !p->usable ) continue; if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ @@ -1719,10 +1738,12 @@ int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg ){ + (void)pVtab; + (void)nArg; if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){ *pxFunc = geopolyOverlapFunc; *ppArg = 0; return SQLITE_INDEX_CONSTRAINT_FUNCTION; } @@ -1788,11 +1809,11 @@ void (*xFinal)(sqlite3_context*); const char *zName; } aAgg[] = { { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" }, }; - int i; + unsigned int i; for(i=0; i=1300 pCoord->u = _byteswap_ulong(*(u32*)p); #elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 pCoord->u = __builtin_bswap32(*(u32*)p); #elif SQLITE_BYTEORDER==4321 @@ -553,11 +553,11 @@ p[0] = (i>> 8)&0xFF; p[1] = (i>> 0)&0xFF; } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; - assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( (((sqlite3_uint64)p)&3)==0 ); /* p is always 4-byte aligned */ assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); #if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 i = __builtin_bswap32(pCoord->u); memcpy(p, &i, 4); @@ -1281,11 +1281,11 @@ pCellData += 8 + 4*(p->iCoord&0xfe); assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + assert( (((sqlite3_uint64)pCellData)&3)==0 ); /* 4-byte aligned */ switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ case RTREE_FALSE: break; /* Never satisfied */ case RTREE_EQ: RTREE_DECODE_COORD(eInt, pCellData, val); @@ -1334,11 +1334,11 @@ assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); pCellData += 8 + p->iCoord*4; - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + assert( (((sqlite3_uint64)pCellData)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ case RTREE_FALSE: break; /* Never satisfied */ case RTREE_LE: if( xN <= p->u.rValue ) return; break; Index: ext/session/sessionat.test ================================================================== --- ext/session/sessionat.test +++ ext/session/sessionat.test @@ -241,10 +241,64 @@ proc xConfict {args} { return "OMIT" } do_test $tn.6.3 { sqlite3changeset_apply db $cinv xConflict execsql { SELECT * FROM t7 } } {1 1 ccc 2 2 ccc 3 3 ccc} + + #----------------------------------------------------------------------- + reset_test + do_execsql_test $tn.7.0 { + CREATE TABLE t8(a PRIMARY KEY, b, c); + } + do_execsql_test -db db2 $tn.7.1 { + CREATE TABLE t8(a PRIMARY KEY, b, c, d DEFAULT 'D', e DEFAULT 'E'); + } + + do_then_apply_sql { + INSERT INTO t8 VALUES(1, 2, 3); + INSERT INTO t8 VALUES(4, 5, 6); + } + do_execsql_test $tn.7.2.1 { + SELECT * FROM t8 + } {1 2 3 4 5 6} + do_execsql_test -db db2 $tn.7.2.2 { + SELECT * FROM t8 + } {1 2 3 D E 4 5 6 D E} + + do_then_apply_sql { + UPDATE t8 SET c=45 WHERE a=4; + } + do_execsql_test $tn.7.3.1 { + SELECT * FROM t8 + } {1 2 3 4 5 45} + do_execsql_test -db db2 $tn.7.3.2 { + SELECT * FROM t8 + } {1 2 3 D E 4 5 45 D E} + + #----------------------------------------------------------------------- + reset_test + do_execsql_test $tn.8.0 { + CREATE TABLE t9(a PRIMARY KEY, b, c, d, e, f, g, h); + } + do_execsql_test -db db2 $tn.8.1 { + CREATE TABLE t9(a PRIMARY KEY, b, c, d, e, f, g, h, i, j, k, l); + } + do_then_apply_sql { + INSERT INTO t9 VALUES(1, 2, 3, 4, 5, 6, 7, 8); + } + do_then_apply_sql { + UPDATE t9 SET h=450 WHERE a=1 + } + do_execsql_test -db db2 $tn.8.2 { + SELECT * FROM t9 + } {1 2 3 4 5 6 7 450 {} {} {} {}} + do_then_apply_sql { + UPDATE t9 SET h=NULL + } + do_execsql_test -db db2 $tn.8.2 { + SELECT * FROM t9 + } {1 2 3 4 5 6 7 {} {} {} {} {}} }] } catch { db close } catch { db2 close } Index: ext/session/sqlite3session.c ================================================================== --- ext/session/sqlite3session.c +++ ext/session/sqlite3session.c @@ -1497,10 +1497,12 @@ ){ sqlite3_session *pSession; int nDb = sqlite3Strlen30(zDb); assert( sqlite3_mutex_held(db->mutex) ); + (void)iKey1; + (void)iKey2; for(pSession=(sqlite3_session *)pCtx; pSession; pSession=pSession->pNext){ SessionTable *pTab; /* If this session is attached to a different database ("main", "temp" @@ -1573,10 +1575,11 @@ static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); } static int sessionDiffDepth(void *pCtx){ + (void)pCtx; return 0; } /* ** Install the diff hooks on the session object passed as the only @@ -1646,11 +1649,10 @@ return zRet; } static char *sessionSelectFindNew( - int nCol, const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ const char *zTbl, /* Table name */ const char *zExpr ){ @@ -1670,11 +1672,11 @@ const char *zDb1, const char *zDb2, char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(pTab->nCol, zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew(zDb1, zDb2, pTab->zName,zExpr); if( zStmt==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -4187,11 +4189,10 @@ ** If the iterator currently points to an INSERT record, bind values from the ** new.* record to the SELECT statement. Or, if it points to a DELETE or ** UPDATE, bind values from the old.* record. */ static int sessionSeekToRow( - sqlite3 *db, /* Database handle */ sqlite3_changeset_iter *pIter, /* Changeset iterator */ u8 *abPK, /* Primary key flags array */ sqlite3_stmt *pSelect /* SELECT statement from sessionSelectRow() */ ){ int rc; /* Return code */ @@ -4317,11 +4318,11 @@ assert( SQLITE_CHANGESET_CONFLICT+1==SQLITE_CHANGESET_CONSTRAINT ); assert( SQLITE_CHANGESET_DATA+1==SQLITE_CHANGESET_NOTFOUND ); /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ @@ -4491,11 +4492,11 @@ assert( op==SQLITE_INSERT ); if( p->bStat1 ){ /* Check if there is a conflicting row. For sqlite_stat1, this needs ** to be done using a SELECT, as there is no PRIMARY KEY in the ** database schema to throw an exception if a duplicate is inserted. */ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); if( rc==SQLITE_ROW ){ rc = SQLITE_CONSTRAINT; sqlite3_reset(p->pSelect); } } Index: ext/session/test_session.c ================================================================== --- ext/session/test_session.c +++ ext/session/test_session.c @@ -1299,11 +1299,19 @@ Tcl_SetObjResult(interp, objv[1]); return TCL_OK; } /* +** Run some sanity checks on the changeset in nChangeset byte buffer +** pChangeset. If any fail, return a non-zero value and, optionally, +** set output variable (*pzErr) to point to a buffer containing an +** English language error message describing the problem. In this +** case it is the responsibility of the caller to free the buffer +** using sqlite3_free(). ** +** Or, if the changeset appears to be well-formed, this function +** returns SQLITE_OK and sets (*pzErr) to NULL. */ static int sqlite3_test_changeset( int nChangeset, void *pChangeset, char **pzErr Index: ext/wasm/GNUmakefile ================================================================== --- ext/wasm/GNUmakefile +++ ext/wasm/GNUmakefile @@ -12,16 +12,32 @@ # default, all = build in dev mode # # o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level indicated # by the target name. Rebuild is necessary for all components to get # the desired optimization level. +# +# quick, q = do just a minimal build (sqlite3.js/wasm, tester1) for +# faster development-mode turnaround. +# +# qo2, qoz = a combination of quick+o2/oz. # # dist = create end user deliverables. Add dist.build=oX to build # with a specific optimization level, where oX is one of the -# above-listed o? target names. +# above-listed o? or qo? target names. +# +# snapshot = like dist, but uses a zip file name which clearly +# marks it as a prerelease/snapshot build. # # clean = clean up +# +# Required tools beyond those needed for the canonical builds: +# +# - Emscripten SDK: https://emscripten.org/docs/getting_started/downloads.html +# - The bash shell +# - GNU make, GNU sed, GNU awk, GNU grep (all in the $PATH) +# - wasm-strip for release builds: https://github.com/WebAssembly/wabt +# - InfoZip for 'dist' zip file ######################################################################## SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- @@ -69,21 +85,23 @@ dir.api := api dir.jacc := jaccwabyt dir.common := common dir.fiddle := fiddle dir.tool := $(dir.top)/tool +CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ + ######################################################################## # dir.dout = output dir for deliverables. # -# MAINTENANCE REMINDER: the output .js and .wasm files of emcc must be -# in _this_ dir, rather than a subdir, or else parts of the generated -# code get confused and cannot load property. Specifically, when X.js -# loads X.wasm, whether or not X.js uses the correct path for X.wasm -# depends on how it's loaded: an HTML script tag will resolve it -# intuitively, whereas a Worker's call to importScripts() will not. -# That's a fundamental incompatibility with how URL resolution in -# JS happens between those two contexts. See: +# MAINTENANCE REMINDER: the output .js and .wasm files of certain emcc +# buildables must be in _this_ dir, rather than a subdir, or else +# parts of the generated code get confused and cannot load +# property. Specifically, when X.js loads X.wasm, whether or not X.js +# uses the correct path for X.wasm depends on how it's loaded: an HTML +# script tag will resolve it intuitively, whereas a Worker's call to +# importScripts() will not. That's a fundamental incompatibility with +# how URL resolution in JS happens between those two contexts. See: # # https://zzz.buzz/2017/03/14/relative-uris-in-web-development/ # # We unfortunately have no way, from Worker-initiated code, to # automatically resolve the path from X.js to X.wasm. @@ -102,17 +120,16 @@ endif ifeq (,$(wildcard $(dir.tmp))) dir._tmp := $(shell mkdir -p $(dir.tmp)) endif -cflags.common := -I. -I.. -I$(dir.top) -CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ -emcc.WASM_BIGINT ?= 1 sqlite3.c := $(dir.top)/sqlite3.c sqlite3.h := $(dir.top)/sqlite3.h +# Most SQLITE_OPT flags are set in sqlite3-wasm.c but we need them +# made explicit here for building speedtest1.c. SQLITE_OPT = \ - -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ -DSQLITE_ENABLE_STMTVTAB \ -DSQLITE_ENABLE_DBPAGE_VTAB \ @@ -128,20 +145,107 @@ -DSQLITE_TEMP_STORE=3 \ -DSQLITE_OS_KV_OPTIONAL=1 \ '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ -DSQLITE_USE_URI=1 \ -DSQLITE_WASM_ENABLE_C_TESTS -# ^^^ most flags are set in sqlite3-wasm.c but we need them -# made explicit here for building speedtest1.c. + +$(sqlite3.c) $(sqlite3.h): + $(MAKE) -C $(dir.top) sqlite3.c + +.PHONY: clean distclean +clean: + -rm -f $(CLEAN_FILES) +distclean: clean + -rm -f $(DISTCLEAN_FILES) -ifneq (,$(filter release,$(MAKECMDGOALS))) -emcc_opt ?= -Oz -flto +ifeq (release,$(filter release,$(MAKECMDGOALS))) + ifeq (,$(wasm-strip)) + $(error Cannot make release-quality binary because wasm-strip is not available. \ + See notes in the warning above) + endif else + $(info Development build. Use '$(MAKE) release' for a smaller release build.) +endif + +# bin.version-info = binary to output various sqlite3 version info for +# embedding in the JS files and in building the distribution zip file. +# It must NOT be in $(dir.tmp) because we need it to survive the +# cleanup process for the dist build to work properly. +bin.version-info := $(dir.wasm)/version-info +$(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) + $(CC) -O0 -I$(dir.top) -o $@ $< +DISTCLEAN_FILES += $(bin.version-info) + +# bin.stripcomments is used for stripping C/C++-style comments from JS +# files. The JS files contain large chunks of documentation which we +# don't need for all builds. That app's -k flag is of particular +# importance here, as it allows us to retain the opening comment +# blocks, which contain the license header and version info. +bin.stripccomments := $(dir.tool)/stripccomments +$(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) + $(CC) -o $@ $< +DISTCLEAN_FILES += $(bin.stripccomments) + + +######################################################################## +# C-PP.FILTER: a $(call)able to transform $(1) to $(2) via ./c-pp -f +# $(1) ... +# +# Historical notes: +# +# - We first attempted to use gcc and/or clang to preprocess JS files +# in the same way we would normally do C files, but C-specific quirks +# of each makes that untennable. +# +# - We implemented c-pp.c (the C-Minus Pre-processor) as a custom +# generic/file-format-agnostic preprocessor to enable us to pack +# code for different target builds into the same JS files. Most +# notably, some ES6 module (a.k.a. ESM) features cannot legally be +# referenced at all in non-ESM code, e.g. the "import" and "export" +# keywords. This preprocessing step permits us to swap out sections +# of code where necessary for ESM and non-ESM (a.k.a. vanilla JS) +# require different implementations. The alternative to such +# preprocessing, would be to have separate source files for ES6 +# builds, which would have a higher maintenance burden than c-pp.c +# seems likely to. +# +# c-pp.c was written specifically for the sqlite project's JavaScript +# builds but is maintained as a standalone project: +# https://fossil.wanderinghorse.net/r/c-pp +bin.c-pp := ./c-pp +$(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) + $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ + -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ + -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=3 +define C-PP.FILTER +# Create $2 from $1 using $(bin.c-pp) +# $1 = Input file: c-pp -f $(1).js +# $2 = Output file: c-pp -o $(2).js +# $3 = optional c-pp -D... flags +$(2): $(1) $$(MAKEFILE) $$(bin.c-pp) + $$(bin.c-pp) -f $(1) -o $$@ $(3) +CLEAN_FILES += $(2) +endef +c-pp.D.vanilla ?= +c-pp.D.esm ?= -Dtarget=es6-module +# /end C-PP.FILTER +######################################################################## + + +# cflags.common = C compiler flags for all builds +cflags.common := -I. -I.. -I$(dir.top) +# emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API +# disables certain features if BigInt is not enabled and such builds +# _are not tested_ on any regular basis. +emcc.WASM_BIGINT ?= 1 + +# emcc_opt = optimization-related flags. These are primarily used by +# the various oX targets. build times for -O levels higher than 0 are +# painful at dev-time. emcc_opt ?= -O0 -# ^^^^ build times for -O levels higher than 0 are painful at -# dev-time. -endif + # When passing emcc_opt from the CLI, += and re-assignment have no # effect, so emcc_opt+=-g3 doesn't work. So... emcc_opt_full := $(emcc_opt) -g3 # ^^^ ALWAYS use -g3. See below for why. # @@ -163,81 +267,67 @@ # symbol names. -Oz gives ever-so-slightly better compression than # -Os: not quite 1% in some completely unscientific tests. Runtime # speed for the unit tests is all over the place either way so it's # difficult to say whether -Os gives any speed benefit over -Oz. # -# (Much later: -O2 consistently gives the best speeds.) +# Much practice has demonstrated that -O2 consistently gives the best +# runtime speeds, but not by a large enough factor to rule out use of +# -Oz when small deliverable size is a priority. ######################################################################## - -$(sqlite3.c) $(sqlite3.h): - $(MAKE) -C $(dir.top) sqlite3.c - -.PHONY: clean distclean -clean: - -rm -f $(CLEAN_FILES) -distclean: clean - -rm -f $(DISTCLEAN_FILES) - -ifeq (release,$(filter release,$(MAKECMDGOALS))) - ifeq (,$(wasm-strip)) - $(error Cannot make release-quality binary because wasm-strip is not available. \ - See notes in the warning above) - endif -else - $(info Development build. Use '$(MAKE) release' for a smaller release build.) -endif - -bin.version-info := $(dir.wasm)/version-info -# ^^^^ NOT in $(dir.tmp) because we need it to survive the cleanup -# process for the dist build to work properly. -$(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) - $(CC) -O0 -I$(dir.top) -o $@ $< -DISTCLEAN_FILES += $(bin.version-info) - -bin.stripccomments := $(dir.tool)/stripccomments -$(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) - $(CC) -o $@ $< -DISTCLEAN_FILES += $(bin.stripccomments) - +# EXPORTED_FUNCTIONS.* = files for use with Emscripten's +# -sEXPORTED_FUNCTION flag. EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) - cat $(EXPORTED_FUNCTIONS.api.in) > $@ + cp $(EXPORTED_FUNCTIONS.api.in) $@ +# sqlite3-license-version.js = generated JS file with the license +# header and version info. sqlite3-license-version.js := $(dir.tmp)/sqlite3-license-version.js +# sqlite3-license-version-header.js = JS file containing only the +# license header. sqlite3-license-version-header.js := $(dir.api)/sqlite3-license-version-header.js +# sqlite3-api-build-version.js = generated JS file which populates the +# sqlite3.version object using $(bin.version-info). sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js -# sqlite3-api.jses = the list of JS files which make up $(sqlite3-api.js), in -# the order they need to be assembled. +# sqlite3-api.jses = the list of JS files which make up +# $(sqlite3-api.js.in), in the order they need to be assembled. sqlite3-api.jses := $(sqlite3-license-version.js) sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js sqlite3-api.jses += $(sqlite3-api-build-version.js) sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js +sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # "External" API files which are part of our distribution # but not part of the sqlite3-api.js amalgamation. SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js sqlite3-worker1.js := $(dir.api)/sqlite3-worker1.js sqlite3-worker1-promiser.js := $(dir.api)/sqlite3-worker1-promiser.js -define CP_XAPI +# COPY_XAPI = a $(call)able function to copy $1 to $(dir.dout), where +# $1 must be one of the "external" JS API files. +define COPY_XAPI sqlite3-api.ext.jses += $$(dir.dout)/$$(notdir $(1)) $$(dir.dout)/$$(notdir $(1)): $(1) $$(MAKEFILE) cp $$< $$@ endef $(foreach X,$(SOAP.js) $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js),\ - $(eval $(call CP_XAPI,$(X)))) -all: $(sqlite3-api.ext.jses) + $(eval $(call COPY_XAPI,$(X)))) +all quick: $(sqlite3-api.ext.jses) +q: quick -sqlite3-api.js := $(dir.tmp)/sqlite3-api.js -$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) +# sqlite3-api.js.in = the generated sqlite3-api.js before it gets +# preprocessed. It contains all of $(sqlite3-api.jses) but none of the +# Emscripten-specific headers and footers. +sqlite3-api.js.in := $(dir.tmp)/sqlite3-api.c-pp.js +$(sqlite3-api.js.in): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @for i in $(sqlite3-api.jses); do \ echo "/* BEGIN FILE: $$i */"; \ cat $$i; \ echo "/* END FILE: $$i */"; \ @@ -250,36 +340,12 @@ echo -n ' sqlite3.version = '; \ $(bin.version-info) --json; \ echo ';'; \ echo '});'; \ } > $@ - -######################################################################## -# --post-js and --pre-js are emcc flags we use to append/prepend JS to -# the generated emscripten module file. -pre-js.js := $(dir.api)/pre-js.js -post-js.js := $(dir.tmp)/post-js.js -post-jses := \ - $(dir.api)/post-js-header.js \ - $(sqlite3-api.js) \ - $(dir.api)/post-js-footer.js -$(post-js.js): $(post-jses) $(MAKEFILE) - @echo "Making $@..." - @for i in $(post-jses); do \ - echo "/* BEGIN FILE: $$i */"; \ - cat $$i; \ - echo "/* END FILE: $$i */"; \ - done > $@ -extern-post-js.js := $(dir.api)/extern-post-js.js -extern-pre-js.js := $(dir.api)/extern-pre-js.js -pre-post-common.flags := \ - --post-js=$(post-js.js) \ - --extern-post-js=$(extern-post-js.js) \ - --extern-pre-js=$(sqlite3-license-version.js) -pre-post-jses.deps := $(post-js.js) \ - $(extern-post-js.js) $(extern-pre-js.js) $(sqlite3-license-version.js) -$(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) $(MAKEFILE) +$(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ + $(MAKEFILE) @echo "Making $@..."; { \ cat $(sqlite3-license-version-header.js); \ echo '/*'; \ echo '** This code was built from sqlite3 version...'; \ echo "** "; \ @@ -287,55 +353,120 @@ -e '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo '*/'; \ } > $@ ######################################################################## -# call-make-pre-js creates rules for pre-js-$(1).js. $1 = the base -# name of the JS file on whose behalf this pre-js is for. +# --post-js and --pre-js are emcc flags we use to append/prepend JS to +# the generated emscripten module file. The following rules generate +# various versions of those files for the vanilla and ESM builds. +pre-js.js.in := $(dir.api)/pre-js.c-pp.js +pre-js.js.esm := $(dir.tmp)/pre-js.esm.js +pre-js.js.vanilla := $(dir.tmp)/pre-js.vanilla.js +$(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.esm),$(c-pp.D.esm))) +post-js.js.in := $(dir.tmp)/post-js.c-pp.js +post-js.js.vanilla := $(dir.tmp)/post-js.vanilla.js +post-js.js.esm := $(dir.tmp)/post-js.esm.js +post-jses.js := \ + $(dir.api)/post-js-header.js \ + $(sqlite3-api.js.in) \ + $(dir.api)/post-js-footer.js +$(post-js.js.in): $(post-jses.js) $(MAKEFILE) + @echo "Making $@..." + @for i in $(post-jses.js); do \ + echo "/* BEGIN FILE: $$i */"; \ + cat $$i; \ + echo "/* END FILE: $$i */"; \ + done > $@ +$(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.FILTER,$(post-js.js.in),$(post-js.js.esm),$(c-pp.D.esm))) + +# extern-post-js* and extern-pre-js* are files for use with +# Emscripten's --extern-pre-js and --extern-post-js flags. These +# rules make different copies for the vanilla and ESM builds. +extern-post-js.js.in := $(dir.api)/extern-post-js.c-pp.js +extern-post-js.js.vanilla := $(dir.tmp)/extern-post-js.vanilla.js +extern-post-js.js.esm := $(dir.tmp)/extern-post-js.esm.js +$(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.vanilla),$(c-pp.D.vanilla))) +$(eval $(call C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.esm),$(c-pp.D.esm))) +extern-pre-js.js := $(dir.api)/extern-pre-js.js + +# Emscripten flags for --[extern-][pre|post]-js=... for the +# various builds. +pre-post-common.flags := \ + --extern-pre-js=$(sqlite3-license-version.js) +pre-post-common.flags.vanilla := \ + $(pre-post-common.flags) \ + --post-js=$(post-js.js.vanilla) \ + --extern-post-js=$(extern-post-js.js.vanilla) +pre-post-common.flags.esm := \ + $(pre-post-common.flags) \ + --post-js=$(post-js.js.esm) \ + --extern-post-js=$(extern-post-js.js.esm) + +# pre-post-jses.deps.* = a list of dependencies for the +# --[extern-][pre/post]-js files. +pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) +pre-post-jses.deps.vanilla := $(pre-post-jses.deps.common) \ + $(post-js.js.vanilla) $(extern-post-js.js.vanilla) +pre-post-jses.deps.esm := $(pre-post-jses.deps.common) \ + $(post-js.js.esm) $(extern-post-js.js.esm) + +######################################################################## +# call-make-pre-js is a $(call)able which creates rules for +# pre-js-$(1).js. $1 = the base name of the JS file on whose behalf +# this pre-js is for. $2 is the build mode: one of (vanilla, esm). +# This sets up --[extern-][pre/post]-js flags in +# $(pre-post-$(1).flags.$(2)) and dependencies in +# $(pre-post-$(1).deps.$(2)). define call-make-pre-js -pre-post-$(1).flags ?= -$$(dir.tmp)/pre-js-$(1).js: $$(pre-js.js) $$(MAKEFILE) - cp $$(pre-js.js) $$@ +pre-post-$(1).flags.$(2) ?= +$$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(2)) $$(MAKEFILE) + cp $$(pre-js.js.$(2)) $$@ @if [ sqlite3-wasmfs = $(1) ]; then \ echo "delete Module[xNameOfInstantiateWasm] /*for WASMFS build*/;"; \ elif [ sqlite3 != $(1) ]; then \ echo "Module[xNameOfInstantiateWasm].uri = '$(1).wasm';"; \ fi >> $$@ -pre-post-$(1).deps := $$(pre-post-jses.deps) $$(dir.tmp)/pre-js-$(1).js -pre-post-$(1).flags += --pre-js=$$(dir.tmp)/pre-js-$(1).js +pre-post-$(1).deps.$(2) := \ + $$(pre-post-jses.deps.$(2)) \ + $$(dir.tmp)/pre-js-$(1)-$(2).js +pre-post-$(1).flags.$(2) += \ + $$(pre-post-common.flags.$(2)) \ + --pre-js=$$(dir.tmp)/pre-js-$(1)-$(2).js endef -#$(error $(call call-make-pre-js,sqlite3-wasmfs)) # /post-js and pre-js ######################################################################## ######################################################################## # emcc flags for .c/.o/.wasm/.js. emcc.flags := -#emcc.flags += -v # _very_ loud but also informative about what it's doing -# -g3 is needed to keep -O2 and higher from creating broken JS via -# minification. +ifeq (1,$(emcc.verbose)) +emcc.flags += -v +# -v is _very_ loud but also informative about what it's doing +endif ######################################################################## # emcc flags for .c/.o. emcc.cflags := emcc.cflags += -std=c99 -fPIC -# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs. +# -------------^^^^^^^^ we need c99 for $(sqlite3-wasm.c). emcc.cflags += -I. -I$(dir.top) ######################################################################## -# emcc flags specific to building the final .js/.wasm file... +# emcc flags specific to building .js/.wasm files... emcc.jsflags := -fPIC emcc.jsflags += --minify 0 emcc.jsflags += --no-entry +emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) emcc.jsflags += -sMODULARIZE emcc.jsflags += -sSTRICT_JS emcc.jsflags += -sDYNAMIC_EXECUTION=0 emcc.jsflags += -sNO_POLYFILL emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.api) emcc.exportedRuntimeMethods := \ - -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory - # FS ==> stdio/POSIX I/O proxies + -sEXPORTED_RUNTIME_METHODS=wasmMemory # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY emcc.environment := -sENVIRONMENT=web,worker @@ -364,37 +495,52 @@ emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) # /INITIAL_MEMORY ######################################################################## emcc.jsflags += $(emcc.environment) -#emcc.jsflags += -sTOTAL_STACK=4194304 - +emcc.jsflags += -sSTACK_SIZE=512KB +# ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 5MB to +# a mere 64KB, which leads to silent memory corruption via the kvvfs +# VFS, which requires twice that for its xRead() and xWrite() methods. +######################################################################## +# $(sqlite3.js.init-func) is the name Emscripten assigns our exported +# module init/load function. This symbol name is hard-coded in +# $(extern-post-js.js) as well as in numerous docs. +# +# "sqlite3InitModule" is the symbol we document for client use, so +# that's the symbol name which must be exported, whether it comes from +# Emscripten or our own code in extern-post-js.js. +# +# That said... we can change $(sqlite3.js.init-func) as long as the +# name "sqlite3InitModule" is the one which gets exposed via the +# resulting JS files. That can be accomplished via +# extern-post-js.js. However... using a temporary symbol name here +# and then adding sqlite3InitModule() ourselves results in 2 global +# symbols: we cannot "delete" the Emscripten-defined +# $(sqlite3.js.init-func) because it's declared with "var". sqlite3.js.init-func := sqlite3InitModule -# ^^^^ $(sqlite3.js.init-func) symbol name is hard-coded in -# $(extern-post-js.js) as well as in numerous docs. If changed, it -# needs to be globally modified in *.js and all related documentation. - emcc.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. #emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() #emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS #emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API -#emcc.jsflags += -sABORTING_MALLOC +#emcc.jsflags += -sABORTING_MALLOC # only for experimentation emcc.jsflags += -sALLOW_TABLE_GROWTH -# -sALLOW_TABLE_GROWTH is required for installing new SQL UDFs +# ^^^^ -sALLOW_TABLE_GROWTH is required for installing new SQL UDFs emcc.jsflags += -Wno-limited-postlink-optimizations -# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. -#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why -# https://lld.llvm.org/WebAssembly.html -emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 +# ^^^^ emcc likes to warn when we have "limited optimizations" via the +# -g3 flag. +# emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why. + +# Re. undefined symbol handling, see: https://lld.llvm.org/WebAssembly.html +emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=1 emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --allow-undefined #emcc.jsflags += --import-undefined #emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic #emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all -emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) ######################################################################## # -sMEMORY64=1 fails to load, erroring with: # invalid memory limits flags 0x5 # (enable via --experimental-wasm-memory64) @@ -402,66 +548,111 @@ # ^^^^ MEMORY64=2 builds and loads but dies when we do things like: # # new Uint8Array(wasm.heap8u().buffer, ptr, n) # # because ptr is now a BigInt, so is invalid for passing to arguments -# which have strict must-be-a-Number requirements. +# which have strict must-be-a-Number requirements. That aspect will +# make any eventual port to 64-bit address space extremely painful, as +# such constructs are found all over the place in the source code. ######################################################################## ######################################################################## # -sSINGLE_FILE: -# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js#L1704 -# -sSINGLE_FILE=1 would be really nice but we have to build with -g3 +# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js +# +# -sSINGLE_FILE=1 would be _really_ nice but we have to build with -g3 # for -O2 and higher to work (else minification breaks the code) and # cannot wasm-strip the binary before it gets encoded into the JS -# file. The result is that the generated JS file is, because of the -g3 -# debugging info, _huge_. -######################################################################## - -######################################################################## -# AN EXPERIMENT: undocumented Emscripten feature: if the target file -# extension is "mjs", it defaults to ES6 module builds: -# https://github.com/emscripten-core/emscripten/issues/14383 -ifeq (,$(filter esm,$(MAKECMDGOALS))) -sqlite3.js.ext := js -else -esm.deps := $(filter-out esm,$(MAKECMDGOALS)) -esm: $(if $(esm.deps),$(esm.deps),all) -sqlite3.js.ext := mjs -endif -# /esm -######################################################################## -sqlite3.js := $(dir.dout)/sqlite3.$(sqlite3.js.ext) +# file. The result is that the generated JS file is, because of the +# -g3 debugging info, _huge_. +######################################################################## + +sqlite3.js := $(dir.dout)/sqlite3.js +sqlite3.mjs := $(dir.dout)/sqlite3.mjs +# Undocumented Emscripten feature: if the target file extension is +# "mjs", it defaults to ES6 module builds: +# https://github.com/emscripten-core/emscripten/issues/14383 sqlite3.wasm := $(dir.dout)/sqlite3.wasm sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter # (predictably) results in a slightly faster binary, but we're close # enough to the target speed requirements that the 500ms makes a # difference. Thus we build all binaries against sqlite3-wasm.c # instead of building a shared copy of sqlite3-wasm.o. -$(eval $(call call-make-pre-js,sqlite3)) -$(sqlite3.js): -$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ - $(EXPORTED_FUNCTIONS.api) \ - $(pre-post-sqlite3.deps) +$(eval $(call call-make-pre-js,sqlite3,vanilla)) +$(eval $(call call-make-pre-js,sqlite3,esm)) +$(sqlite3.js) $(sqlite3.mjs): $(MAKEFILE) $(sqlite3-wasm.c) \ + $(EXPORTED_FUNCTIONS.api) +$(sqlite3.js): $(pre-post-sqlite3.deps.vanilla) +$(sqlite3.mjs): $(pre-post-sqlite3.deps.esm) +######################################################################## +# SQLITE3.xJS.RECIPE = the $(call)able recipe body for $(sqlite3.js) +# and $(sqlite3.mjs). $1 = one of (vanilla, esm). +# +# Reminder for ESM builds: even if we use -sEXPORT_ES6=0, emcc _still_ +# adds: +# +# export default $(sqlite3.js.init-func); +# +# when building *.mjs, which is bad because we need to export an +# overwritten version of that function and cannot "export default" +# twice. Because of this, we have to sed $(sqlite3.mjs) to remove the +# _first_ instance (only) of /^export default/. +# +# Upstream RFE: +# https://github.com/emscripten-core/emscripten/issues/18237 +######################################################################## +# SQLITE3.xJS.EXPORT-DEFAULT is part of SQLITE3[-WASMFS].xJS.RECIPE, +# factored into a separate piece to avoid code duplication. $1 is +# the build mode: one of (vanilla, esm). +define SQLITE3.xJS.ESM-EXPORT-DEFAULT +if [ esm = $(1) ]; then \ + echo "Fragile workaround for an Emscripten annoyance. See SQLITE3.xJS.RECIPE."; \ + sed -i -e '0,/^export default/{/^export default/d;}' $@ || exit $$?; \ + if ! grep -q '^export default' $@; then \ + echo "Cannot find export default." 1>&2; \ + exit 1; \ + fi; \ +fi +endef +define SQLITE3.xJS.RECIPE @echo "Building $@ ..." $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ - $(emcc.jsflags) $(pre-post-common.flags) $(pre-post-sqlite3.flags) \ + $(emcc.jsflags) \ + $(pre-post-sqlite3.flags.$(1)) $(emcc.flags.sqlite3.$(1)) \ $(cflags.common) $(SQLITE_OPT) $(sqlite3-wasm.c) + @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(1)) chmod -x $(sqlite3.wasm) $(maybe-wasm-strip) $(sqlite3.wasm) @ls -la $@ $(sqlite3.wasm) +endef +emcc.flags.sqlite3.vanilla := +emcc.flags.sqlite3.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META +$(sqlite3.js): + $(call SQLITE3.xJS.RECIPE,vanilla) +$(sqlite3.mjs): + $(call SQLITE3.xJS.RECIPE,esm) +######################################################################## +# We have to ensure that we do not build both $(sqlite3.js) and +# $(sqlite3.mjs) in parallel because both result in the creation of +# $(sqlite3.wasm). We have no(?) way to build just the .mjs file +# without also building the .wasm file. i.e. we're building +# $(sqlite3.wasm) twice, but that's apparently unavoidable (and +# harmless, just a waste of build time). $(sqlite3.wasm): $(sqlite3.js) -CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) -all: $(sqlite3.js) -wasm: $(sqlite3.js) -# End main Emscripten-based module build +$(sqlite3.mjs): $(sqlite3.js) +CLEAN_FILES += $(sqlite3.js) $(sqlite3.mjs) $(sqlite3.wasm) +all: $(sqlite3.js) $(sqlite3.mjs) +quick: $(sqlite3.js) +quick: $(sqlite3.mjs) # for the sake of the snapshot build +# End main $(sqlite3.js) build ######################################################################## ######################################################################## -# batch-runner.js... +# batch-runner.js is part of one of the test apps which reads in SQL +# dumps generated by $(speedtest1) and executes them. dir.sql := sql speedtest1 := ../../speedtest1 speedtest1.c := ../../test/speedtest1.c speedtest1.sql := $(dir.sql)/speedtest1.sql speedtest1.cliflags := --size 25 --big-transactions @@ -479,34 +670,35 @@ # pieces each time is an unnecessary time sink. batch: batch-runner.list all: batch # end batch-runner.js ######################################################################## -# speedtest1.js... -# speedtest1-common.eflags = emcc flags used by multiple builds of speedtest1 -# speedtest1.eflags = emcc flags used by main build of speedtest1 -speedtest1-common.eflags := $(emcc_opt_full) -speedtest1.eflags := -speedtest1.eflags += -sENVIRONMENT=web -speedtest1.eflags += -sALLOW_MEMORY_GROWTH -speedtest1.eflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) -speedtest1-common.eflags += -sINVOKE_RUN=0 -speedtest1-common.eflags += --no-entry -#speedtest1-common.eflags += -flto -speedtest1-common.eflags += -sABORTING_MALLOC -speedtest1-common.eflags += -sSTRICT_JS -speedtest1-common.eflags += -sMODULARIZE -speedtest1-common.eflags += -Wno-limited-postlink-optimizations +# Wasmified speedtest1 is our primary benchmarking tool. +# +# emcc.speedtest1.common = emcc flags used by multiple builds of speedtest1 +# emcc.speedtest1 = emcc flags used by main build of speedtest1 +emcc.speedtest1.common := $(emcc_opt_full) +emcc.speedtest1 := +emcc.speedtest1 += -sENVIRONMENT=web +emcc.speedtest1 += -sALLOW_MEMORY_GROWTH +emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) +emcc.speedtest1.common += -sINVOKE_RUN=0 +emcc.speedtest1.common += --no-entry +#emcc.speedtest1.common += -flto +emcc.speedtest1.common += -sABORTING_MALLOC +emcc.speedtest1.common += -sSTRICT_JS +emcc.speedtest1.common += -sMODULARIZE +emcc.speedtest1.common += -Wno-limited-postlink-optimizations EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) -speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) -speedtest1-common.eflags += $(emcc.exportedRuntimeMethods) -speedtest1-common.eflags += -sALLOW_TABLE_GROWTH -speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0 -speedtest1-common.eflags += --minify 0 -speedtest1-common.eflags += -sEXPORT_NAME=$(sqlite3.js.init-func) -speedtest1-common.eflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) -speedtest1-common.eflags += $(pre-post-common.flags) +emcc.speedtest1.common += -sSTACK_SIZE=512KB +emcc.speedtest1.common += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) +emcc.speedtest1.common += $(emcc.exportedRuntimeMethods) +emcc.speedtest1.common += -sALLOW_TABLE_GROWTH +emcc.speedtest1.common += -sDYNAMIC_EXECUTION=0 +emcc.speedtest1.common += --minify 0 +emcc.speedtest1.common += -sEXPORT_NAME=$(sqlite3.js.init-func) +emcc.speedtest1.common += -sWASM_BIGINT=$(emcc.WASM_BIGINT) speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get # this error from emscripten: # @@ -526,21 +718,21 @@ $(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) @echo "Making $@ ..." @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js -speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) -speedtest1.cflags := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM +speedtest1.wasm := $(dir.dout)/speedtest1.wasm +cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.c) -$(eval $(call call-make-pre-js,speedtest1)) +$(eval $(call call-make-pre-js,speedtest1,vanilla)) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ - $(pre-post-speedtest1.deps) \ + $(pre-post-speedtest1.deps.vanilla) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \ - $(pre-post-speedtest1.flags) \ + $(emcc.speedtest1) $(emcc.speedtest1.common) \ + $(cflags.speedtest1) $(pre-post-speedtest1.flags.vanilla) \ $(SQLITE_OPT) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cses) -lm $(maybe-wasm-strip) $(speedtest1.wasm) ls -la $@ $(speedtest1.wasm) @@ -548,49 +740,82 @@ speedtest1: $(speedtest1.js) all: speedtest1 CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) # end speedtest1.js ######################################################################## + +######################################################################## +# tester1 is the main unit and regression test application and needs +# to be able to run in 4 separate modes to cover the primary +# client-side use cases: +# +# 1) Load sqlite3 in the main UI thread of a conventional script. +# 2) Load sqlite3 in a conventional Worker thread. +# 3) Load sqlite3 as an ES6 module (ESM) in the main thread. +# 4) Load sqlite3 as an ESM worker. (Not all browsers support this.) +# +# To that end, we require two separate builds of tester1.js: +# +# tester1.js: cases 1 and 2 +# tester1.mjs: cases 3 and 4 +# +# To create those, we filter tester1.c-pp.js with $(bin.c-pp)... +$(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1.js)) +$(eval $(call C-PP.FILTER,tester1.c-pp.js,tester1.mjs,$(c-pp.D.esm))) +$(eval $(call C-PP.FILTER,tester1.c-pp.html,tester1.html)) +$(eval $(call C-PP.FILTER,tester1.c-pp.html,tester1-esm.html,$(c-pp.D.esm))) +tester1: tester1.js tester1.mjs tester1.html tester1-esm.html +all quick: tester1 ######################################################################## # Convenience rules to rebuild with various -Ox levels. Much # experimentation shows -O2 to be the clear winner in terms of speed. # Note that build times with anything higher than -O0 are somewhat # painful. .PHONY: o0 o1 o2 o3 os oz -o-xtra := -flto +o-xtra := +#o-xtra ?= -flto # ^^^^ -flto can have a considerably performance boost at -O0 but -# doubles the build time and seems to have negligible effect on -# higher optimization levels. +# doubles the build time and seems to have negligible, if any, effect +# on higher optimization levels. o0: clean $(MAKE) -e "emcc_opt=-O0" o1: clean $(MAKE) -e "emcc_opt=-O1 $(o-xtra)" o2: clean - $(MAKE) -e "emcc_opt=-O2 $(o-xtra)" + $(MAKE) -j2 -e "emcc_opt=-O2 $(o-xtra)" +qo2: clean + $(MAKE) -j2 -e "emcc_opt=-O2 $(o-xtra)" quick o3: clean $(MAKE) -e "emcc_opt=-O3 $(o-xtra)" os: clean @echo "WARNING: -Os can result in a build with mysteriously missing pieces!" $(MAKE) -e "emcc_opt=-Os $(o-xtra)" oz: clean - $(MAKE) -e "emcc_opt=-Oz $(o-xtra)" + $(MAKE) -j2 -e "emcc_opt=-Oz $(o-xtra)" +qoz: clean + $(MAKE) -j2 -e "emcc_opt=-Oz $(o-xtra)" quick ######################################################################## # Sub-makes... +# sqlite.org/fiddle application... include fiddle.make # Only add wasmfs if wasmfs.enable=1 or we're running (dist)clean +ifneq (,$(filter wasmfs,$(MAKECMDGOALS))) +wasmfs.enable ?= 1 +else wasmfs.enable ?= $(if $(filter %clean,$(MAKECMDGOALS)),1,0) +endif ifeq (1,$(wasmfs.enable)) # wasmfs build disabled 2022-10-19 per /chat discussion. # OPFS-over-wasmfs was initially a stopgap measure and a convenient # point of comparison for the OPFS sqlite3_vfs's performance, but it # currently doubles our deliverables and build maintenance burden for -# little, if any, benefit. +# little benefit. # ######################################################################## # Some platforms do not support the WASMFS build. Raspberry Pi OS is one # of them. As such platforms are discovered, add their (uname -m) name # to PLATFORMS_WITH_NO_WASMFS to exclude the wasmfs build parts. @@ -605,20 +830,16 @@ endif endif # /wasmfs ######################################################################## -######################################################################## -# Create deliverables: -ifneq (,$(filter dist,$(MAKECMDGOALS))) -include dist.make -endif - ######################################################################## # Push files to public wasm-testing.sqlite.org server -wasm-testing.include = $(dir.dout) *.js *.html \ - batch-runner.list $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) +wasm-testing.include = *.js *.mjs *.html \ + ./tests \ + batch-runner.list \ + $(dir.dout) $(dir.sql) $(dir.common) $(dir.fiddle) $(dir.jacc) wasm-testing.exclude = sql/speedtest1.sql wasm-testing.dir = /jail/sites/wasm-testing wasm-testing.dest ?= wasm-testing:$(wasm-testing.dir) # ---------------------^^^^^^^^^^^^ ssh alias .PHONY: push-testing @@ -631,25 +852,37 @@ echo "SSH failed: it's likely that stale content will be served via old gzip files." ######################################################################## # If we find a copy of the sqlite.org/wasm docs checked out, copy # certain files over to it, noting that some need automatable edits... -WDOCS.home ?= ../../../wdoc +wasm.docs.home ?= ../../../wasm +wasm.docs.found = $(if $(wildcard $(wasm.docs.home)/api-index.md),\ + $(wildcard $(wasm.docs.home)),) .PHONY: update-docs -ifneq (,$(wildcard $(WDOCS.home)/api-index.md)) -WDOCS.jswasm := $(WDOCS.home)/jswasm +ifeq (,$(wasm.docs.found)) +update-docs: + @echo "Cannot find wasm docs checkout."; \ + echo "Pass wasm.docs.home=/path/to/wasm/docs/checkout or edit this makefile to suit."; \ + exit 127 +else +wasm.docs.jswasm := $(wasm.docs.home)/jswasm update-docs: $(bin.stripccomments) $(sqlite3.js) $(sqlite3.wasm) @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!" - cp $(sqlite3.wasm) $(WDOCS.jswasm)/. + cp $(sqlite3.wasm) $(wasm.docs.jswasm)/. $(bin.stripccomments) -k -k < $(sqlite3.js) \ - | sed -e '/^[ \t]*$$/d' > $(WDOCS.jswasm)/sqlite3.js - cp demo-123.js demo-123.html demo-123-worker.html $(WDOCS.home) + | sed -e '/^[ \t]*$$/d' > $(wasm.docs.jswasm)/sqlite3.js + cp demo-123.js demo-123.html demo-123-worker.html $(wasm.docs.home) sed -n -e '/EXTRACT_BEGIN/,/EXTRACT_END/p' \ - module-symbols.html > $(WDOCS.home)/module-symbols.html -else -update-docs: - @echo "Cannot find wasm docs checkout."; \ - echo "Pass WDOCS.home=/path/to/wasm/docs/checkout or edit this makefile to suit."; \ - exit 127 + module-symbols.html > $(wasm.docs.home)/module-symbols.html endif # end /wasm docs ######################################################################## + +######################################################################## +# Create main client downloadable zip file: +ifneq (,$(filter dist snapshot,$(MAKECMDGOALS))) +include dist.make +endif + +# Run local web server for the test/demo pages. +httpd: + althttpd -max-age 1 -enable-sab -page index.html Index: ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api ================================================================== --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -1,18 +1,25 @@ +_malloc +_free +_realloc _sqlite3_aggregate_context _sqlite3_bind_blob _sqlite3_bind_double _sqlite3_bind_int _sqlite3_bind_int64 _sqlite3_bind_null _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index +_sqlite3_bind_pointer _sqlite3_bind_text +_sqlite3_busy_handler +_sqlite3_busy_timeout _sqlite3_changes _sqlite3_changes64 _sqlite3_clear_bindings _sqlite3_close_v2 +_sqlite3_collation_needed _sqlite3_column_blob _sqlite3_column_bytes _sqlite3_column_count _sqlite3_column_count _sqlite3_column_double @@ -19,20 +26,31 @@ _sqlite3_column_int _sqlite3_column_int64 _sqlite3_column_name _sqlite3_column_text _sqlite3_column_type +_sqlite3_column_value _sqlite3_compileoption_get _sqlite3_compileoption_used +_sqlite3_complete +_sqlite3_context_db_handle +_sqlite3_create_collation +_sqlite3_create_collation_v2 _sqlite3_create_function _sqlite3_create_function_v2 +_sqlite3_create_module +_sqlite3_create_module_v2 _sqlite3_create_window_function _sqlite3_data_count _sqlite3_db_filename _sqlite3_db_handle _sqlite3_db_name +_sqlite3_db_status +_sqlite3_declare_vtab _sqlite3_deserialize +_sqlite3_drop_modules +_sqlite3_errcode _sqlite3_errmsg _sqlite3_error_offset _sqlite3_errstr _sqlite3_exec _sqlite3_expanded_sql @@ -39,20 +57,28 @@ _sqlite3_extended_errcode _sqlite3_extended_result_codes _sqlite3_file_control _sqlite3_finalize _sqlite3_free +_sqlite3_get_auxdata _sqlite3_initialize +_sqlite3_keyword_count +_sqlite3_keyword_name +_sqlite3_keyword_check +_sqlite3_last_insert_rowid _sqlite3_libversion _sqlite3_libversion_number +_sqlite3_limit _sqlite3_malloc _sqlite3_malloc64 _sqlite3_msize _sqlite3_open _sqlite3_open_v2 +_sqlite3_overload_function _sqlite3_prepare_v2 _sqlite3_prepare_v3 +_sqlite3_progress_handler _sqlite3_randomness _sqlite3_realloc _sqlite3_realloc64 _sqlite3_reset _sqlite3_result_blob @@ -62,33 +88,104 @@ _sqlite3_result_error_nomem _sqlite3_result_error_toobig _sqlite3_result_int _sqlite3_result_int64 _sqlite3_result_null +_sqlite3_result_pointer +_sqlite3_result_subtype _sqlite3_result_text +_sqlite3_result_zeroblob +_sqlite3_result_zeroblob64 _sqlite3_serialize +_sqlite3_set_authorizer +_sqlite3_set_auxdata +_sqlite3_set_last_insert_rowid _sqlite3_shutdown _sqlite3_sourceid _sqlite3_sql +_sqlite3_status +_sqlite3_status64 _sqlite3_step +_sqlite3_stmt_isexplain +_sqlite3_stmt_readonly +_sqlite3_stmt_status _sqlite3_strglob +_sqlite3_stricmp _sqlite3_strlike +_sqlite3_strnicmp +_sqlite3_table_column_metadata _sqlite3_total_changes _sqlite3_total_changes64 _sqlite3_trace_v2 +_sqlite3_txn_state _sqlite3_uri_boolean _sqlite3_uri_int64 _sqlite3_uri_key _sqlite3_uri_parameter _sqlite3_user_data _sqlite3_value_blob _sqlite3_value_bytes _sqlite3_value_double +_sqlite3_value_dup +_sqlite3_value_free +_sqlite3_value_frombind _sqlite3_value_int _sqlite3_value_int64 +_sqlite3_value_nochange +_sqlite3_value_numeric_type +_sqlite3_value_pointer +_sqlite3_value_subtype _sqlite3_value_text _sqlite3_value_type _sqlite3_vfs_find _sqlite3_vfs_register _sqlite3_vfs_unregister -_malloc -_free +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value +_sqlite3changegroup_add +_sqlite3changegroup_add_strm +_sqlite3changegroup_delete +_sqlite3changegroup_new +_sqlite3changegroup_output +_sqlite3changegroup_output_strm +_sqlite3changeset_apply +_sqlite3changeset_apply_strm +_sqlite3changeset_apply_v2 +_sqlite3changeset_apply_v2_strm +_sqlite3changeset_concat +_sqlite3changeset_concat_strm +_sqlite3changeset_conflict +_sqlite3changeset_finalize +_sqlite3changeset_fk_conflicts +_sqlite3changeset_invert +_sqlite3changeset_invert_strm +_sqlite3changeset_new +_sqlite3changeset_next +_sqlite3changeset_old +_sqlite3changeset_op +_sqlite3changeset_pk +_sqlite3changeset_start +_sqlite3changeset_start_strm +_sqlite3changeset_start_v2 +_sqlite3changeset_start_v2_strm +_sqlite3session_attach +_sqlite3session_changeset +_sqlite3session_changeset_size +_sqlite3session_changeset_strm +_sqlite3session_config +_sqlite3session_create +_sqlite3session_delete +_sqlite3session_diff +_sqlite3session_enable +_sqlite3session_indirect +_sqlite3session_isempty +_sqlite3session_memory_used +_sqlite3session_object_config +_sqlite3session_patchset +_sqlite3session_patchset_strm +_sqlite3session_table_filter Index: ext/wasm/api/README.md ================================================================== --- ext/wasm/api/README.md +++ ext/wasm/api/README.md @@ -15,72 +15,80 @@ 3. Certain components must be in their own standalone files in order to be loaded as JS Workers. Note that the structure described here is the current state of things, -not necessarily the "final" state. +as of this writing, but is not set in stone forever and may change +at any time. The overall idea is that the following files get concatenated together, in the listed order, the resulting file is loaded by a browser client: -- `sqlite3-api-prologue.js`\ +- **`sqlite3-api-prologue.js`**\ Contains the initial bootstrap setup of the sqlite3 API objects. This is exposed as a function, rather than objects, so that the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. -- `../common/whwasmutil.js`\ +- **`../common/whwasmutil.js`**\ A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be more portable to arbitrary WASM toolchains. This API is configurable, in principle, for use with arbitrary WASM toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- `../jaccwabyt/jaccwabyt.js`\ +- **`../jaccwabyt/jaccwabyt.js`**\ Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an independent spinoff project, conceived for the sqlite3 project but maintained separately. -- `sqlite3-api-glue.js`\ - Invokes functionality exposed by the previous two files to - flesh out low-level parts of `sqlite3-api-prologue.js`. Most of - these pieces related to the `sqlite3.capi.wasm` object. -- `sqlite3-api-build-version.js`\ +- **`sqlite3-api-glue.js`**\ + Invokes functionality exposed by the previous two files to flesh out + low-level parts of `sqlite3-api-prologue.js`. Most of these pieces + related to populating the `sqlite3.capi.wasm` object. This file + also deletes most global-scope symbols the above files create, + effectively moving them into the scope being used for initializing + the API. +- **`/sqlite3-api-build-version.js`**\ Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- `sqlite3-api-oo1.js`\ +- **`sqlite3-api-oo1.js`**\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker1.js`\ +- **`sqlite3-api-worker1.js`**\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker1.js`\ + - **`sqlite3-worker1.js`**\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module and runs the Worker #1 API which is implemented in `sqlite3-api-worker1.js`. - - `sqlite3-worker1-promiser.js`\ + - **`sqlite3-worker1-promiser.js`**\ Is likewise not part of the amalgamated sources and provides a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- `sqlite3-api-opfs.js`\ +- **`sqlite3-v-helper.js`**\ + Installs `sqlite3.vfs` and `sqlite3.vtab`, namespaces which contain + helpers for use by downstream code which creates `sqlite3_vfs` + and `sqlite3_module` implementations. +- **`sqlite3-vfs-opfs.c-pp.js`**\ is an sqlite3 VFS implementation which supports Google Chrome's Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage for database files in a browser. It requires... - - `sqlite3-opfs-async-proxy.js`\ + - **`sqlite3-opfs-async-proxy.js`**\ is the asynchronous backend part of the OPFS proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be started in its own Worker, is not part of the amalgamation. - **`api/sqlite3-api-cleanup.js`**\ @@ -93,10 +101,17 @@ installed by the previous files. As of this writing, this code ensures that the previous files leave no more than a single global symbol installed. When adapting the API for non-Emscripten toolchains, this "should" be the only file where changes are needed. + +**Files with the extension `.c-pp.js`** are intended [to be processed +with `c-pp`](#c-pp), noting that such preprocessing may be applied +after all of the relevant files are concatenated. That extension is +used primarily to keep the code maintainers cognisant of the fact that +those files contain constructs which will not run as-is in JavaScript. + The build process glues those files together, resulting in `sqlite3-api.js`, which is everything except for the `post-js-*.js` files, and `sqlite3.js`, which is the Emscripten-generated amalgamated output and includes the `post-js-*.js` parts, as well as the Emscripten-provided module loading pieces. @@ -112,11 +127,11 @@ - `extern-pre-js.js`\ Emscripten-specific header for Emscripten's `--extern-pre-js` flag. As of this writing, that file is only used for experimentation purposes and holds no code relevant to the production deliverables. -- `pre-js.js`\ +- `pre-js.c-pp.js`\ Emscripten-specific header for Emscripten's `--pre-js` flag. This file is intended as a place to override certain Emscripten behavior before it starts up, but corner-case Emscripten bugs keep that from being a reality. - `post-js-header.js`\ @@ -123,11 +138,22 @@ Emscripten-specific header for the `--post-js` input. It opens up a lexical scope by starting a post-run handler for Emscripten. - `post-js-footer.js`\ Emscripten-specific footer for the `--post-js` input. This closes off the lexical scope opened by `post-js-header.js`. -- `extern-post-js.js`\ +- `extern-post-js.c-pp.js`\ Emscripten-specific header for Emscripten's `--extern-post-js` flag. This file overwrites the Emscripten-installed `sqlite3InitModule()` function with one which, after the module is loaded, also initializes the asynchronous parts of the sqlite3 module. For example, the OPFS VFS support. + + +Preprocessing of Source Files +------------------------------------------------------------------------ + +Certain files in the build require preprocessing to filter in/out +parts which differ between vanilla JS builds and ES6 Module +(a.k.a. esm) builds. The preprocessor application itself is in +[`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details +of such preprocessing are maintained in +[`GNUMakefile`](/file/ext/wasm/GNUmakefile). ADDED ext/wasm/api/extern-post-js.c-pp.js Index: ext/wasm/api/extern-post-js.c-pp.js ================================================================== --- /dev/null +++ ext/wasm/api/extern-post-js.c-pp.js @@ -0,0 +1,121 @@ + +/* ^^^^ ACHTUNG: blank line at the start is necessary because + Emscripten will not add a newline in some cases and we need + a blank line for a sed-based kludge for the ES6 build. */ +/* extern-post-js.js must be appended to the resulting sqlite3.js + file. It gets its name from being used as the value for the + --extern-post-js=... Emscripten flag. Note that this code, unlike + most of the associated JS code, runs outside of the + Emscripten-generated module init scope, in the current + global scope. */ +//#if target=es6-module +const toExportForES6 = +//#endif +(function(){ + /** + In order to hide the sqlite3InitModule()'s resulting + Emscripten module from downstream clients (and simplify our + documentation by being able to elide those details), we hide that + function and expose a hand-written sqlite3InitModule() to return + the sqlite3 object (most of the time). + + Unfortunately, we cannot modify the module-loader/exporter-based + impls which Emscripten installs at some point in the file above + this. + */ + const originalInit = + /* Maintenance reminder: DO NOT use `self.` here. It's correct + for non-ES6 Module cases but wrong for ES6 modules because those + resolve this symbol differently. */ sqlite3InitModule; + if(!originalInit){ + throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); + } + /** + We need to add some state which our custom Module.locateFile() + can see, but an Emscripten limitation currently prevents us from + attaching it to the sqlite3InitModule function object: + + https://github.com/emscripten-core/emscripten/issues/18071 + + The only(?) current workaround is to temporarily stash this state + into the global scope and delete it when sqlite3InitModule() + is called. + */ + const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ + moduleScript: self?.document?.currentScript, + isWorker: ('undefined' !== typeof WorkerGlobalScope), + location: self.location, + urlParams: new URL(self.location.href).searchParams + }); + initModuleState.debugModule = + (new URL(self.location.href).searchParams).has('sqlite3.debugModule') + ? (...args)=>console.warn('sqlite3.debugModule:',...args) + : ()=>{}; + + if(initModuleState.urlParams.has('sqlite3.dir')){ + initModuleState.sqlite3Dir = initModuleState.urlParams.get('sqlite3.dir') +'/'; + }else if(initModuleState.moduleScript){ + const li = initModuleState.moduleScript.src.split('/'); + li.pop(); + initModuleState.sqlite3Dir = li.join('/') + '/'; + } + + self.sqlite3InitModule = function ff(...args){ + //console.warn("Using replaced sqlite3InitModule()",self.location); + return originalInit(...args).then((EmscriptenModule)=>{ + if(self.window!==self && + (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] + || EmscriptenModule['_pthread_self'] + || 'function'===typeof threadAlert + || self.location.pathname.endsWith('.worker.js') + )){ + /** Workaround for wasmfs-generated worker, which calls this + routine from each individual thread and requires that its + argument be returned. All of the criteria above are fragile, + based solely on inspection of the offending code, not public + Emscripten details. */ + return EmscriptenModule; + } + const s = EmscriptenModule.sqlite3; + s.scriptInfo = initModuleState; + //console.warn("sqlite3.scriptInfo =",s.scriptInfo); + if(ff.__isUnderTest) s.__isUnderTest = true; + const f = s.asyncPostInit; + delete s.asyncPostInit; + return f(); + }).catch((e)=>{ + console.error("Exception loading sqlite3 module:",e); + throw e; + }); + }; + self.sqlite3InitModule.ready = originalInit.ready; + + if(self.sqlite3InitModuleState.moduleScript){ + const sim = self.sqlite3InitModuleState; + let src = sim.moduleScript.src.split('/'); + src.pop(); + sim.scriptDir = src.join('/') + '/'; + } + initModuleState.debugModule('sqlite3InitModuleState =',initModuleState); + if(0){ + console.warn("Replaced sqlite3InitModule()"); + console.warn("self.location.href =",self.location.href); + if('undefined' !== typeof document){ + console.warn("document.currentScript.src =", + document?.currentScript?.src); + } + } + /* Replace the various module exports performed by the Emscripten + glue... */ + if (typeof exports === 'object' && typeof module === 'object'){ + module.exports = sqlite3InitModule; + }else if (typeof exports === 'object'){ + exports["sqlite3InitModule"] = sqlite3InitModule; + } + /* AMD modules get injected in a way we cannot override, + so we can't handle those here. */ + return self.sqlite3InitModule /* required for ESM */; +})(); +//#if target=es6-module +export default toExportForES6; +//#endif DELETED ext/wasm/api/extern-post-js.js Index: ext/wasm/api/extern-post-js.js ================================================================== --- ext/wasm/api/extern-post-js.js +++ /dev/null @@ -1,103 +0,0 @@ -/* extern-post-js.js must be appended to the resulting sqlite3.js - file. It gets its name from being used as the value for the - --extern-post-js=... Emscripten flag. Note that this code, unlike - most of the associated JS code, runs outside of the - Emscripten-generated module init scope, in the current - global scope. */ -(function(){ - /** - In order to hide the sqlite3InitModule()'s resulting Emscripten - module from downstream clients (and simplify our documentation by - being able to elide those details), we rewrite - sqlite3InitModule() to return the sqlite3 object. - - Unfortunately, we cannot modify the module-loader/exporter-based - impls which Emscripten installs at some point in the file above - this. - */ - const originalInit = self.sqlite3InitModule; - if(!originalInit){ - throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); - } - /** - We need to add some state which our custom Module.locateFile() - can see, but an Emscripten limitation currently prevents us from - attaching it to the sqlite3InitModule function object: - - https://github.com/emscripten-core/emscripten/issues/18071 - - The only(?) current workaround is to temporarily stash this state - into the global scope and delete it when sqlite3InitModule() - is called. - */ - const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ - moduleScript: self?.document?.currentScript, - isWorker: ('undefined' !== typeof WorkerGlobalScope), - location: self.location, - urlParams: new URL(self.location.href).searchParams - }); - initModuleState.debugModule = - (new URL(self.location.href).searchParams).has('sqlite3.debugModule') - ? (...args)=>console.warn('sqlite3.debugModule:',...args) - : ()=>{}; - - if(initModuleState.urlParams.has('sqlite3.dir')){ - initModuleState.sqlite3Dir = initModuleState.urlParams.get('sqlite3.dir') +'/'; - }else if(initModuleState.moduleScript){ - const li = initModuleState.moduleScript.src.split('/'); - li.pop(); - initModuleState.sqlite3Dir = li.join('/') + '/'; - } - - self.sqlite3InitModule = (...args)=>{ - //console.warn("Using replaced sqlite3InitModule()",self.location); - return originalInit(...args).then((EmscriptenModule)=>{ - if(self.window!==self && - (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] - || EmscriptenModule['_pthread_self'] - || 'function'===typeof threadAlert - || self.location.pathname.endsWith('.worker.js') - )){ - /** Workaround for wasmfs-generated worker, which calls this - routine from each individual thread and requires that its - argument be returned. All of the criteria above are fragile, - based solely on inspection of the offending code, not public - Emscripten details. */ - return EmscriptenModule; - } - EmscriptenModule.sqlite3.scriptInfo = initModuleState; - //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo); - const f = EmscriptenModule.sqlite3.asyncPostInit; - delete EmscriptenModule.sqlite3.asyncPostInit; - return f(); - }).catch((e)=>{ - console.error("Exception loading sqlite3 module:",e); - throw e; - }); - }; - self.sqlite3InitModule.ready = originalInit.ready; - - if(self.sqlite3InitModuleState.moduleScript){ - const sim = self.sqlite3InitModuleState; - let src = sim.moduleScript.src.split('/'); - src.pop(); - sim.scriptDir = src.join('/') + '/'; - } - initModuleState.debugModule('sqlite3InitModuleState =',initModuleState); - if(0){ - console.warn("Replaced sqlite3InitModule()"); - console.warn("self.location.href =",self.location.href); - if('undefined' !== typeof document){ - console.warn("document.currentScript.src =", - document?.currentScript?.src); - } - } - /* Replace the various module exports performed by the Emscripten - glue... */ - if (typeof exports === 'object' && typeof module === 'object') - module.exports = sqlite3InitModule; - else if (typeof exports === 'object') - exports["sqlite3InitModule"] = sqlite3InitModule; - /* AMD modules get injected in a way we cannot override, - so we can't handle those here. */ -})(); Index: ext/wasm/api/post-js-header.js ================================================================== --- ext/wasm/api/post-js-header.js +++ ext/wasm/api/post-js-header.js @@ -17,9 +17,10 @@ - common/whwasmutil.js => Replacements for much of Emscripten's glue - jaccwaby/jaccwabyt.js => Jaccwabyt (C/JS struct binding) - sqlite3-api-glue.js => glues previous parts together - sqlite3-api-oo.js => SQLite3 OO API #1 - sqlite3-api-worker1.js => Worker-based API - - sqlite3-api-opfs.js => OPFS VFS + - sqlite3-vfs-helper.js => Internal-use utilities for... + - sqlite3-vfs-opfs.js => OPFS VFS - sqlite3-api-cleanup.js => final API cleanup - post-js-footer.js => closes this postRun() function */ ADDED ext/wasm/api/pre-js.c-pp.js Index: ext/wasm/api/pre-js.c-pp.js ================================================================== --- /dev/null +++ ext/wasm/api/pre-js.c-pp.js @@ -0,0 +1,105 @@ +/** + BEGIN FILE: api/pre-js.js + + This file is intended to be prepended to the sqlite3.js build using + Emscripten's --pre-js=THIS_FILE flag (or equivalent). +*/ + +// See notes in extern-post-js.js +const sqlite3InitModuleState = self.sqlite3InitModuleState || Object.create(null); +delete self.sqlite3InitModuleState; +sqlite3InitModuleState.debugModule('self.location =',self.location); + +/** + This custom locateFile() tries to figure out where to load `path` + from. The intent is to provide a way for foo/bar/X.js loaded from a + Worker constructor or importScripts() to be able to resolve + foo/bar/X.wasm (in the latter case, with some help): + + 1) If URL param named the same as `path` is set, it is returned. + + 2) If sqlite3InitModuleState.sqlite3Dir is set, then (thatName + path) + is returned (note that it's assumed to end with '/'). + + 3) If this code is running in the main UI thread AND it was loaded + from a SCRIPT tag, the directory part of that URL is used + as the prefix. (This form of resolution unfortunately does not + function for scripts loaded via importScripts().) + + 4) If none of the above apply, (prefix+path) is returned. +*/ +Module['locateFile'] = function(path, prefix) { +//#if target=es6-module + return new URL(path, import.meta.url).href; +//#else + 'use strict'; + let theFile; + const up = this.urlParams; + if(up.has(path)){ + theFile = up.get(path); + }else if(this.sqlite3Dir){ + theFile = this.sqlite3Dir + path; + }else if(this.scriptDir){ + theFile = this.scriptDir + path; + }else{ + theFile = prefix + path; + } + sqlite3InitModuleState.debugModule( + "locateFile(",arguments[0], ',', arguments[1],")", + 'sqlite3InitModuleState.scriptDir =',this.scriptDir, + 'up.entries() =',Array.from(up.entries()), + "result =", theFile + ); + return theFile; +//#endif /* SQLITE_JS_EMS */ +}.bind(sqlite3InitModuleState); + +/** + Bug warning: a custom Module.instantiateWasm() does not work + in WASMFS builds: + + https://github.com/emscripten-core/emscripten/issues/17951 + + In such builds we must disable this. +*/ +const xNameOfInstantiateWasm = true + ? 'instantiateWasm' + : 'emscripten-bug-17951'; +Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ + imports.env.foo = function(){}; + const uri = Module.locateFile( + callee.uri, ( + ('undefined'===typeof scriptDirectory/*var defined by Emscripten glue*/) + ? "" : scriptDirectory) + ); + sqlite3InitModuleState.debugModule( + "instantiateWasm() uri =", uri + ); + const wfetch = ()=>fetch(uri, {credentials: 'same-origin'}); + const loadWasm = WebAssembly.instantiateStreaming + ? async ()=>{ + return WebAssembly.instantiateStreaming(wfetch(), imports) + .then((arg)=>onSuccess(arg.instance, arg.module)); + } + : async ()=>{ // Safari < v15 + return wfetch() + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, imports)) + .then((arg)=>onSuccess(arg.instance, arg.module)); + }; + loadWasm(); + return {}; +}; +/* + It is literally impossible to reliably get the name of _this_ script + at runtime, so impossible to derive X.wasm from script name + X.js. Thus we need, at build-time, to redefine + Module[xNameOfInstantiateWasm].uri by appending it to a build-specific + copy of this file with the name of the wasm file. This is apparently + why Emscripten hard-codes the name of the wasm file into their glue + scripts. +*/ +Module[xNameOfInstantiateWasm].uri = 'sqlite3.wasm'; +/* END FILE: api/pre-js.js, noting that the build process may add a + line after this one to change the above .uri to a build-specific + one. */ DELETED ext/wasm/api/pre-js.js Index: ext/wasm/api/pre-js.js ================================================================== --- ext/wasm/api/pre-js.js +++ /dev/null @@ -1,100 +0,0 @@ -/** - BEGIN FILE: api/pre-js.js - - This file is intended to be prepended to the sqlite3.js build using - Emscripten's --pre-js=THIS_FILE flag (or equivalent). -*/ - -// See notes in extern-post-js.js -const sqlite3InitModuleState = self.sqlite3InitModuleState || Object.create(null); -delete self.sqlite3InitModuleState; -sqlite3InitModuleState.debugModule('self.location =',self.location); - -/** - This custom locateFile() tries to figure out where to load `path` - from. The intent is to provide a way for foo/bar/X.js loaded from a - Worker constructor or importScripts() to be able to resolve - foo/bar/X.wasm (in the latter case, with some help): - - 1) If URL param named the same as `path` is set, it is returned. - - 2) If sqlite3InitModuleState.sqlite3Dir is set, then (thatName + path) - is returned (note that it's assumed to end with '/'). - - 3) If this code is running in the main UI thread AND it was loaded - from a SCRIPT tag, the directory part of that URL is used - as the prefix. (This form of resolution unfortunately does not - function for scripts loaded via importScripts().) - - 4) If none of the above apply, (prefix+path) is returned. -*/ -Module['locateFile'] = function(path, prefix) { - let theFile; - const up = this.urlParams; - if(up.has(path)){ - theFile = up.get(path); - }else if(this.sqlite3Dir){ - theFile = this.sqlite3Dir + path; - }else if(this.scriptDir){ - theFile = this.scriptDir + path; - }else{ - theFile = prefix + path; - } - sqlite3InitModuleState.debugModule( - "locateFile(",arguments[0], ',', arguments[1],")", - 'sqlite3InitModuleState.scriptDir =',this.scriptDir, - 'up.entries() =',Array.from(up.entries()), - "result =", theFile - ); - return theFile; -}.bind(sqlite3InitModuleState); - -/** - Bug warning: a custom Module.instantiateWasm() does not work - in WASMFS builds: - - https://github.com/emscripten-core/emscripten/issues/17951 - - In such builds we must disable this. -*/ -const xNameOfInstantiateWasm = true - ? 'instantiateWasm' - : 'emscripten-bug-17951'; -Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ - imports.env.foo = function(){}; - const uri = Module.locateFile( - callee.uri, ( - ('undefined'===typeof scriptDirectory/*var defined by Emscripten glue*/) - ? '' : scriptDirectory) - ); - sqlite3InitModuleState.debugModule( - "instantiateWasm() uri =", uri - ); - const wfetch = ()=>fetch(uri, {credentials: 'same-origin'}); - const loadWasm = WebAssembly.instantiateStreaming - ? async ()=>{ - return WebAssembly.instantiateStreaming(wfetch(), imports) - .then((arg)=>onSuccess(arg.instance, arg.module)); - } - : async ()=>{ // Safari < v15 - return wfetch() - .then(response => response.arrayBuffer()) - .then(bytes => WebAssembly.instantiate(bytes, imports)) - .then((arg)=>onSuccess(arg.instance, arg.module)); - }; - loadWasm(); - return {}; -}; -/* - It is literally impossible to reliably get the name of _this_ script - at runtime, so impossible to derive X.wasm from script name - X.js. Thus we need, at build-time, to redefine - Module[xNameOfInstantiateWasm].uri by appending it to a build-specific - copy of this file with the name of the wasm file. This is apparently - why Emscripten hard-codes the name of the wasm file into their glue - scripts. -*/ -Module[xNameOfInstantiateWasm].uri = 'sqlite3.wasm'; -/* END FILE: api/pre-js.js, noting that the build process may add a - line after this one to change the above .uri to a build-specific - one. */ Index: ext/wasm/api/sqlite3-api-cleanup.js ================================================================== --- ext/wasm/api/sqlite3-api-cleanup.js +++ ext/wasm/api/sqlite3-api-cleanup.js @@ -20,16 +20,14 @@ /** Install a suitable default configuration for sqlite3ApiBootstrap(). */ const SABC = Object.assign( Object.create(null), { - Module: Module /* ==> Currently needs to be exposed here for - test code. NOT part of the public API. */, exports: Module['asm'], memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ }, - self.sqlite3ApiConfig || Object.create(null) + self.sqlite3ApiConfig || {} ); /** For current (2022-08-22) purposes, automatically call sqlite3ApiBootstrap(). That decision will be revisited at some @@ -56,15 +54,13 @@ if(self.location && +self.location.port > 1024){ console.warn("Installing sqlite3 bits as global S for local dev/test purposes."); self.S = sqlite3; } - /* Clean up temporary references to our APIs... */ - delete sqlite3.util /* arguable, but these are (currently) internal-use APIs */; Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to pass the sqlite3 object off to the client. */; }else{ console.warn("This is not running in an Emscripten module context, so", "self.sqlite3ApiBootstrap() is _not_ being called due to lack", "of config info for the WASM environment.", "It must be called manually."); } Index: ext/wasm/api/sqlite3-api-glue.js ================================================================== --- ext/wasm/api/sqlite3-api-glue.js +++ ext/wasm/api/sqlite3-api-glue.js @@ -22,65 +22,628 @@ const toss3 = sqlite3.SQLite3Error.toss; const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; self.WhWasmUtilInstaller(wasm); delete self.WhWasmUtilInstaller; + { + /** + Find a mapping for SQLITE_WASM_DEALLOC, which the API + guarantees is a WASM pointer to the same underlying function as + wasm.dealloc() (noting that wasm.dealloc() is permitted to be a + JS wrapper around the WASM function). There is unfortunately no + O(1) algorithm for finding this pointer: we have to walk the + WASM indirect function table to find it. However, experience + indicates that that particular function is always very close to + the front of the table (it's been entry #3 in all relevant + tests). + */ + const dealloc = wasm.exports[sqlite3.config.deallocExportName]; + const nFunc = wasm.functionTable().length; + let i; + for(i = 0; i < nFunc; ++i){ + const e = wasm.functionEntry(i); + if(dealloc === e){ + capi.SQLITE_WASM_DEALLOC = i; + break; + } + } + if(dealloc !== wasm.functionEntry(capi.SQLITE_WASM_DEALLOC)){ + toss("Internal error: cannot find function pointer for SQLITE_WASM_DEALLOC."); + } + } + + /** + Signatures for the WASM-exported C-side functions. Each entry + is an array with 2+ elements: + + [ "c-side name", + "result type" (wasm.xWrap() syntax), + [arg types in xWrap() syntax] + // ^^^ this needn't strictly be an array: it can be subsequent + // elements instead: [x,y,z] is equivalent to x,y,z + ] + + Note that support for the API-specific data types in the + result/argument type strings gets plugged in at a later phase in + the API initialization process. + */ + wasm.bindingSignatures = [ + // Please keep these sorted by function name! + ["sqlite3_aggregate_context","void*", "sqlite3_context*", "int"], + /* sqlite3_bind_blob() and sqlite3_bind_text() have hand-written + bindings to permit more flexible inputs. */ + ["sqlite3_bind_double","int", "sqlite3_stmt*", "int", "f64"], + ["sqlite3_bind_int","int", "sqlite3_stmt*", "int", "int"], + ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], + ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], + ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], + ["sqlite3_bind_pointer", "int", + "sqlite3_stmt*", "int", "*", "string:static", "*"], + ["sqlite3_busy_handler","int", [ + "sqlite3*", + new wasm.xWrap.FuncPtrAdapter({ + signature: 'i(pi)', + contextKey: (argv,argIndex)=>argv[0/* sqlite3* */] + }), + "*" + ]], + ["sqlite3_busy_timeout","int", "sqlite3*", "int"], + /*[sqlite3_close_v2() is implemented by hand to perform some + extra work. "sqlite3_close_v2", "int", "sqlite3*"],*/ + ["sqlite3_changes", "int", "sqlite3*"], + ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], + ["sqlite3_collation_needed", "int", "sqlite3*", "*", "*"/*=>v(ppis)*/], + ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], + ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_count", "int", "sqlite3_stmt*"], + ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], + ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_value","sqlite3_value*", "sqlite3_stmt*", "int"], + ["sqlite3_compileoption_get", "string", "int"], + ["sqlite3_compileoption_used", "int", "string"], + ["sqlite3_complete", "int", "string:flexible"], + ["sqlite3_context_db_handle", "sqlite3*", "sqlite3_context*"], + + /* sqlite3_create_function(), sqlite3_create_function_v2(), and + sqlite3_create_window_function() use hand-written bindings to + simplify handling of their function-type arguments. */ + /* sqlite3_create_collation() and sqlite3_create_collation_v2() + use hand-written bindings to simplify passing of the callback + function. + ["sqlite3_create_collation", "int", + "sqlite3*", "string", "int",//SQLITE_UTF8 is the only legal value + "*", "*"], + ["sqlite3_create_collation_v2", "int", + "sqlite3*", "string", "int",//SQLITE_UTF8 is the only legal value + "*", "*", "*"], + */ + ["sqlite3_data_count", "int", "sqlite3_stmt*"], + ["sqlite3_db_filename", "string", "sqlite3*", "string"], + ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], + ["sqlite3_db_name", "string", "sqlite3*", "int"], + ["sqlite3_db_status", "int", "sqlite3*", "int", "*", "*", "int"], + ["sqlite3_errcode", "int", "sqlite3*"], + ["sqlite3_errmsg", "string", "sqlite3*"], + ["sqlite3_error_offset", "int", "sqlite3*"], + ["sqlite3_errstr", "string", "int"], + ["sqlite3_exec", "int", [ + "sqlite3*", "string:flexible", + new wasm.xWrap.FuncPtrAdapter({ + signature: 'i(pipp)', + bindScope: 'transient', + callProxy: (callback)=>{ + let aNames; + return (pVoid, nCols, pColVals, pColNames)=>{ + try { + const aVals = wasm.cArgvToJs(nCols, pColVals); + if(!aNames) aNames = wasm.cArgvToJs(nCols, pColNames); + return callback(aVals, aNames) | 0; + }catch(e){ + /* If we set the db error state here, the higher-level + exec() call replaces it with its own, so we have no way + of reporting the exception message except the console. We + must not propagate exceptions through the C API. Though + we make an effort to report OOM here, sqlite3_exec() + translates that into SQLITE_ABORT as well. */ + return e.resultCode || capi.SQLITE_ERROR; + } + } + } + }), + "*", "**" + ]], + ["sqlite3_expanded_sql", "string", "sqlite3_stmt*"], + ["sqlite3_extended_errcode", "int", "sqlite3*"], + ["sqlite3_extended_result_codes", "int", "sqlite3*", "int"], + ["sqlite3_file_control", "int", "sqlite3*", "string", "int", "*"], + ["sqlite3_finalize", "int", "sqlite3_stmt*"], + ["sqlite3_free", undefined,"*"], + ["sqlite3_get_auxdata", "*", "sqlite3_context*", "int"], + ["sqlite3_initialize", undefined], + /*["sqlite3_interrupt", undefined, "sqlite3*" + ^^^ we cannot actually currently support this because JS is + single-threaded and we don't have a portable way to access a DB + from 2 SharedWorkers concurrently. ],*/ + ["sqlite3_keyword_count", "int"], + ["sqlite3_keyword_name", "int", ["int", "**", "*"]], + ["sqlite3_keyword_check", "int", ["string", "int"]], + ["sqlite3_libversion", "string"], + ["sqlite3_libversion_number", "int"], + ["sqlite3_limit", "int", ["sqlite3*", "int", "int"]], + ["sqlite3_malloc", "*","int"], + ["sqlite3_open", "int", "string", "*"], + ["sqlite3_open_v2", "int", "string", "*", "int", "string"], + /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled + separately due to us requiring two different sets of semantics + for those, depending on how their SQL argument is provided. */ + /* sqlite3_randomness() uses a hand-written wrapper to extend + the range of supported argument types. */ + ["sqlite3_progress_handler", undefined, [ + "sqlite3*", "int", new wasm.xWrap.FuncPtrAdapter({ + name: 'xProgressHandler', + signature: 'i(p)', + bindScope: 'context', + contextKey: (argv,argIndex)=>argv[0/* sqlite3* */] + }), "*" + ]], + ["sqlite3_realloc", "*","*","int"], + ["sqlite3_reset", "int", "sqlite3_stmt*"], + ["sqlite3_result_blob", undefined, "sqlite3_context*", "*", "int", "*"], + ["sqlite3_result_double", undefined, "sqlite3_context*", "f64"], + ["sqlite3_result_error", undefined, "sqlite3_context*", "string", "int"], + ["sqlite3_result_error_code", undefined, "sqlite3_context*", "int"], + ["sqlite3_result_error_nomem", undefined, "sqlite3_context*"], + ["sqlite3_result_error_toobig", undefined, "sqlite3_context*"], + ["sqlite3_result_int", undefined, "sqlite3_context*", "int"], + ["sqlite3_result_null", undefined, "sqlite3_context*"], + ["sqlite3_result_pointer", undefined, + "sqlite3_context*", "*", "string:static", "*"], + ["sqlite3_result_subtype", undefined, "sqlite3_value*", "int"], + ["sqlite3_result_text", undefined, "sqlite3_context*", "string", "int", "*"], + ["sqlite3_result_zeroblob", undefined, "sqlite3_context*", "int"], + ["sqlite3_set_authorizer", "int", [ + "sqlite3*", + new wasm.xWrap.FuncPtrAdapter({ + name: "sqlite3_set_authorizer::xAuth", + signature: "i(pi"+"ssss)", + contextKey: (argv, argIndex)=>argv[0/*(sqlite3*)*/], + callProxy: (callback)=>{ + return (pV, iCode, s0, s1, s2, s3)=>{ + try{ + s0 = s0 && wasm.cstrToJs(s0); s1 = s1 && wasm.cstrToJs(s1); + s2 = s2 && wasm.cstrToJs(s2); s3 = s3 && wasm.cstrToJs(s3); + return callback(pV, iCode, s0, s1, s2, s3) || 0; + }catch(e){ + return e.resultCode || capi.SQLITE_ERROR; + } + } + } + }), + "*"/*pUserData*/ + ]], + ["sqlite3_set_auxdata", undefined, [ + "sqlite3_context*", "int", "*", + new wasm.xWrap.FuncPtrAdapter({ + name: 'xDestroyAuxData', + signature: 'v(*)', + contextKey: (argv, argIndex)=>argv[0/* sqlite3_context* */] + }) + ]], + ["sqlite3_shutdown", undefined], + ["sqlite3_sourceid", "string"], + ["sqlite3_sql", "string", "sqlite3_stmt*"], + ["sqlite3_status", "int", "int", "*", "*", "int"], + ["sqlite3_step", "int", "sqlite3_stmt*"], + ["sqlite3_stmt_isexplain", "int", ["sqlite3_stmt*"]], + ["sqlite3_stmt_readonly", "int", ["sqlite3_stmt*"]], + ["sqlite3_stmt_status", "int", "sqlite3_stmt*", "int", "int"], + ["sqlite3_strglob", "int", "string","string"], + ["sqlite3_stricmp", "int", "string", "string"], + ["sqlite3_strlike", "int", "string", "string","int"], + ["sqlite3_strnicmp", "int", "string", "string", "int"], + ["sqlite3_table_column_metadata", "int", + "sqlite3*", "string", "string", "string", + "**", "**", "*", "*", "*"], + ["sqlite3_total_changes", "int", "sqlite3*"], + ["sqlite3_trace_v2", "int", [ + "sqlite3*", "int", + new wasm.xWrap.FuncPtrAdapter({ + name: 'sqlite3_trace_v2::callback', + signature: 'i(ippp)', + contextKey: (argv,argIndex)=>argv[0/* sqlite3* */] + }), + "*" + ]], + ["sqlite3_txn_state", "int", ["sqlite3*","string"]], + /* Note that sqlite3_uri_...() have very specific requirements for + their first C-string arguments, so we cannot perform any value + conversion on those. */ + ["sqlite3_uri_boolean", "int", "sqlite3_filename", "string", "int"], + ["sqlite3_uri_key", "string", "sqlite3_filename", "int"], + ["sqlite3_uri_parameter", "string", "sqlite3_filename", "string"], + ["sqlite3_user_data","void*", "sqlite3_context*"], + ["sqlite3_value_blob", "*", "sqlite3_value*"], + ["sqlite3_value_bytes","int", "sqlite3_value*"], + ["sqlite3_value_double","f64", "sqlite3_value*"], + ["sqlite3_value_dup", "sqlite3_value*", "sqlite3_value*"], + ["sqlite3_value_free", undefined, "sqlite3_value*"], + ["sqlite3_value_frombind", "int", "sqlite3_value*"], + ["sqlite3_value_int","int", "sqlite3_value*"], + ["sqlite3_value_nochange", "int", "sqlite3_value*"], + ["sqlite3_value_numeric_type", "int", "sqlite3_value*"], + ["sqlite3_value_pointer", "*", "sqlite3_value*", "string:static"], + ["sqlite3_value_subtype", "int", "sqlite3_value*"], + ["sqlite3_value_text", "string", "sqlite3_value*"], + ["sqlite3_value_type", "int", "sqlite3_value*"], + ["sqlite3_vfs_find", "*", "string"], + ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], + ["sqlite3_vfs_unregister", "int", "sqlite3_vfs*"] + ]/*wasm.bindingSignatures*/; + + if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ + /* ^^^ "the problem" is that this is an option feature and the + build-time function-export list does not currently take + optional features into account. */ + wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); + } + + /** + Functions which require BigInt (int64) support are separated from + the others because we need to conditionally bind them or apply + dummy impls, depending on the capabilities of the environment. + + Note that not all of these functions directly require int64 + but are only for use with APIs which require int64. For example, + the vtab-related functions. + */ + wasm.bindingSignatures.int64 = [ + ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], + ["sqlite3_changes64","i64", ["sqlite3*"]], + ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], + ["sqlite3_create_module", "int", + ["sqlite3*","string","sqlite3_module*","*"]], + ["sqlite3_create_module_v2", "int", + ["sqlite3*","string","sqlite3_module*","*","*"]], + ["sqlite3_declare_vtab", "int", ["sqlite3*", "string:flexible"]], + ["sqlite3_deserialize", "int", "sqlite3*", "string", "*", "i64", "i64", "int"] + /* Careful! Short version: de/serialize() are problematic because they + might use a different allocator than the user for managing the + deserialized block. de/serialize() are ONLY safe to use with + sqlite3_malloc(), sqlite3_free(), and its 64-bit variants. */, + ["sqlite3_drop_modules", "int", ["sqlite3*", "**"]], + ["sqlite3_last_insert_rowid", "i64", ["sqlite3*"]], + ["sqlite3_malloc64", "*","i64"], + ["sqlite3_msize", "i64", "*"], + ["sqlite3_overload_function", "int", ["sqlite3*","string","int"]], + ["sqlite3_realloc64", "*","*", "i64"], + ["sqlite3_result_int64", undefined, "*", "i64"], + ["sqlite3_result_zeroblob64", "int", "*", "i64"], + ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], + ["sqlite3_set_last_insert_rowid", undefined, ["sqlite3*", "i64"]], + ["sqlite3_status64", "int", "int", "*", "*", "int"], + ["sqlite3_total_changes64", "i64", ["sqlite3*"]], + ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], + ["sqlite3_value_int64","i64", "sqlite3_value*"], + ["sqlite3_vtab_collation","string","sqlite3_index_info*","int"], + ["sqlite3_vtab_distinct","int", "sqlite3_index_info*"], + ["sqlite3_vtab_in","int", "sqlite3_index_info*", "int", "int"], + ["sqlite3_vtab_in_first", "int", "sqlite3_value*", "**"], + ["sqlite3_vtab_in_next", "int", "sqlite3_value*", "**"], + /*["sqlite3_vtab_config" is variadic and requires a hand-written + proxy.] */ + ["sqlite3_vtab_nochange","int", "sqlite3_context*"], + ["sqlite3_vtab_on_conflict","int", "sqlite3*"], + ["sqlite3_vtab_rhs_value","int", "sqlite3_index_info*", "int", "**"] + ]; + + // Add session/changeset APIs... + if(wasm.bigIntEnabled && !!wasm.exports.sqlite3changegroup_add){ + /* ACHTUNG: 2022-12-23: the session/changeset API bindings are + COMPLETELY UNTESTED. */ + /** + FuncPtrAdapter options for session-related callbacks with the + native signature "i(ps)". This proxy converts the 2nd argument + from a C string to a JS string before passing the arguments on + to the client-provided JS callback. + */ + const __ipsProxy = { + signature: 'i(ps)', + callProxy:(callback)=>{ + return (p,s)=>{ + try{return callback(p, wasm.cstrToJs(s)) | 0} + catch(e){return e.resultCode || capi.SQLITE_ERROR} + } + } + }; + + wasm.bindingSignatures.int64.push(...[ + ['sqlite3changegroup_add', 'int', ['sqlite3_changegroup*', 'int', 'void*']], + ['sqlite3changegroup_add_strm', 'int', [ + 'sqlite3_changegroup*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changegroup_delete', undefined, ['sqlite3_changegroup*']], + ['sqlite3changegroup_new', 'int', ['**']], + ['sqlite3changegroup_output', 'int', ['sqlite3_changegroup*', 'int*', '**']], + ['sqlite3changegroup_output_strm', 'int', [ + 'sqlite3_changegroup*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xOutput', signature: 'i(ppi)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changeset_apply', 'int', [ + 'sqlite3*', 'int', 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xFilter', bindScope: 'transient', ...__ipsProxy + }), + new wasm.xWrap.FuncPtrAdapter({ + name: 'xConflict', signature: 'i(pip)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changeset_apply_strm', 'int', [ + 'sqlite3*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xFilter', bindScope: 'transient', ...__ipsProxy + }), + new wasm.xWrap.FuncPtrAdapter({ + name: 'xConflict', signature: 'i(pip)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changeset_apply_v2', 'int', [ + 'sqlite3*', 'int', 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xFilter', bindScope: 'transient', ...__ipsProxy + }), + new wasm.xWrap.FuncPtrAdapter({ + name: 'xConflict', signature: 'i(pip)', bindScope: 'transient' + }), + 'void*', '**', 'int*', 'int' + + ]], + ['sqlite3changeset_apply_v2_strm', 'int', [ + 'sqlite3*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xFilter', bindScope: 'transient', ...__ipsProxy + }), + new wasm.xWrap.FuncPtrAdapter({ + name: 'xConflict', signature: 'i(pip)', bindScope: 'transient' + }), + 'void*', '**', 'int*', 'int' + ]], + ['sqlite3changeset_concat', 'int', ['int','void*', 'int', 'void*', 'int*', '**']], + ['sqlite3changeset_concat_strm', 'int', [ + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInputA', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInputB', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xOutput', signature: 'i(ppi)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changeset_conflict', 'int', ['sqlite3_changeset_iter*', 'int', '**']], + ['sqlite3changeset_finalize', 'int', ['sqlite3_changeset_iter*']], + ['sqlite3changeset_fk_conflicts', 'int', ['sqlite3_changeset_iter*', 'int*']], + ['sqlite3changeset_invert', 'int', ['int', 'void*', 'int*', '**']], + ['sqlite3changeset_invert_strm', 'int', [ + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xOutput', signature: 'i(ppi)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changeset_new', 'int', ['sqlite3_changeset_iter*', 'int', '**']], + ['sqlite3changeset_next', 'int', ['sqlite3_changeset_iter*']], + ['sqlite3changeset_old', 'int', ['sqlite3_changeset_iter*', 'int', '**']], + ['sqlite3changeset_op', 'int', [ + 'sqlite3_changeset_iter*', '**', 'int*', 'int*','int*' + ]], + ['sqlite3changeset_pk', 'int', ['sqlite3_changeset_iter*', '**', 'int*']], + ['sqlite3changeset_start', 'int', ['**', 'int', '*']], + ['sqlite3changeset_start_strm', 'int', [ + '**', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3changeset_start_v2', 'int', ['**', 'int', '*', 'int']], + ['sqlite3changeset_start_v2_strm', 'int', [ + '**', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xInput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*', 'int' + ]], + ['sqlite3session_attach', 'int', ['sqlite3_session*', 'string']], + ['sqlite3session_changeset', 'int', ['sqlite3_session*', 'int*', '**']], + ['sqlite3session_changeset_size', 'i64', ['sqlite3_session*']], + ['sqlite3session_changeset_strm', 'int', [ + 'sqlite3_session*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xOutput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3session_config', 'int', ['int', 'void*']], + ['sqlite3session_create', 'int', ['sqlite3*', 'string', '**']], + //sqlite3session_delete() is bound manually + ['sqlite3session_diff', 'int', ['sqlite3_session*', 'string', 'string', '**']], + ['sqlite3session_enable', 'int', ['sqlite3_session*', 'int']], + ['sqlite3session_indirect', 'int', ['sqlite3_session*', 'int']], + ['sqlite3session_isempty', 'int', ['sqlite3_session*']], + ['sqlite3session_memory_used', 'i64', ['sqlite3_session*']], + ['sqlite3session_object_config', 'int', ['sqlite3_session*', 'int', 'void*']], + ['sqlite3session_patchset', 'int', ['sqlite3_session*', '*', '**']], + ['sqlite3session_patchset_strm', 'int', [ + 'sqlite3_session*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xOutput', signature: 'i(ppp)', bindScope: 'transient' + }), + 'void*' + ]], + ['sqlite3session_table_filter', undefined, [ + 'sqlite3_session*', + new wasm.xWrap.FuncPtrAdapter({ + name: 'xFilter', ...__ipsProxy, + contextKey: (argv,argIndex)=>argv[0/* (sqlite3_session*) */] + }), + '*' + ]] + ]); + }/*session/changeset APIs*/ + + /** + Functions which are intended solely for API-internal use by the + WASM components, not client code. These get installed into + sqlite3.wasm. Some of them get exposed to clients via variants + named sqlite3_js_...(). + */ + wasm.bindingSignatures.wasm = [ + ["sqlite3_wasm_db_reset", "int", "sqlite3*"], + ["sqlite3_wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"], + ["sqlite3_wasm_vfs_create_file", "int", + "sqlite3_vfs*","string","*", "int"], + ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] + ]; + /** Install JS<->C struct bindings for the non-opaque struct types we need... */ sqlite3.StructBinder = self.Jaccwabyt({ heap: 0 ? wasm.memory : wasm.heap8u, alloc: wasm.alloc, dealloc: wasm.dealloc, - functionTable: wasm.functionTable, bigIntEnabled: wasm.bigIntEnabled, - memberPrefix: '$' + memberPrefix: /* Never change this: this prefix is baked into any + amount of code and client-facing docs. */ '$' }); delete self.Jaccwabyt; - if(0){ - /* "The problem" is that the following isn't even remotely - type-safe. OTOH, nothing about WASM pointers is. */ - const argPointer = wasm.xWrap.argAdapter('*'); - wasm.xWrap.argAdapter('StructType', (v)=>{ - if(v && v.constructor && v instanceof StructBinder.StructType){ - v = v.pointer; - } - return wasm.isPtr(v) - ? argPointer(v) - : toss("Invalid (object) type for StructType-type argument."); - }); - } - {/* Convert Arrays and certain TypedArrays to strings for - 'flexible-string'-type arguments */ - const xString = wasm.xWrap.argAdapter('string'); + 'string:flexible'-type arguments */ + const __xString = wasm.xWrap.argAdapter('string'); + wasm.xWrap.argAdapter( + 'string:flexible', (v)=>__xString(util.flexibleString(v)) + ); + + /** + The 'string:static' argument adapter treats its argument as + either... + + - WASM pointer: assumed to be a long-lived C-string which gets + returned as-is. + + - Anything else: gets coerced to a JS string for use as a map + key. If a matching entry is found (as described next), it is + returned, else wasm.allocCString() is used to create a a new + string, map its pointer to (''+v) for the remainder of the + application's life, and returns that pointer value for this + call and all future calls which are passed a + string-equivalent argument. + + Use case: sqlite3_bind_pointer() and sqlite3_result_pointer() + call for "a static string and preferably a string + literal". This converter is used to ensure that the string + value seen by those functions is long-lived and behaves as they + need it to. + */ wasm.xWrap.argAdapter( - 'flexible-string', (v)=>xString(util.flexibleString(v)) + 'string:static', + function(v){ + if(wasm.isPtr(v)) return v; + v = ''+v; + let rc = this[v]; + return rc || (this[v] = wasm.allocCString(v)); + }.bind(Object.create(null)) ); - } - - if(1){// WhWasmUtil.xWrap() bindings... + }/* special-case string-type argument conversions */ + + if(1){// wasm.xWrap() bindings... /** Add some descriptive xWrap() aliases for '*' intended to (A) - initially improve readability/correctness of capi.signatures - and (B) eventually perhaps provide automatic conversion from - higher-level representations, e.g. capi.sqlite3_vfs to + initially improve readability/correctness of + wasm.bindingSignatures and (B) provide automatic conversion + from higher-level representations, e.g. capi.sqlite3_vfs to `sqlite3_vfs*` via capi.sqlite3_vfs.pointer. */ const aPtr = wasm.xWrap.argAdapter('*'); - wasm.xWrap.argAdapter('sqlite3*', aPtr) - ('sqlite3_stmt*', aPtr) + const nilType = function(){}; + wasm.xWrap.argAdapter('sqlite3_filename', aPtr) ('sqlite3_context*', aPtr) ('sqlite3_value*', aPtr) - ('sqlite3_vfs*', aPtr) - ('void*', aPtr); - wasm.xWrap.resultAdapter('sqlite3*', aPtr) - ('sqlite3_context*', aPtr) - ('sqlite3_stmt*', aPtr) - ('sqlite3_vfs*', aPtr) - ('void*', aPtr); + ('void*', aPtr) + ('sqlite3_changegroup*', aPtr) + ('sqlite3_changeset_iter*', aPtr) + //('sqlite3_rebaser*', aPtr) + ('sqlite3_session*', aPtr) + ('sqlite3_stmt*', (v)=> + aPtr((v instanceof (sqlite3?.oo1?.Stmt || nilType)) + ? v.pointer : v)) + ('sqlite3*', (v)=> + aPtr((v instanceof (sqlite3?.oo1?.DB || nilType)) + ? v.pointer : v)) + ('sqlite3_index_info*', (v)=> + aPtr((v instanceof (capi.sqlite3_index_info || nilType)) + ? v.pointer : v)) + ('sqlite3_module*', (v)=> + aPtr((v instanceof (capi.sqlite3_module || nilType)) + ? v.pointer : v)) + /** + `sqlite3_vfs*`: + + - v is-a string: use the result of sqlite3_vfs_find(v) but + throw if it returns 0. + - v is-a capi.sqlite3_vfs: use v.pointer. + - Else return the same as the `'*'` argument conversion. + */ + ('sqlite3_vfs*', (v)=>{ + if('string'===typeof v){ + /* A NULL sqlite3_vfs pointer will be treated as the default + VFS in many contexts. We specifically do not want that + behavior here. */ + return capi.sqlite3_vfs_find(v) + || sqlite3.SQLite3Error.toss( + capi.SQLITE_NOTFOUND, + "Unknown sqlite3_vfs name:", v + ); + } + return aPtr((v instanceof (capi.sqlite3_vfs || nilType)) + ? v.pointer : v); + }); + + const __xRcPtr = wasm.xWrap.resultAdapter('*'); + wasm.xWrap.resultAdapter('sqlite3*', __xRcPtr) + ('sqlite3_context*', __xRcPtr) + ('sqlite3_stmt*', __xRcPtr) + ('sqlite3_value*', __xRcPtr) + ('sqlite3_vfs*', __xRcPtr) + ('void*', __xRcPtr); /** Populate api object with sqlite3_...() by binding the "raw" wasm exports into type-converting proxies using wasm.xWrap(). */ @@ -90,51 +653,66 @@ for(const e of wasm.bindingSignatures.wasm){ wasm[e[0]] = wasm.xWrap.apply(null, e); } /* For C API functions which cannot work properly unless - wasm.bigIntEnabled is true, install a bogus impl which - throws if called when bigIntEnabled is false. */ + wasm.bigIntEnabled is true, install a bogus impl which throws + if called when bigIntEnabled is false. The alternative would be + to elide these functions altogether, which seems likely to + cause more confusion. */ const fI64Disabled = function(fname){ - return ()=>toss(fname+"() disabled due to lack", + return ()=>toss(fname+"() is unavailable due to lack", "of BigInt support in this build."); }; for(const e of wasm.bindingSignatures.int64){ capi[e[0]] = wasm.bigIntEnabled ? wasm.xWrap.apply(null, e) : fI64Disabled(e[0]); } - /* There's no(?) need to expose bindingSignatures to clients, + /* There's no need to expose bindingSignatures to clients, implicitly making it part of the public interface. */ delete wasm.bindingSignatures; if(wasm.exports.sqlite3_wasm_db_error){ - util.sqlite3_wasm_db_error = wasm.xWrap( + const __db_err = wasm.xWrap( 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' ); + /** + Sets the given db's error state. Accepts: + + - (sqlite3*, int code, string msg) + - (sqlite3*, Error e [,string msg = ''+e]) + + If passed a WasmAllocError, the message is ignored and the + result code is SQLITE_NOMEM. If passed any other Error type, + the result code defaults to SQLITE_ERROR unless the Error + object has a resultCode property, in which case that is used + (e.g. SQLite3Error has that). If passed a non-WasmAllocError + exception, the message string defaults to theError.message. + + Returns the resulting code. Pass (pDb,0,0) to clear the error + state. + */ + util.sqlite3_wasm_db_error = function(pDb, resultCode, message){ + if(resultCode instanceof sqlite3.WasmAllocError){ + resultCode = capi.SQLITE_NOMEM; + message = 0 /*avoid allocating message string*/; + }else if(resultCode instanceof Error){ + message = message || ''+resultCode; + resultCode = (resultCode.resultCode || capi.SQLITE_ERROR); + } + return __db_err(pDb, resultCode, message); + }; }else{ util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ console.warn("sqlite3_wasm_db_error() is not exported.",arguments); return errCode; }; } - }/*xWrap() bindings*/; - /** - When registering a VFS and its related components it may be - necessary to ensure that JS keeps a reference to them to keep - them from getting garbage collected. Simply pass each such value - to this function and a reference will be held to it for the life - of the app. - */ - capi.sqlite3_vfs_register.addReference = function f(...args){ - if(!f._) f._ = []; - f._.push(...args); - }; - /** Internal helper to assist in validating call argument counts in the hand-written sqlite3_xyz() wrappers. We do this only for consistency with non-special-case wrappings. */ @@ -142,282 +720,312 @@ return sqlite3.util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE, f+"() requires "+n+" argument"+ (1===n?"":'s')+"."); }; - /** - Helper for flexible-string conversions which require a - byte-length counterpart argument. Passed a value and its - ostensible length, this function returns [V,N], where V - is either v or a transformed copy of v and N is either n, - -1, or the byte length of v (if it's a byte array). - */ - const __flexiString = function(v,n){ - if('string'===typeof v){ - n = -1; - }else if(util.isSQLableTypedArray(v)){ - n = v.byteLength; - v = util.typedArrayToString(v); - }else if(Array.isArray(v)){ - v = v.join(""); - n = -1; - } - return [v, n]; - }; - - if(1){/* Special-case handling of sqlite3_exec() */ - const __exec = wasm.xWrap("sqlite3_exec", "int", - ["sqlite3*", "flexible-string", "*", "*", "**"]); - /* Documented in the api object's initializer. */ - capi.sqlite3_exec = function f(pDb, sql, callback, pVoid, pErrMsg){ - if(f.length!==arguments.length){ - return __dbArgcMismatch(pDb,"sqlite3_exec",f.length); - }else if('function' !== typeof callback){ - return __exec(pDb, sql, callback, pVoid, pErrMsg); - } - /* Wrap the callback in a WASM-bound function and convert the callback's - `(char**)` arguments to arrays of strings... */ - const cbwrap = function(pVoid, nCols, pColVals, pColNames){ - let rc = capi.SQLITE_ERROR; - try { - let aVals = [], aNames = [], i = 0, offset = 0; - for( ; i < nCols; offset += (wasm.ptrSizeof * ++i) ){ - aVals.push( wasm.cstringToJs(wasm.getPtrValue(pColVals + offset)) ); - aNames.push( wasm.cstringToJs(wasm.getPtrValue(pColNames + offset)) ); - } - rc = callback(pVoid, nCols, aVals, aNames) | 0; - /* The first 2 args of the callback are useless for JS but - we want the JS mapping of the C API to be as close to the - C API as possible. */ - }catch(e){ - /* If we set the db error state here, the higher-level exec() call - replaces it with its own, so we have no way of reporting the - exception message except the console. We must not propagate - exceptions through the C API. */ - } - return rc; - }; - let pFunc, rc; - try{ - pFunc = wasm.installFunction("ipipp", cbwrap); - rc = __exec(pDb, sql, pFunc, pVoid, pErrMsg); - }catch(e){ - rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, - "Error running exec(): "+e.message); - }finally{ - if(pFunc) wasm.uninstallFunction(pFunc); - } - return rc; - }; - }/*sqlite3_exec() proxy*/; - - if(1){/* Special-case handling of sqlite3_create_function_v2() - and sqlite3_create_window_function() */ - const sqlite3CreateFunction = wasm.xWrap( - "sqlite3_create_function_v2", "int", - ["sqlite3*", "string"/*funcName*/, "int"/*nArg*/, - "int"/*eTextRep*/, "*"/*pApp*/, - "*"/*xStep*/,"*"/*xFinal*/, "*"/*xValue*/, "*"/*xDestroy*/] - ); - const sqlite3CreateWindowFunction = wasm.xWrap( - "sqlite3_create_window_function", "int", - ["sqlite3*", "string"/*funcName*/, "int"/*nArg*/, - "int"/*eTextRep*/, "*"/*pApp*/, - "*"/*xStep*/,"*"/*xFinal*/, "*"/*xValue*/, - "*"/*xInverse*/, "*"/*xDestroy*/] - ); - - const __udfSetResult = function(pCtx, val){ - //console.warn("udfSetResult",typeof val, val); - switch(typeof val) { - case 'undefined': - /* Assume that the client already called sqlite3_result_xxx(). */ - break; - case 'boolean': - capi.sqlite3_result_int(pCtx, val ? 1 : 0); - break; - case 'bigint': - if(wasm.bigIntEnabled){ - if(util.bigIntFits64(val)) capi.sqlite3_result_int64(pCtx, val); - else toss3("BigInt value",val.toString(),"is too BigInt for int64."); - }else if(util.bigIntFits32(val)){ - capi.sqlite3_result_int(pCtx, Number(val)); - }else if(util.bigIntFitsDouble(val)){ - capi.sqlite3_result_double(pCtx, Number(val)); - }else{ - toss3("BigInt value",val.toString(),"is too BigInt."); - } - break; - case 'number': { - (util.isInt32(val) - ? capi.sqlite3_result_int - : capi.sqlite3_result_double)(pCtx, val); - break; - } - case 'string': - capi.sqlite3_result_text(pCtx, val, -1, capi.SQLITE_TRANSIENT); - break; - case 'object': - if(null===val/*yes, typeof null === 'object'*/) { - capi.sqlite3_result_null(pCtx); - break; - }else if(util.isBindableTypedArray(val)){ - const pBlob = wasm.allocFromTypedArray(val); - capi.sqlite3_result_blob( - pCtx, pBlob, val.byteLength, - wasm.exports[sqlite3.config.deallocExportName] - ); - break; - } - // else fall through - default: - toss3("Don't not how to handle this UDF result value:",(typeof val), val); - }; - }/*__udfSetResult()*/; - - const __udfConvertArgs = function(argc, pArgv){ - let i, pVal, valType, arg; - const tgt = []; - for(i = 0; i < argc; ++i){ - pVal = wasm.getPtrValue(pArgv + (wasm.ptrSizeof * i)); - /** - Curiously: despite ostensibly requiring 8-byte - alignment, the pArgv array is parcelled into chunks of - 4 bytes (1 pointer each). The values those point to - have 8-byte alignment but the individual argv entries - do not. - */ - valType = capi.sqlite3_value_type(pVal); - switch(valType){ - case capi.SQLITE_INTEGER: - if(wasm.bigIntEnabled){ - arg = capi.sqlite3_value_int64(pVal); - if(util.bigIntFitsDouble(arg)) arg = Number(arg); - } - else arg = capi.sqlite3_value_double(pVal)/*yes, double, for larger integers*/; - break; - case capi.SQLITE_FLOAT: - arg = capi.sqlite3_value_double(pVal); - break; - case capi.SQLITE_TEXT: - arg = capi.sqlite3_value_text(pVal); - break; - case capi.SQLITE_BLOB:{ - const n = capi.sqlite3_value_bytes(pVal); - const pBlob = capi.sqlite3_value_blob(pVal); - if(n && !pBlob) sqlite3.WasmAllocError.toss( - "Cannot allocate memory for blob argument of",n,"byte(s)" - ); - arg = n ? wasm.heap8u().slice(pBlob, pBlob + Number(n)) : null; - break; - } - case capi.SQLITE_NULL: - arg = null; break; - default: - toss3("Unhandled sqlite3_value_type()",valType, - "is possibly indicative of incorrect", - "pointer size assumption."); - } - tgt.push(arg); - } - return tgt; - }/*__udfConvertArgs()*/; - - const __udfSetError = (pCtx, e)=>{ - if(e instanceof sqlite3.WasmAllocError){ - capi.sqlite3_result_error_nomem(pCtx); - }else{ - const msg = ('string'===typeof e) ? e : e.message; - capi.sqlite3_result_error(pCtx, msg, -1); - } - }; - - const __xFunc = function(callback){ - return function(pCtx, argc, pArgv){ - try{ __udfSetResult(pCtx, callback(pCtx, ...__udfConvertArgs(argc, pArgv))) } - catch(e){ - //console.error('xFunc() caught:',e); - __udfSetError(pCtx, e); - } - }; - }; - - const __xInverseAndStep = function(callback){ - return function(pCtx, argc, pArgv){ - try{ callback(pCtx, ...__udfConvertArgs(argc, pArgv)) } - catch(e){ __udfSetError(pCtx, e) } - }; - }; - - const __xFinalAndValue = function(callback){ - return function(pCtx){ - try{ __udfSetResult(pCtx, callback(pCtx)) } - catch(e){ __udfSetError(pCtx, e) } - }; - }; - - const __xDestroy = function(callback){ - return function(pVoid){ - try{ callback(pVoid) } - catch(e){ console.error("UDF xDestroy method threw:",e) } - }; - }; - - const __xMap = Object.assign(Object.create(null), { - xFunc: {sig:'v(pip)', f:__xFunc}, - xStep: {sig:'v(pip)', f:__xInverseAndStep}, - xInverse: {sig:'v(pip)', f:__xInverseAndStep}, - xFinal: {sig:'v(p)', f:__xFinalAndValue}, - xValue: {sig:'v(p)', f:__xFinalAndValue}, - xDestroy: {sig:'v(p)', f:__xDestroy} - }); - - const __xWrapFuncs = function(theFuncs, tgtUninst){ - const rc = [] - let k; - for(k in theFuncs){ - let fArg = theFuncs[k]; - if('function'===typeof fArg){ - const w = __xMap[k]; - fArg = wasm.installFunction(w.sig, w.f(fArg)); - tgtUninst.push(fArg); - } - rc.push(fArg); - } - return rc; - }; + /** Code duplication reducer for functions which take an encoding + argument and require SQLITE_UTF8. Sets the db error code to + SQLITE_FORMAT and returns that code. */ + const __errEncoding = (pDb)=>{ + return util.sqlite3_wasm_db_error( + pDb, capi.SQLITE_FORMAT, "SQLITE_UTF8 is the only supported encoding." + ); + }; + + /** + __dbCleanupMap is infrastructure for recording registration of + UDFs and collations so that sqlite3_close_v2() can clean up any + automated JS-to-WASM function conversions installed by those. + */ + const __argPDb = (pDb)=>wasm.xWrap.argAdapter('sqlite3*')(pDb); + const __argStr = (str)=>wasm.isPtr(str) ? wasm.cstrToJs(str) : str; + const __dbCleanupMap = function( + pDb, mode/*0=remove, >0=create if needed, <0=do not create if missing*/ + ){ + pDb = __argPDb(pDb); + let m = this.dbMap.get(pDb); + if(!mode){ + this.dbMap.delete(pDb); + return m; + }else if(!m && mode>0){ + this.dbMap.set(pDb, (m = Object.create(null))); + } + return m; + }.bind(Object.assign(Object.create(null),{ + dbMap: new Map + })); + + __dbCleanupMap.addCollation = function(pDb, name){ + const m = __dbCleanupMap(pDb, 1); + if(!m.collation) m.collation = new Set; + m.collation.add(__argStr(name).toLowerCase()); + }; + + /** + Intended to be called _only_ from sqlite3_close_v2(), + passed its non-0 db argument. + + This function freees up certain automatically-installed WASM + function bindings which were installed on behalf of the given db, + as those may otherwise leak. + + Notable caveat: this is only ever run via + sqlite3.capi.sqlite3_close_v2(). If a client, for whatever + reason, uses sqlite3.wasm.exports.sqlite3_close_v2() (the + function directly exported from WASM), this cleanup will not + happen. + + This is not a silver bullet for avoiding automation-related + leaks but represents "an honest effort." + + The issue being addressed here is covered at: + + https://sqlite.org/wasm/doc/trunk/api-c-style.md#convert-func-ptr + */ + __dbCleanupMap.cleanup = function(pDb){ + pDb = __argPDb(pDb); + //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; + /** + Installing NULL functions in the C API will remove those + bindings. The FuncPtrAdapter which sits between us and the C + API will also treat that as an opportunity to + wasm.uninstallFunction() any WASM function bindings it has + installed for pDb. + */ + try{capi.sqlite3_busy_handler(pDb, 0, 0)} catch(e){/*ignored*/} + try{capi.sqlite3_progress_handler(pDb, 0, 0, 0)} catch(e){/*ignored*/} + try{capi.sqlite3_trace_v2(pDb, 0, 0, 0, 0)} catch(e){/*ignored*/} + try{capi.sqlite3_set_authorizer(pDb, 0, 0)} catch(e){/*ignored*/} + const m = __dbCleanupMap(pDb, 0); + if(!m) return; + if(m.collation){ + for(const name of m.collation){ + try{ + capi.sqlite3_create_collation_v2( + pDb, name, capi.SQLITE_UTF8, 0, 0, 0 + ); + }catch(e){ + /*ignored*/ + } + } + delete m.collation; + } + if(m.udf){ + //TODO: map and clean up UDFs. + } + }; + + {/* Binding of sqlite3_close_v2() */ + const __sqlite3CloseV2 = wasm.xWrap("sqlite3_close_v2", "int", "sqlite3*"); + capi.sqlite3_close_v2 = function(pDb){ + if(1!==arguments.length) return __dbArgcMismatch(pDb, 'sqlite3_close_v2', 1); + if(pDb){ + try{__dbCleanupMap.cleanup(pDb)} catch(e){/*ignored*/} + } + return __sqlite3CloseV2(pDb); + }; + }/*sqlite3_close_v2()*/ + + if(capi.sqlite3session_table_filter){ + const __sqlite3SessionDelete = wasm.xWrap( + 'sqlite3session_delete', undefined, ['sqlite3_session*'] + ); + capi.sqlite3session_delete = function(pSession){ + if(1!==arguments.length){ + return __dbArgcMismatch(pDb, 'sqlite3session_delete', 1); + /* Yes, we're returning a value from a void function. That seems + like the lesser evil compared to not maintaining arg-count + consistency as we do with other similar bindings. */ + } + else if(pSession){ + //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; + capi.sqlite3session_table_filter(pSession, 0, 0); + } + __sqlite3SessionDelete(pSession); + }; + } + + {/* Bindings for sqlite3_create_collation[_v2]() */ + // contextKey() impl for wasm.xWrap.FuncPtrAdapter + const contextKey = (argv,argIndex)=>{ + return 'argv['+argIndex+']:'+argv[0/* sqlite3* */]+ + ':'+wasm.cstrToJs(argv[1/* collation name */]).toLowerCase() + }; + const __sqlite3CreateCollationV2 = wasm.xWrap( + 'sqlite3_create_collation_v2', 'int', [ + 'sqlite3*', 'string', 'int', '*', + new wasm.xWrap.FuncPtrAdapter({ + /* int(*xCompare)(void*,int,const void*,int,const void*) */ + name: 'xCompare', signature: 'i(pipip)', contextKey + }), + new wasm.xWrap.FuncPtrAdapter({ + /* void(*xDestroy(void*) */ + name: 'xDestroy', signature: 'v(p)', contextKey + }) + ] + ); + + /** + Works exactly like C's sqlite3_create_collation_v2() except that: + + 1) It returns capi.SQLITE_FORMAT if the 3rd argument contains + any encoding-related value other than capi.SQLITE_UTF8. No + other encodings are supported. As a special case, if the + bottom 4 bits of that argument are 0, SQLITE_UTF8 is + assumed. + + 2) It accepts JS functions for its function-pointer arguments, + for which it will install WASM-bound proxies. The bindings + are "permanent," in that they will stay in the WASM environment + until it shuts down unless the client calls this again with the + same collation name and a value of 0 or null for the + the function pointer(s). + + For consistency with the C API, it requires the same number of + arguments. It returns capi.SQLITE_MISUSE if passed any other + argument count. + + Returns 0 on success, non-0 on error, in which case the error + state of pDb (of type `sqlite3*` or argument-convertible to it) + may contain more information. + */ + capi.sqlite3_create_collation_v2 = function(pDb,zName,eTextRep,pArg,xCompare,xDestroy){ + if(6!==arguments.length) return __dbArgcMismatch(pDb, 'sqlite3_create_collation_v2', 6); + else if( 0 === (eTextRep & 0xf) ){ + eTextRep |= capi.SQLITE_UTF8; + }else if( capi.SQLITE_UTF8 !== (eTextRep & 0xf) ){ + return __errEncoding(pDb); + } + try{ + const rc = __sqlite3CreateCollationV2(pDb, zName, eTextRep, pArg, xCompare, xDestroy); + if(xCompare) __dbCleanupMap.addCollation(pDb, zName); + return rc; + }catch(e){ + return util.sqlite3_wasm_db_error(pDb, e); + } + }; + + capi.sqlite3_create_collation = (pDb,zName,eTextRep,pArg,xCompare)=>{ + return (5===arguments.length) + ? capi.sqlite3_create_collation_v2(pDb,zName,eTextRep,pArg,xCompare,0) + : __dbArgcMismatch(pDb, 'sqlite3_create_collation', 5); + }; + + }/*sqlite3_create_collation() and friends*/ + + {/* Special-case handling of sqlite3_create_function_v2() + and sqlite3_create_window_function(). */ + /** FuncPtrAdapter for contextKey() for sqlite3_create_function() + and friends. */ + const contextKey = function(argv,argIndex){ + return ( + argv[0/* sqlite3* */] + +':'+(argv[2/*number of UDF args*/] < 0 ? -1 : argv[2]) + +':'+argIndex/*distinct for each xAbc callback type*/ + +':'+wasm.cstrToJs(argv[1]).toLowerCase() + ) + }; + + /** + JS proxies for the various sqlite3_create[_window]_function() + callbacks, structured in a form usable by wasm.xWrap.FuncPtrAdapter. + */ + const __cfProxy = Object.assign(Object.create(null), { + xInverseAndStep: { + signature:'v(pip)', contextKey, + callProxy: (callback)=>{ + return (pCtx, argc, pArgv)=>{ + try{ callback(pCtx, ...capi.sqlite3_values_to_js(argc, pArgv)) } + catch(e){ capi.sqlite3_result_error_js(pCtx, e) } + }; + } + }, + xFinalAndValue: { + signature:'v(p)', contextKey, + callProxy: (callback)=>{ + return (pCtx)=>{ + try{ capi.sqlite3_result_js(pCtx, callback(pCtx)) } + catch(e){ capi.sqlite3_result_error_js(pCtx, e) } + }; + } + }, + xFunc: { + signature:'v(pip)', contextKey, + callProxy: (callback)=>{ + return (pCtx, argc, pArgv)=>{ + try{ + capi.sqlite3_result_js( + pCtx, + callback(pCtx, ...capi.sqlite3_values_to_js(argc, pArgv)) + ); + }catch(e){ + //console.error('xFunc() caught:',e); + capi.sqlite3_result_error_js(pCtx, e); + } + }; + } + }, + xDestroy: { + signature:'v(p)', contextKey, + //Arguable: a well-behaved destructor doesn't require a proxy. + callProxy: (callback)=>{ + return (pVoid)=>{ + try{ callback(pVoid) } + catch(e){ console.error("UDF xDestroy method threw:",e) } + }; + } + } + })/*__cfProxy*/; + + const __sqlite3CreateFunction = wasm.xWrap( + "sqlite3_create_function_v2", "int", [ + "sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + new wasm.xWrap.FuncPtrAdapter({name: 'xFunc', ...__cfProxy.xFunc}), + new wasm.xWrap.FuncPtrAdapter({name: 'xStep', ...__cfProxy.xInverseAndStep}), + new wasm.xWrap.FuncPtrAdapter({name: 'xFinal', ...__cfProxy.xFinalAndValue}), + new wasm.xWrap.FuncPtrAdapter({name: 'xDestroy', ...__cfProxy.xDestroy}) + ] + ); + + const __sqlite3CreateWindowFunction = wasm.xWrap( + "sqlite3_create_window_function", "int", [ + "sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + new wasm.xWrap.FuncPtrAdapter({name: 'xStep', ...__cfProxy.xInverseAndStep}), + new wasm.xWrap.FuncPtrAdapter({name: 'xFinal', ...__cfProxy.xFinalAndValue}), + new wasm.xWrap.FuncPtrAdapter({name: 'xValue', ...__cfProxy.xFinalAndValue}), + new wasm.xWrap.FuncPtrAdapter({name: 'xInverse', ...__cfProxy.xInverseAndStep}), + new wasm.xWrap.FuncPtrAdapter({name: 'xDestroy', ...__cfProxy.xDestroy}) + ] + ); /* Documented in the api object's initializer. */ capi.sqlite3_create_function_v2 = function f( pDb, funcName, nArg, eTextRep, pApp, xFunc, //void (*xFunc)(sqlite3_context*,int,sqlite3_value**) xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) xFinal, //void (*xFinal)(sqlite3_context*) xDestroy //void (*xDestroy)(void*) ){ - if(f.length!==arguments.length){ + if( f.length!==arguments.length ){ return __dbArgcMismatch(pDb,"sqlite3_create_function_v2",f.length); + }else if( 0 === (eTextRep & 0xf) ){ + eTextRep |= capi.SQLITE_UTF8; + }else if( capi.SQLITE_UTF8 !== (eTextRep & 0xf) ){ + return __errEncoding(pDb); } - /* Wrap the callbacks in a WASM-bound functions... */ - const uninstall = [/*funcs to uninstall on error*/]; - let rc; try{ - const funcArgs = __xWrapFuncs({xFunc, xStep, xFinal, xDestroy}, - uninstall); - rc = sqlite3CreateFunction(pDb, funcName, nArg, eTextRep, - pApp, ...funcArgs); + return __sqlite3CreateFunction(pDb, funcName, nArg, eTextRep, + pApp, xFunc, xStep, xFinal, xDestroy); }catch(e){ console.error("sqlite3_create_function_v2() setup threw:",e); - for(let v of uninstall){ - wasm.uninstallFunction(v); - } - rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, - "Creation of UDF threw: "+e.message); + return util.sqlite3_wasm_db_error(pDb, e, "Creation of UDF threw: "+e); } - return rc; }; + /* Documented in the api object's initializer. */ capi.sqlite3_create_function = function f( pDb, funcName, nArg, eTextRep, pApp, xFunc, xStep, xFinal ){ return (f.length===arguments.length) @@ -429,127 +1037,111 @@ /* Documented in the api object's initializer. */ capi.sqlite3_create_window_function = function f( pDb, funcName, nArg, eTextRep, pApp, xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) xFinal, //void (*xFinal)(sqlite3_context*) - xValue, //void (*xFinal)(sqlite3_context*) - xInverse,//void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xValue, //void (*xValue)(sqlite3_context*) + xInverse,//void (*xInverse)(sqlite3_context*,int,sqlite3_value**) xDestroy //void (*xDestroy)(void*) ){ - if(f.length!==arguments.length){ + if( f.length!==arguments.length ){ return __dbArgcMismatch(pDb,"sqlite3_create_window_function",f.length); + }else if( 0 === (eTextRep & 0xf) ){ + eTextRep |= capi.SQLITE_UTF8; + }else if( capi.SQLITE_UTF8 !== (eTextRep & 0xf) ){ + return __errEncoding(pDb); } - /* Wrap the callbacks in a WASM-bound functions... */ - const uninstall = [/*funcs to uninstall on error*/]; - let rc; try{ - const funcArgs = __xWrapFuncs({xStep, xFinal, xValue, xInverse, xDestroy}, - uninstall); - rc = sqlite3CreateWindowFunction(pDb, funcName, nArg, eTextRep, - pApp, ...funcArgs); + return __sqlite3CreateWindowFunction(pDb, funcName, nArg, eTextRep, + pApp, xStep, xFinal, xValue, + xInverse, xDestroy); }catch(e){ console.error("sqlite3_create_window_function() setup threw:",e); - for(let v of uninstall){ - wasm.uninstallFunction(v); - } - rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, - "Creation of UDF threw: "+e.message); + return util.sqlite3_wasm_db_error(pDb, e, "Creation of UDF threw: "+e); } - return rc; }; /** - A helper for UDFs implemented in JS and bound to WASM by the - client. Given a JS value, udfSetResult(pCtx,X) calls one of the - sqlite3_result_xyz(pCtx,...) routines, depending on X's data - type: - - - `null`: sqlite3_result_null() - - `boolean`: sqlite3_result_int() - - `number`: sqlite3_result_int() or sqlite3_result_double() - - `string`: sqlite3_result_text() - - Uint8Array or Int8Array: sqlite3_result_blob() - - `undefined`: indicates that the UDF called one of the - `sqlite3_result_xyz()` routines on its own, making this - function a no-op. Results are _undefined_ if this function is - passed the `undefined` value but did _not_ call one of the - `sqlite3_result_xyz()` routines. - - Anything else triggers sqlite3_result_error(). + A _deprecated_ alias for capi.sqlite3_result_js() which + predates the addition of that function in the public API. */ capi.sqlite3_create_function_v2.udfSetResult = capi.sqlite3_create_function.udfSetResult = - capi.sqlite3_create_window_function.udfSetResult = __udfSetResult; + capi.sqlite3_create_window_function.udfSetResult = capi.sqlite3_result_js; /** - A helper for UDFs implemented in JS and bound to WASM by the - client. When passed the - (argc,argv) values from the UDF-related functions which receive - them (xFunc, xStep, xInverse), it creates a JS array - representing those arguments, converting each to JS in a manner - appropriate to its data type: numeric, text, blob - (Uint8Array), or null. - - Results are undefined if it's passed anything other than those - two arguments from those specific contexts. - - Thus an argc of 4 will result in a length-4 array containing - the converted values from the corresponding argv. - - The conversion will throw only on allocation error or an internal - error. + A _deprecated_ alias for capi.sqlite3_values_to_js() which + predates the addition of that function in the public API. */ capi.sqlite3_create_function_v2.udfConvertArgs = capi.sqlite3_create_function.udfConvertArgs = - capi.sqlite3_create_window_function.udfConvertArgs = __udfConvertArgs; + capi.sqlite3_create_window_function.udfConvertArgs = capi.sqlite3_values_to_js; /** - A helper for UDFs implemented in JS and bound to WASM by the - client. It expects to be a passed `(sqlite3_context*, Error)` - (an exception object or message string). And it sets the - current UDF's result to sqlite3_result_error_nomem() or - sqlite3_result_error(), depending on whether the 2nd argument - is a sqlite3.WasmAllocError object or not. + A _deprecated_ alias for capi.sqlite3_result_error_js() which + predates the addition of that function in the public API. */ capi.sqlite3_create_function_v2.udfSetError = capi.sqlite3_create_function.udfSetError = - capi.sqlite3_create_window_function.udfSetError = __udfSetError; + capi.sqlite3_create_window_function.udfSetError = capi.sqlite3_result_error_js; }/*sqlite3_create_function_v2() and sqlite3_create_window_function() proxies*/; if(1){/* Special-case handling of sqlite3_prepare_v2() and sqlite3_prepare_v3() */ + + /** + Helper for string:flexible conversions which require a + byte-length counterpart argument. Passed a value and its + ostensible length, this function returns [V,N], where V + is either v or a transformed copy of v and N is either n, + -1, or the byte length of v (if it's a byte array). + */ + const __flexiString = (v,n)=>{ + if('string'===typeof v){ + n = -1; + }else if(util.isSQLableTypedArray(v)){ + n = v.byteLength; + v = util.typedArrayToString(v); + }else if(Array.isArray(v)){ + v = v.join(""); + n = -1; + } + return [v, n]; + }; + /** Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). */ - const __prepare = Object.create(null); - /** - This binding expects a JS string as its 2nd argument and - null as its final argument. In order to compile multiple - statements from a single string, the "full" impl (see - below) must be used. - */ - __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', - "int", ["sqlite3*", "string", - "int"/*ignored for this impl!*/, - "int", "**", - "**"/*MUST be 0 or null or undefined!*/]); - /** - Impl which requires that the 2nd argument be a pointer - to the SQL string, instead of being converted to a - string. This variant is necessary for cases where we - require a non-NULL value for the final argument - (exec()'ing multiple statements from one input - string). For simpler cases, where only the first - statement in the SQL string is required, the wrapper - named sqlite3_prepare_v2() is sufficient and easier to - use because it doesn't require dealing with pointers. - */ - __prepare.full = wasm.xWrap('sqlite3_prepare_v3', - "int", ["sqlite3*", "*", "int", "int", - "**", "**"]); - - /* Documented in the api object's initializer. */ + const __prepare = { + /** + This binding expects a JS string as its 2nd argument and + null as its final argument. In order to compile multiple + statements from a single string, the "full" impl (see + below) must be used. + */ + basic: wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "string", + "int"/*ignored for this impl!*/, + "int", "**", + "**"/*MUST be 0 or null or undefined!*/]), + /** + Impl which requires that the 2nd argument be a pointer + to the SQL string, instead of being converted to a + string. This variant is necessary for cases where we + require a non-NULL value for the final argument + (exec()'ing multiple statements from one input + string). For simpler cases, where only the first + statement in the SQL string is required, the wrapper + named sqlite3_prepare_v2() is sufficient and easier to + use because it doesn't require dealing with pointers. + */ + full: wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "*", "int", "int", + "**", "**"]) + }; + + /* Documented in the capi object's initializer. */ capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ if(f.length!==arguments.length){ return __dbArgcMismatch(pDb,"sqlite3_prepare_v3",f.length); } const [xSql, xSqlLen] = __flexiString(sql, sqlLen); @@ -562,35 +1154,170 @@ "Invalid SQL argument type for sqlite3_prepare_v2/v3()." ); } }; - /* Documented in the api object's initializer. */ + /* Documented in the capi object's initializer. */ capi.sqlite3_prepare_v2 = function f(pDb, sql, sqlLen, ppStmt, pzTail){ return (f.length===arguments.length) ? capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail) : __dbArgcMismatch(pDb,"sqlite3_prepare_v2",f.length); }; - }/*sqlite3_prepare_v2/v3()*/; + + }/*sqlite3_prepare_v2/v3()*/ + + {/*sqlite3_bind_text/blob()*/ + const __bindText = wasm.xWrap("sqlite3_bind_text", "int", [ + "sqlite3_stmt*", "int", "string", "int", "*" + ]); + const __bindBlob = wasm.xWrap("sqlite3_bind_blob", "int", [ + "sqlite3_stmt*", "int", "*", "int", "*" + ]); + + /** Documented in the capi object's initializer. */ + capi.sqlite3_bind_text = function f(pStmt, iCol, text, nText, xDestroy){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(capi.sqlite3_db_handle(pStmt), + "sqlite3_bind_text", f.length); + }else if(wasm.isPtr(text) || null===text){ + return __bindText(pStmt, iCol, text, nText, xDestroy); + }else if(text instanceof ArrayBuffer){ + text = new Uint8Array(text); + }else if(Array.isArray(pMem)){ + text = pMem.join(''); + } + let p, n; + try{ + if(util.isSQLableTypedArray(text)){ + p = wasm.allocFromTypedArray(text); + n = text.byteLength; + }else if('string'===typeof text){ + [p, n] = wasm.allocCString(text); + }else{ + return util.sqlite3_wasm_db_error( + capi.sqlite3_db_handle(pStmt), capi.SQLITE_MISUSE, + "Invalid 3rd argument type for sqlite3_bind_text()." + ); + } + return __bindText(pStmt, iCol, p, n, capi.SQLITE_WASM_DEALLOC); + }catch(e){ + wasm.dealloc(p); + return util.sqlite3_wasm_db_error( + capi.sqlite3_db_handle(pStmt), e + ); + } + }/*sqlite3_bind_text()*/; + + /** Documented in the capi object's initializer. */ + capi.sqlite3_bind_blob = function f(pStmt, iCol, pMem, nMem, xDestroy){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(capi.sqlite3_db_handle(pStmt), + "sqlite3_bind_blob", f.length); + }else if(wasm.isPtr(pMem) || null===pMem){ + return __bindBlob(pStmt, iCol, pMem, nMem, xDestroy); + }else if(pMem instanceof ArrayBuffer){ + pMem = new Uint8Array(pMem); + }else if(Array.isArray(pMem)){ + pMem = pMem.join(''); + } + let p, n; + try{ + if(util.isBindableTypedArray(pMem)){ + p = wasm.allocFromTypedArray(pMem); + n = nMem>=0 ? nMem : pMem.byteLength; + }else if('string'===typeof pMem){ + [p, n] = wasm.allocCString(pMem); + }else{ + return util.sqlite3_wasm_db_error( + capi.sqlite3_db_handle(pStmt), capi.SQLITE_MISUSE, + "Invalid 3rd argument type for sqlite3_bind_blob()." + ); + } + return __bindBlob(pStmt, iCol, p, n, capi.SQLITE_WASM_DEALLOC); + }catch(e){ + wasm.dealloc(p); + return util.sqlite3_wasm_db_error( + capi.sqlite3_db_handle(pStmt), e + ); + } + }/*sqlite3_bind_blob()*/; + + }/*sqlite3_bind_text/blob()*/ + + {/* sqlite3_config() */ + /** + Wraps a small subset of the C API's sqlite3_config() options. + Unsupported options trigger the return of capi.SQLITE_NOTFOUND. + Passing fewer than 2 arguments triggers return of + capi.SQLITE_MISUSE. + */ + capi.sqlite3_config = function(op, ...args){ + if(arguments.length<2) return capi.SQLITE_MISUSE; + switch(op){ + case capi.SQLITE_CONFIG_COVERING_INDEX_SCAN: // 20 /* int */ + case capi.SQLITE_CONFIG_MEMSTATUS:// 9 /* boolean */ + case capi.SQLITE_CONFIG_SMALL_MALLOC: // 27 /* boolean */ + case capi.SQLITE_CONFIG_SORTERREF_SIZE: // 28 /* int nByte */ + case capi.SQLITE_CONFIG_STMTJRNL_SPILL: // 26 /* int nByte */ + case capi.SQLITE_CONFIG_URI:// 17 /* int */ + return wasm.exports.sqlite3_wasm_config_i(op, args[0]); + case capi.SQLITE_CONFIG_LOOKASIDE: // 13 /* int int */ + return wasm.exports.sqlite3_wasm_config_ii(op, args[0], args[1]); + case capi.SQLITE_CONFIG_MEMDB_MAXSIZE: // 29 /* sqlite3_int64 */ + return wasm.exports.sqlite3_wasm_config_j(op, args[0]); + case capi.SQLITE_CONFIG_GETMALLOC: // 5 /* sqlite3_mem_methods* */ + case capi.SQLITE_CONFIG_GETMUTEX: // 11 /* sqlite3_mutex_methods* */ + case capi.SQLITE_CONFIG_GETPCACHE2: // 19 /* sqlite3_pcache_methods2* */ + case capi.SQLITE_CONFIG_GETPCACHE: // 15 /* no-op */ + case capi.SQLITE_CONFIG_HEAP: // 8 /* void*, int nByte, int min */ + case capi.SQLITE_CONFIG_LOG: // 16 /* xFunc, void* */ + case capi.SQLITE_CONFIG_MALLOC:// 4 /* sqlite3_mem_methods* */ + case capi.SQLITE_CONFIG_MMAP_SIZE: // 22 /* sqlite3_int64, sqlite3_int64 */ + case capi.SQLITE_CONFIG_MULTITHREAD: // 2 /* nil */ + case capi.SQLITE_CONFIG_MUTEX: // 10 /* sqlite3_mutex_methods* */ + case capi.SQLITE_CONFIG_PAGECACHE: // 7 /* void*, int sz, int N */ + case capi.SQLITE_CONFIG_PCACHE2: // 18 /* sqlite3_pcache_methods2* */ + case capi.SQLITE_CONFIG_PCACHE: // 14 /* no-op */ + case capi.SQLITE_CONFIG_PCACHE_HDRSZ: // 24 /* int *psz */ + case capi.SQLITE_CONFIG_PMASZ: // 25 /* unsigned int szPma */ + case capi.SQLITE_CONFIG_SERIALIZED: // 3 /* nil */ + case capi.SQLITE_CONFIG_SINGLETHREAD: // 1 /* nil */: + case capi.SQLITE_CONFIG_SQLLOG: // 21 /* xSqllog, void* */ + case capi.SQLITE_CONFIG_WIN32_HEAPSIZE: // 23 /* int nByte */ + default: + return capi.SQLITE_NOTFOUND; + } + }; + }/* sqlite3_config() */ {/* Import C-level constants and structs... */ const cJson = wasm.xCall('sqlite3_wasm_enum_json'); if(!cJson){ toss("Maintenance required: increase sqlite3_wasm_enum_json()'s", "static buffer size!"); } - wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); - for(const t of ['access', 'blobFinalizers', 'dataTypes', - 'encodings', 'fcntl', 'flock', 'ioCap', - 'openFlags', 'prepareFlags', 'resultCodes', - 'serialize', 'syncFlags', 'trace', 'udfFlags', - 'version' - ]){ + wasm.ctype = JSON.parse(wasm.cstrToJs(cJson)); + // Groups of SQLITE_xyz macros... + const defineGroups = ['access', 'authorizer', + 'blobFinalizers', 'changeset', + 'config', 'dataTypes', + 'dbConfig', 'dbStatus', + 'encodings', 'fcntl', 'flock', 'ioCap', + 'limits', 'openFlags', + 'prepareFlags', 'resultCodes', + 'sqlite3Status', + 'stmtStatus', 'syncFlags', + 'trace', 'txnState', 'udfFlags', + 'version' ]; + if(wasm.bigIntEnabled){ + defineGroups.push('serialize', 'session', 'vtab'); + } + for(const t of defineGroups){ for(const e of Object.entries(wasm.ctype[t])){ - // ^^^ [k,v] there triggers a buggy code transormation via one - // of the Emscripten-driven optimizers. + // ^^^ [k,v] there triggers a buggy code transformation via + // one of the Emscripten-driven optimizers. capi[e[0]] = e[1]; } } const __rcMap = Object.create(null); for(const t of ['resultCodes']){ @@ -603,23 +1330,42 @@ string, or undefined if no such mapping is found. */ capi.sqlite3_js_rc_str = (rc)=>__rcMap[rc]; /* Bind all registered C-side structs... */ const notThese = Object.assign(Object.create(null),{ - // Structs NOT to register - WasmTestStruct: true + // For each struct to NOT register, map its name to true: + WasmTestStruct: true, + /* We unregister the kvvfs VFS from Worker threads below. */ + sqlite3_kvvfs_methods: !util.isUIThread(), + /* sqlite3_index_info and friends require int64: */ + sqlite3_index_info: !wasm.bigIntEnabled, + sqlite3_index_constraint: !wasm.bigIntEnabled, + sqlite3_index_orderby: !wasm.bigIntEnabled, + sqlite3_index_constraint_usage: !wasm.bigIntEnabled }); - if(!util.isUIThread()){ - /* We remove the kvvfs VFS from Worker threads below. */ - notThese.sqlite3_kvvfs_methods = true; - } for(const s of wasm.ctype.structs){ if(!notThese[s.name]){ capi[s.name] = sqlite3.StructBinder(s); } } - }/*end C constant imports*/ + if(capi.sqlite3_index_info){ + /* Move these inner structs into sqlite3_index_info. Binding + ** them to WASM requires that we create global-scope structs to + ** model them with, but those are no longer needed after we've + ** passed them to StructBinder. */ + for(const k of ['sqlite3_index_constraint', + 'sqlite3_index_orderby', + 'sqlite3_index_constraint_usage']){ + capi.sqlite3_index_info[k] = capi[k]; + delete capi[k]; + } + capi.sqlite3_vtab_config = wasm.xWrap( + 'sqlite3_wasm_vtab_config','int',[ + 'sqlite3*', 'int', 'int'] + ); + }/* end vtab-related setup */ + }/*end C constant and struct imports*/ const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); if( pKvvfs ){/* kvvfs-specific glue */ if(util.isUIThread()){ const kvvfsMethods = new capi.sqlite3_kvvfs_methods( @@ -626,15 +1372,14 @@ wasm.exports.sqlite3_wasm_kvvfs_methods() ); delete capi.sqlite3_kvvfs_methods; const kvvfsMakeKey = wasm.exports.sqlite3_wasm_kvvfsMakeKeyOnPstack, - pstack = wasm.pstack, - pAllocRaw = wasm.exports.sqlite3_wasm_pstack_alloc; + pstack = wasm.pstack; const kvvfsStorage = (zClass)=> - ((115/*=='s'*/===wasm.getMemValue(zClass)) + ((115/*=='s'*/===wasm.peek(zClass)) ? sessionStorage : localStorage); /** Implementations for members of the object referred to by sqlite3_wasm_kvvfs_methods(). We swap out the native @@ -646,25 +1391,25 @@ const stack = pstack.pointer, astack = wasm.scopedAllocPush(); try { const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return -3/*OOM*/; - const jKey = wasm.cstringToJs(zXKey); + const jKey = wasm.cstrToJs(zXKey); const jV = kvvfsStorage(zClass).getItem(jKey); if(!jV) return -1; const nV = jV.length /* Note that we are relying 100% on v being ASCII so that jV.length is equal to the C-string's byte length. */; if(nBuf<=0) return nV; else if(1===nBuf){ - wasm.setMemValue(zBuf, 0); + wasm.poke(zBuf, 0); return nV; } const zV = wasm.scopedAllocCString(jV); if(nBuf > nV + 1) nBuf = nV + 1; wasm.heap8u().copyWithin(zBuf, zV, zV + nBuf - 1); - wasm.setMemValue(zBuf + nBuf - 1, 0); + wasm.poke(zBuf + nBuf - 1, 0); return nBuf - 1; }catch(e){ console.error("kvstorageRead()",e); return -2; }finally{ @@ -675,12 +1420,12 @@ xWrite: (zClass, zKey, zData)=>{ const stack = pstack.pointer; try { const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return 1/*OOM*/; - const jKey = wasm.cstringToJs(zXKey); - kvvfsStorage(zClass).setItem(jKey, wasm.cstringToJs(zData)); + const jKey = wasm.cstrToJs(zXKey); + kvvfsStorage(zClass).setItem(jKey, wasm.cstrToJs(zData)); return 0; }catch(e){ console.error("kvstorageWrite()",e); return capi.SQLITE_IOERR; }finally{ @@ -690,11 +1435,11 @@ xDelete: (zClass, zKey)=>{ const stack = pstack.pointer; try { const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return 1/*OOM*/; - kvvfsStorage(zClass).removeItem(wasm.cstringToJs(zXKey)); + kvvfsStorage(zClass).removeItem(wasm.cstrToJs(zXKey)); return 0; }catch(e){ console.error("kvstorageDelete()",e); return capi.SQLITE_IOERR; }finally{ @@ -715,6 +1460,7 @@ be used that way but it's not really intended to be. */ capi.sqlite3_vfs_unregister(pKvvfs); } }/*pKvvfs*/ + wasm.xWrap.FuncPtrAdapter.warnOnUse = true; }); Index: ext/wasm/api/sqlite3-api-oo1.js ================================================================== --- ext/wasm/api/sqlite3-api-oo1.js +++ ext/wasm/api/sqlite3-api-oo1.js @@ -53,16 +53,17 @@ // Documented in DB.checkRc() const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ if(sqliteResultCode){ if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; toss3( - "sqlite result code",sqliteResultCode+":", + "sqlite3 result code",sqliteResultCode+":", (dbPtr ? capi.sqlite3_errmsg(dbPtr) : capi.sqlite3_errstr(sqliteResultCode)) ); } + return arguments[0]; }; /** sqlite3_trace_v2() callback which gets installed by the DB ctor if its open-flags contain "t". @@ -69,18 +70,20 @@ */ const __dbTraceToConsole = wasm.installFunction('i(ippp)', function(t,c,p,x){ if(capi.SQLITE_TRACE_STMT===t){ // x == SQL, p == sqlite3_stmt* - console.log("SQL TRACE #"+(++this.counter), - wasm.cstringToJs(x)); + console.log("SQL TRACE #"+(++this.counter)+' via sqlite3@'+c+':', + wasm.cstrToJs(x)); } }.bind({counter: 0})); /** - A map of sqlite3_vfs pointers to SQL code to run when the DB - constructor opens a database with the given VFS. + A map of sqlite3_vfs pointers to SQL code or a callback function + to run when the DB constructor opens a database with the given + VFS. In the latter case, the call signature is (theDbObject,sqlite3Namespace) + and the callback is expected to throw on error. */ const __vfsPostOpenSql = Object.create(null); /** A proxy for DB class constructors. It must be called with the @@ -134,11 +137,11 @@ || 'string'!==typeof flagsStr || (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){ console.error("Invalid DB ctor args",opt,arguments); toss3("Invalid arguments for DB constructor."); } - let fnJs = ('number'===typeof fn) ? wasm.cstringToJs(fn) : fn; + let fnJs = ('number'===typeof fn) ? wasm.cstrToJs(fn) : fn; const vfsCheck = ctor._name2vfs[fnJs]; if(vfsCheck){ vfsName = vfsCheck.vfs; fn = fnJs = vfsCheck.filename(fnJs); } @@ -151,40 +154,54 @@ oflags |= capi.SQLITE_OPEN_EXRESCODE; const stack = wasm.pstack.pointer; try { const pPtr = wasm.pstack.allocPtr() /* output (sqlite3**) arg */; let rc = capi.sqlite3_open_v2(fn, pPtr, oflags, vfsName || 0); - pDb = wasm.getPtrValue(pPtr); + pDb = wasm.peekPtr(pPtr); checkSqlite3Rc(pDb, rc); + capi.sqlite3_extended_result_codes(pDb, 1); if(flagsStr.indexOf('t')>=0){ capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, - __dbTraceToConsole, 0); - } - // Check for per-VFS post-open SQL... - const pVfs = capi.sqlite3_js_db_vfs(pDb); - //console.warn("Opened db",fn,"with vfs",vfsName,pVfs); - if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); - const postInitSql = __vfsPostOpenSql[pVfs]; - if(postInitSql){ - rc = capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0); - checkSqlite3Rc(pDb, rc); - } + __dbTraceToConsole, pDb); + } }catch( e ){ if( pDb ) capi.sqlite3_close_v2(pDb); throw e; }finally{ wasm.pstack.restore(stack); } this.filename = fnJs; __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); + try{ + // Check for per-VFS post-open SQL/callback... + const pVfs = capi.sqlite3_js_db_vfs(pDb); + if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); + const postInitSql = __vfsPostOpenSql[pVfs]; + if(postInitSql instanceof Function){ + postInitSql(this, sqlite3); + }else if(postInitSql){ + checkSqlite3Rc( + pDb, capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0) + ); + } + }catch(e){ + this.close(); + throw e; + } }; /** Sets SQL which should be exec()'d on a DB instance after it is - opened with the given VFS pointer. This is intended only for use - by DB subclasses or sqlite3_vfs implementations. + opened with the given VFS pointer. The SQL may be any type + supported by the "string:flexible" function argument conversion. + Alternately, the 2nd argument may be a function, in which case it + is called with (theOo1DbObject,sqlite3Namespace) at the end of + the DB() constructor. The function must throw on error, in which + case the db is closed and the exception is propagated. This + function is intended only for use by DB subclasses or sqlite3_vfs + implementations. */ dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ __vfsPostOpenSql[pVfs] = sql; }; @@ -198,13 +215,12 @@ If passed an object, any additional properties it has are copied as-is into the new object. */ dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ const arg = {}; - if(1===arguments.length && 'object'===typeof arguments[0]){ - const x = arguments[0]; - Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(1===arguments.length && arguments[0] && 'object'===typeof arguments[0]){ + Object.assign(arg, arguments[0]); if(undefined===arg.flags) arg.flags = 'c'; if(undefined===arg.vfs) arg.vfs = null; if(undefined===arg.filename) arg.filename = ':memory:'; }else{ arg.filename = filename; @@ -319,11 +335,11 @@ - `parameterCount`: the number of bindable paramters in the query. */ const Stmt = function(){ if(BindTypes!==arguments[2]){ - toss3("Do not call the Stmt constructor directly. Use DB.prepare()."); + toss3(capi.SQLITE_MISUSE, "Do not call the Stmt constructor directly. Use DB.prepare()."); } this.db = arguments[0]; __ptrMap.set(this, arguments[1]); this.columnCount = capi.sqlite3_column_count(this.pointer); this.parameterCount = capi.sqlite3_bind_parameter_count(this.pointer); @@ -421,57 +437,69 @@ break; default: if(util.isInt32(opt.rowMode)){ out.cbArg = (stmt)=>stmt.get(opt.rowMode); break; - }else if('string'===typeof opt.rowMode && opt.rowMode.length>1){ - /* "$X", ":X", and "@X" fetch column named "X" (case-sensitive!) */ - const prefix = opt.rowMode[0]; - if(':'===prefix || '@'===prefix || '$'===prefix){ - out.cbArg = function(stmt){ - const rc = stmt.get(this.obj)[this.colName]; - return (undefined===rc) ? toss3("exec(): unknown result column:",this.colName) : rc; - }.bind({ - obj:Object.create(null), - colName: opt.rowMode.substr(1) - }); - break; - } + }else if('string'===typeof opt.rowMode + && opt.rowMode.length>1 + && '$'===opt.rowMode[0]){ + /* "$X": fetch column named "X" (case-sensitive!). Prior + to 2022-12-14 ":X" and "@X" were also permitted, but + having so many options is unnecessary and likely to + cause confusion. */ + const $colName = opt.rowMode.substr(1); + out.cbArg = (stmt)=>{ + const rc = stmt.get(Object.create(null))[$colName]; + return (undefined===rc) + ? toss3(capi.SQLITE_NOTFOUND, + "exec(): unknown result column:",$colName) + : rc; + }; + break; } toss3("Invalid rowMode:",opt.rowMode); } } return out; }; /** - Internal impl of the DB.selectArray() and + Internal impl of the DB.selectValue(), selectArray(), and selectObject() methods. */ - const __selectFirstRow = (db, sql, bind, getArg)=>{ - let stmt, rc; + const __selectFirstRow = (db, sql, bind, ...getArgs)=>{ + const stmt = db.prepare(sql); try { - stmt = db.prepare(sql).bind(bind); - if(stmt.step()) rc = stmt.get(getArg); + return stmt.bind(bind).step() ? stmt.get(...getArgs) : undefined; }finally{ - if(stmt) stmt.finalize(); + stmt.finalize(); } - return rc; }; + /** + Internal impl of the DB.selectArrays() and selectObjects() + methods. + */ + const __selectAll = + (db, sql, bind, rowMode)=>db.exec({ + sql, bind, rowMode, returnValue: 'resultRows' + }); + /** Expects to be given a DB instance or an `sqlite3*` pointer (may be null) and an sqlite3 API result code. If the result code is not falsy, this function throws an SQLite3Error with an error - message from sqlite3_errmsg(), using dbPtr as the db handle, or - sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a - non-error code like SQLITE_ROW or SQLITE_DONE, it will still - throw but the error string might be "Not an error." The various - non-0 non-error codes need to be checked for in - client code where they are expected. + message from sqlite3_errmsg(), using db (or, if db is-a DB, + db.pointer) as the db handle, or sqlite3_errstr() if db is + falsy. Note that if it's passed a non-error code like SQLITE_ROW + or SQLITE_DONE, it will still throw but the error string might be + "Not an error." The various non-0 non-error codes need to be + checked for in client code where they are expected. + + If it does not throw, it returns its first argument. */ - DB.checkRc = checkSqlite3Rc; + DB.checkRc = (db,resultCode)=>checkSqlite3Rc(db,resultCode); DB.prototype = { /** Returns true if this db handle is open, else false. */ isOpen: function(){ return !!this.pointer; @@ -492,14 +520,16 @@ If this.onclose.after is a function then it is called after the db is closed but before auxiliary state like this.filename is cleared. - Both onclose handlers are passed this object. If this db is not - opened, neither of the handlers are called. Any exceptions the - handlers throw are ignored because "destructors must not - throw." + Both onclose handlers are passed this object, with the onclose + object as their "this," noting that the db will have been + closed when onclose.after is called. If this db is not opened + when close() is called, neither of the handlers are called. Any + exceptions the handlers throw are ignored because "destructors + must not throw." Note that garbage collection of a db handle, if it happens at all, will never trigger close(), so onclose handlers are not a reliable way to implement close-time cleanup or maintenance of a db. @@ -572,14 +602,14 @@ const pVfs = capi.sqlite3_js_db_vfs( affirmDbOpen(this).pointer, dbName ); if(pVfs){ const v = new capi.sqlite3_vfs(pVfs); - try{ rc = wasm.cstringToJs(v.$zName) } + try{ rc = wasm.cstrToJs(v.$zName) } finally { v.dispose() } } - return rc; + return rc; }, /** Compiles the given SQL and returns a prepared Stmt. This is the only way to create new Stmt objects. Throws on error. @@ -606,11 +636,11 @@ const stack = wasm.pstack.pointer; let ppStmt, pStmt; try{ ppStmt = wasm.pstack.alloc(8)/* output (sqlite3_stmt**) arg */; DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); - pStmt = wasm.getPtrValue(ppStmt); + pStmt = wasm.peekPtr(ppStmt); } finally { wasm.pstack.restore(stack); } if(!pStmt) toss3("Cannot prepare empty SQL."); @@ -664,25 +694,31 @@ any) is triggered (regardless of whether the query produces any result rows). If no statement has result columns, this value is unchanged. Achtung: an SQL result may have multiple columns with identical names. - - `callback` = a function which gets called for each row of - the result set, but only if that statement has any result + - `callback` = a function which gets called for each row of the + result set, but only if that statement has any result _rows_. The callback's "this" is the options object, noting that this function synthesizes one if the caller does not pass one to exec(). The second argument passed to the callback is always the current Stmt object, as it's needed if the caller wants to fetch the column names or some such (noting that they could also be fetched via `this.columnNames`, if the client - provides the `columnNames` option). + provides the `columnNames` option). If the callback returns a + literal `false` (as opposed to any other falsy value, e.g. an + implicit `undefined` return), any ongoing statement-`step()` + iteration stops without an error. The return value of the + callback is otherwise ignored. ACHTUNG: The callback MUST NOT modify the Stmt object. Calling any of the Stmt.get() variants, Stmt.getColumnName(), or similar, is legal, but calling step() or finalize() is not. Member methods which are illegal in this context will - trigger an exception. + trigger an exception, but clients must also refrain from using + any lower-level (C-style) APIs which might modify the + statement. The first argument passed to the callback defaults to an array of values from the current result row but may be changed with ... - `rowMode` = specifies the type of he callback's first argument. @@ -709,21 +745,19 @@ B) An integer, indicating a zero-based column in the result row. Only that one single value will be passed on. C) A string with a minimum length of 2 and leading character of - ':', '$', or '@' will fetch the row as an object, extract that - one field, and pass that field's value to the callback. Note - that these keys are case-sensitive so must match the case used - in the SQL. e.g. `"select a A from t"` with a `rowMode` of - `'$A'` would work but `'$a'` would not. A reference to a column - not in the result set will trigger an exception on the first - row (as the check is not performed until rows are fetched). - Note also that `$` is a legal identifier character in JS so - need not be quoted. (Design note: those 3 characters were - chosen because they are the characters support for naming bound - parameters.) + '$' will fetch the row as an object, extract that one field, + and pass that field's value to the callback. Note that these + keys are case-sensitive so must match the case used in the + SQL. e.g. `"select a A from t"` with a `rowMode` of `'$A'` + would work but `'$a'` would not. A reference to a column not in + the result set will trigger an exception on the first row (as + the check is not performed until rows are fetched). Note also + that `$` is a legal identifier character in JS so need not be + quoted. Any other `rowMode` value triggers an exception. - `resultRows`: if this is an array, it functions similarly to the `callback` option: each row of the result set (if any), @@ -768,49 +802,50 @@ const callback = opt.callback; const resultRows = Array.isArray(opt.resultRows) ? opt.resultRows : undefined; let stmt; let bind = opt.bind; - let evalFirstResult = !!(arg.cbArg || opt.columnNames) /* true to evaluate the first result-returning query */; + let evalFirstResult = !!( + arg.cbArg || opt.columnNames || resultRows + ) /* true to step through the first result-returning statement */; const stack = wasm.scopedAllocPush(); + const saveSql = Array.isArray(opt.saveSql) ? opt.saveSql : undefined; try{ const isTA = util.isSQLableTypedArray(arg.sql) /* Optimization: if the SQL is a TypedArray we can save some string conversion costs. */; /* Allocate the two output pointers (ppStmt, pzTail) and heap space for the SQL (pSql). When prepare_v2() returns, pzTail will point to somewhere in pSql. */ let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); - const ppStmt = wasm.scopedAlloc(/* output (sqlite3_stmt**) arg and pzTail */ - (2 * wasm.ptrSizeof) - + (sqlByteLen + 1/* SQL + NUL */)); + const ppStmt = wasm.scopedAlloc( + /* output (sqlite3_stmt**) arg and pzTail */ + (2 * wasm.ptrSizeof) + (sqlByteLen + 1/* SQL + NUL */) + ); const pzTail = ppStmt + wasm.ptrSizeof /* final arg to sqlite3_prepare_v2() */; let pSql = pzTail + wasm.ptrSizeof; const pSqlEnd = pSql + sqlByteLen; if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); - wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); - while(pSql && wasm.getMemValue(pSql, 'i8') + wasm.poke(pSql + sqlByteLen, 0/*NUL terminator*/); + while(pSql && wasm.peek(pSql, 'i8') /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we use i32 or similar then we read 4 bytes, read stuff around the NUL terminator, and get stuck in and endless loop at the end of the SQL, endlessly re-preparing an empty statement. */ ){ - wasm.setPtrValue(ppStmt, 0); - wasm.setPtrValue(pzTail, 0); + wasm.pokePtr([ppStmt, pzTail], 0); DB.checkRc(this, capi.sqlite3_prepare_v3( this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail )); - const pStmt = wasm.getPtrValue(ppStmt); - pSql = wasm.getPtrValue(pzTail); + const pStmt = wasm.peekPtr(ppStmt); + pSql = wasm.peekPtr(pzTail); sqlByteLen = pSqlEnd - pSql; if(!pStmt) continue; - if(Array.isArray(opt.saveSql)){ - opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); - } + if(saveSql) saveSql.push(capi.sqlite3_sql(pStmt).trim()); stmt = new Stmt(this, pStmt, BindTypes); if(bind && stmt.parameterCount){ stmt.bind(bind); bind = null; } @@ -819,15 +854,19 @@ in the SQL which potentially has them. */ evalFirstResult = false; if(Array.isArray(opt.columnNames)){ stmt.getColumnNames(opt.columnNames); } - while(!!arg.cbArg && stmt.step()){ - stmt._isLocked = true; - const row = arg.cbArg(stmt); - if(resultRows) resultRows.push(row); - if(callback) callback.call(opt, row, stmt); + if(arg.cbArg || resultRows){ + for(; stmt.step(); stmt._isLocked = false){ + stmt._isLocked = true; + const row = arg.cbArg(stmt); + if(resultRows) resultRows.push(row); + if(callback && false === callback.call(opt, row, stmt)){ + break; + } + } stmt._isLocked = false; } }else{ stmt.step(); } @@ -844,14 +883,15 @@ } wasm.scopedAllocPop(stack); } return arg.returnVal(); }/*exec()*/, + /** - Creates a new scalar UDF (User-Defined Function) which is - accessible via SQL code. This function may be called in any - of the following forms: + Creates a new UDF (User-Defined Function) which is accessible + via SQL code. This function may be called in any of the + following forms: - (name, function) - (name, function, optionsObject) - (name, optionsObject) - (optionsObject) @@ -863,14 +903,16 @@ The first two call forms can only be used for creating scalar functions. Creating an aggregate or window function requires the options-object form (see below for details). - UDFs cannot currently be removed from a DB handle after they're - added. More correctly, they can be removed as documented for - sqlite3_create_function_v2(), but doing so will "leak" the - JS-created WASM binding of those functions. + UDFs can be removed as documented for + sqlite3_create_function_v2() and + sqlite3_create_window_function(), but doing so will "leak" the + JS-created WASM binding of those functions (meaning that their + entries in the WASM indirect function table still + exist). Eliminating that potential leak is a pending TODO. On success, returns this object. Throws on error. When called from SQL arguments to the UDF, and its result, will be converted between JS and SQL with as much fidelity as @@ -1037,19 +1079,35 @@ the same as not passing a value. Throws on error (e.g. malformed SQL). */ selectValue: function(sql,bind,asType){ - let stmt, rc; + return __selectFirstRow(this, sql, bind, 0, asType); + }, + + /** + Runs the given query and returns an array of the values from + the first result column of each row of the result set. The 2nd + argument is an optional value for use in a single-argument call + to Stmt.bind(). The 3rd argument may be any value suitable for + use as the 2nd argument to Stmt.get(). If a 3rd argument is + desired but no bind data are needed, pass `undefined` for the 2nd + argument. + + If there are no result rows, an empty array is returned. + */ + selectValues: function(sql,bind,asType){ + const stmt = this.prepare(sql), rc = []; try { - stmt = this.prepare(sql).bind(bind); - if(stmt.step()) rc = stmt.get(0,asType); + stmt.bind(bind); + while(stmt.step()) rc.push(stmt.get(0,asType)); }finally{ - if(stmt) stmt.finalize(); + stmt.finalize(); } return rc; }, + /** Prepares the given SQL, step()s it one time, and returns an array containing the values of the first result row. If it has no results, `undefined` is returned. @@ -1078,14 +1136,37 @@ Throws on error (e.g. malformed SQL). */ selectObject: function(sql,bind){ return __selectFirstRow(this, sql, bind, {}); }, + + /** + Runs the given SQL and returns an array of all results, with + each row represented as an array, as per the 'array' `rowMode` + option to `exec()`. An empty result set resolves + to an empty array. The second argument, if any, is treated as + the 'bind' option to a call to exec(). + */ + selectArrays: function(sql,bind){ + return __selectAll(this, sql, bind, 'array'); + }, + + /** + Works identically to selectArrays() except that each value + in the returned array is an object, as per the 'object' `rowMode` + option to `exec()`. + */ + selectObjects: function(sql,bind){ + return __selectAll(this, sql, bind, 'object'); + }, /** Returns the number of currently-opened Stmt handles for this db - handle, or 0 if this DB instance is closed. + handle, or 0 if this DB instance is closed. Note that only + handles prepared via this.prepare() are counted, and not + handles prepared using capi.sqlite3_prepare_v3() (or + equivalent). */ openStatementCount: function(){ return this.pointer ? Object.keys(__stmtMap.get(this)).length : 0; }, @@ -1128,10 +1209,18 @@ return rc; }catch(e){ this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); throw e; } + }, + + /** + A convenience form of DB.checkRc(this,resultCode). If it does + not throw, it returns this object. + */ + checkRc: function(resultCode){ + return DB.checkRc(this, resultCode); } }/*DB.prototype*/; /** Throws if the given Stmt has been finalized, else stmt is @@ -1156,11 +1245,12 @@ case BindTypes.bigint: if(wasm.bigIntEnabled) return t; /* else fall through */ default: //console.log("isSupportedBindType",t,v); - return util.isBindableTypedArray(v) ? BindTypes.blob : undefined; + return (util.isBindableTypedArray(v) || (v instanceof ArrayBuffer)) + ? BindTypes.blob : undefined; } }; /** If isSupportedBindType(v) returns a truthy value, this @@ -1210,43 +1300,23 @@ given index (numeric or named) using the given bindType (see the BindTypes enum) and value. Throws on error. Returns stmt on success. */ const bindOne = function f(stmt,ndx,bindType,val){ - affirmUnlocked(stmt, 'bind()'); + affirmUnlocked(affirmStmtOpen(stmt), 'bind()'); if(!f._){ f._tooBigInt = (v)=>toss3( "BigInt value is too big to store without precision loss:", v ); /* Reminder: when not in BigInt mode, it's impossible for JS to represent a number out of the range we can bind, so we have no range checking. */ f._ = { string: function(stmt, ndx, val, asBlob){ - if(1){ - /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = wasm.scopedAllocPush(); - try{ - const n = wasm.jstrlen(val); - const pStr = wasm.scopedAlloc(n); - wasm.jstrcpy(val, wasm.heap8u(), pStr, n, false); - const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; - return f(stmt.pointer, ndx, pStr, n, capi.SQLITE_TRANSIENT); - }finally{ - wasm.scopedAllocPop(stack); - } - }else{ - const bytes = wasm.jstrToUintArray(val,false); - const pStr = wasm.alloc(bytes.length || 1); - wasm.heap8u().set(bytes.length ? bytes : [0], pStr); - try{ - const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; - return f(stmt.pointer, ndx, pStr, bytes.length, capi.SQLITE_TRANSIENT); - }finally{ - wasm.dealloc(pStr); - } - } + const [pStr, n] = wasm.allocCString(val, true); + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, n, capi.SQLITE_WASM_DEALLOC); } }; }/* static init */ affirmSupportedBindType(val); ndx = affirmParamIndex(stmt,ndx); @@ -1287,40 +1357,29 @@ rc = capi.sqlite3_bind_int(stmt.pointer, ndx, val ? 1 : 0); break; case BindTypes.blob: { if('string'===typeof val){ rc = f._.string(stmt, ndx, val, true); + break; + }else if(val instanceof ArrayBuffer){ + val = new Uint8Array(val); }else if(!util.isBindableTypedArray(val)){ toss3("Binding a value as a blob requires", - "that it be a string, Uint8Array, or Int8Array."); - }else if(1){ - /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = wasm.scopedAllocPush(); - try{ - const pBlob = wasm.scopedAlloc(val.byteLength || 1); - wasm.heap8().set(val.byteLength ? val : [0], pBlob) - rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, - capi.SQLITE_TRANSIENT); - }finally{ - wasm.scopedAllocPop(stack); - } - }else{ - const pBlob = wasm.allocFromTypedArray(val); - try{ - rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, - capi.SQLITE_TRANSIENT); - }finally{ - wasm.dealloc(pBlob); - } - } + "that it be a string, Uint8Array, Int8Array, or ArrayBuffer."); + } + const pBlob = wasm.alloc(val.byteLength || 1); + wasm.heap8().set(val.byteLength ? val : [0], pBlob) + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_WASM_DEALLOC); break; } default: console.warn("Unsupported bind() argument type:",val); toss3("Unsupported bind() argument type: "+(typeof val)); } if(rc) DB.checkRc(stmt.db.pointer, rc); + stmt._mayGet = false; return stmt; }; Stmt.prototype = { /** @@ -1404,12 +1463,12 @@ throw. - Strings are bound as strings (use bindAsBlob() to force blob binding). - - Uint8Array and Int8Array instances are bound as blobs. - (TODO: binding the other TypedArray types.) + - Uint8Array, Int8Array, and ArrayBuffer instances are bound as + blobs. (TODO? binding the other TypedArray types.) If passed an array, each element of the array is bound at the parameter index equal to the array index plus 1 (because arrays are 0-based but binding is 1-based). @@ -1460,12 +1519,14 @@ if(1!==arguments.length){ toss3("When binding an array, an index argument is not permitted."); } arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); return this; + }else if(arg instanceof ArrayBuffer){ + arg = new Uint8Array(arg); } - else if('object'===typeof arg/*null was checked above*/ + if('object'===typeof arg/*null was checked above*/ && !util.isBindableTypedArray(arg)){ /* Treat each property of arg as a named bound parameter. */ if(1!==arguments.length){ toss3("When binding an object, an index argument is not permitted."); } @@ -1483,11 +1544,11 @@ Special case of bind() which binds the given value using the BLOB binding mechanism instead of the default selected one for the value. The ndx may be a numbered or named bind index. The value must be of type string, null/undefined (both get treated as null), or a TypedArray of a type supported by the bind() - API. + API. This API cannot bind numbers as blobs. If passed a single argument, a bind index of 1 is assumed and the first argument is the value. */ bindAsBlob: function(ndx,arg){ @@ -1499,13 +1560,11 @@ const t = affirmSupportedBindType(arg); if(BindTypes.string !== t && BindTypes.blob !== t && BindTypes.null !== t){ toss3("Invalid value type for bindAsBlob()"); } - bindOne(this, ndx, BindTypes.blob, arg); - this._mayGet = false; - return this; + return bindOne(this, ndx, BindTypes.blob, arg); }, /** Steps the statement one time. If the result indicates that a row of data is available, a truthy value is returned. If no row of data is available, a falsy @@ -1567,11 +1626,11 @@ this.finalize(); return rc; }, /** Fetches the value from the given 0-based column index of - the current data row, throwing if index is out of range. + the current data row, throwing if index is out of range. Requires that step() has just returned a truthy value, else an exception is thrown. By default it will determine the data type of the result @@ -1747,14 +1806,10 @@ Object.defineProperty(DB.prototype, 'pointer', prop); } /** The OO API's public namespace. */ sqlite3.oo1 = { - version: { - lib: capi.sqlite3_libversion(), - ooApi: "0.1" - }, DB, Stmt }/*oo1 object*/; if(util.isUIThread()){ DELETED ext/wasm/api/sqlite3-api-opfs.js Index: ext/wasm/api/sqlite3-api-opfs.js ================================================================== --- ext/wasm/api/sqlite3-api-opfs.js +++ /dev/null @@ -1,1311 +0,0 @@ -/* - 2022-09-18 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file holds the synchronous half of an sqlite3_vfs - implementation which proxies, in a synchronous fashion, the - asynchronous Origin-Private FileSystem (OPFS) APIs using a second - Worker, implemented in sqlite3-opfs-async-proxy.js. This file is - intended to be appended to the main sqlite3 JS deliverable somewhere - after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. -*/ -'use strict'; -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -/** - installOpfsVfs() returns a Promise which, on success, installs an - sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs - which accept a VFS. It is intended to be called via - sqlite3ApiBootstrap.initializersAsync or an equivalent mechanism. - - The installed VFS uses the Origin-Private FileSystem API for - all file storage. On error it is rejected with an exception - explaining the problem. Reasons for rejection include, but are - not limited to: - - - The counterpart Worker (see below) could not be loaded. - - - The environment does not support OPFS. That includes when - this function is called from the main window thread. - - Significant notes and limitations: - - - As of this writing, OPFS is still very much in flux and only - available in bleeding-edge versions of Chrome (v102+, noting that - that number will increase as the OPFS API matures). - - - The OPFS features used here are only available in dedicated Worker - threads. This file tries to detect that case, resulting in a - rejected Promise if those features do not seem to be available. - - - It requires the SharedArrayBuffer and Atomics classes, and the - former is only available if the HTTP server emits the so-called - COOP and COEP response headers. These features are required for - proxying OPFS's synchronous API via the synchronous interface - required by the sqlite3_vfs API. - - - This function may only be called a single time. When called, this - function removes itself from the sqlite3 object. - - All arguments to this function are for internal/development purposes - only. They do not constitute a public API and may change at any - time. - - The argument may optionally be a plain object with the following - configuration options: - - - proxyUri: as described above - - - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables - logging of errors. 2 enables logging of warnings and errors. 3 - additionally enables debugging info. - - - sanityChecks (=false): if true, some basic sanity tests are - run on the OPFS VFS API after it's initialized, before the - returned Promise resolves. - - On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. -*/ -const installOpfsVfs = function callee(options){ - if(!self.SharedArrayBuffer || - !self.Atomics || - !self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ - return Promise.reject( - new Error("This environment does not have OPFS support.") - ); - } - if(!options || 'object'!==typeof options){ - options = Object.create(null); - } - const urlParams = new URL(self.location.href).searchParams; - if(undefined===options.verbose){ - options.verbose = urlParams.has('opfs-verbose') ? 3 : 2; - } - if(undefined===options.sanityChecks){ - options.sanityChecks = urlParams.has('opfs-sanity-check'); - } - if(undefined===options.proxyUri){ - options.proxyUri = callee.defaultProxyUri; - } - - if('function' === typeof options.proxyUri){ - options.proxyUri = options.proxyUri(); - } - const thePromise = new Promise(function(promiseResolve, promiseReject_){ - const loggers = { - 0:console.error.bind(console), - 1:console.warn.bind(console), - 2:console.log.bind(console) - }; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = function(...args){throw new Error(args.join(' '))}; - const capi = sqlite3.capi; - const wasm = sqlite3.wasm; - const sqlite3_vfs = capi.sqlite3_vfs; - const sqlite3_file = capi.sqlite3_file; - const sqlite3_io_methods = capi.sqlite3_io_methods; - /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. - */ - const opfsUtil = Object.create(null); - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { - dump: function(){ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; - } - console.log(self.location.href, - "metrics for",self.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t, - "ms (incl. "+w+" ms of waiting on the async side)"); - console.log("Serialization metrics:",metrics.s11n); - W.postMessage({type:'opfs-async-metrics'}); - }, - reset: function(){ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - } - }/*metrics*/; - const promiseReject = function(err){ - opfsVfs.dispose(); - return promiseReject_(err); - }; - const W = new Worker(options.proxyUri); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; - const opfsVfs = new sqlite3_vfs(); - const opfsIoMethods = new sqlite3_io_methods(); - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/*sure, why not?*/; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.ondispose = [ - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), - 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() - ]; - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); - state.verbose = options.verbose; - state.littleEndian = (()=>{ - const buffer = new ArrayBuffer(2); - new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); - // Int16Array uses the platform's endianness. - return new Int16Array(buffer)[0] === 256; - })(); - /** - Whether the async counterpart should log exceptions to - the serialization channel. That produces a great deal of - noise for seemingly innocuous things like xAccess() checks - for missing files, so this option may have one of 3 values: - - 0 = no exception logging - - 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). - - 2 = log all exceptions. - */ - state.asyncS11nExceptions = 1; - /* Size of file I/O buffer block. 64k = max sqlite3 page size, and - xRead/xWrite() will never deal in blocks larger than that. */ - state.fileBufferSize = 1024 * 64; - state.sabS11nOffset = state.fileBufferSize; - /** - The size of the block in our SAB for serializing arguments and - result values. Needs to be large enough to hold serialized - values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. - */ - state.sabS11nSize = opfsVfs.$mxPathname * 2; - /** - The SAB used for all data I/O between the synchronous and - async halves (file i/o and arg/result s11n). - */ - state.sabIO = new SharedArrayBuffer( - state.fileBufferSize/* file i/o block */ - + state.sabS11nSize/* argument/result serialization block */ - ); - state.opIds = Object.create(null); - const metrics = Object.create(null); - { - /* Indexes for use in our SharedArrayBuffer... */ - let i = 0; - /* SAB slot used to communicate which operation is desired - between both workers. This worker writes to it and the other - listens for changes. */ - state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ - state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xDeleteNoWait = i++; - state.opIds.xFileControl = i++; - state.opIds.xFileSize = i++; - state.opIds.xLock = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xUnlock = i++; - state.opIds.xWrite = i++; - state.opIds.mkdir = i++; - state.opIds['opfs-async-metrics'] = i++; - state.opIds['opfs-async-shutdown'] = i++; - /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ - state.opIds.retry = i++; - state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); - } - /** - SQLITE_xxx constants to export to the async worker - counterpart... - */ - state.sq3Codes = Object.create(null); - [ - 'SQLITE_ACCESS_EXISTS', - 'SQLITE_ACCESS_READWRITE', - 'SQLITE_ERROR', - 'SQLITE_IOERR', - 'SQLITE_IOERR_ACCESS', - 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_LOCK', - 'SQLITE_IOERR_READ', - 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_TRUNCATE', - 'SQLITE_IOERR_UNLOCK', - 'SQLITE_IOERR_WRITE', - 'SQLITE_LOCK_EXCLUSIVE', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_SHARED', - 'SQLITE_MISUSE', - 'SQLITE_NOTFOUND', - 'SQLITE_OPEN_CREATE', - 'SQLITE_OPEN_DELETEONCLOSE', - 'SQLITE_OPEN_READONLY' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); - - /** - Runs the given operation (by name) in the async worker - counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the aruguments for the - async op. - */ - const opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); - state.s11n.serialize(...args); - Atomics.store(state.sabOPView, state.opIds.rc, -1); - Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - /* async thread will take over here */; - const t = performance.now(); - Atomics.wait(state.sabOPView, state.opIds.rc, -1) - /* When this wait() call returns, the async half will have - completed the operation and reported its results. */; - const rc = Atomics.load(state.sabOPView, state.opIds.rc); - metrics[op].wait += performance.now() - t; - if(rc && state.asyncS11nExceptions){ - const err = state.s11n.deserialize(); - if(err) error(op+"() async error:",...err); - } - return rc; - }; - - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; - - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset]. Only one dataset is recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(''); - }; - - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error (e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc.). - - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - */ - const installMethod = function callee(tgt, name, func){ - if(!(tgt instanceof sqlite3.StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - const fProxy = 0 - /** This middle-man proxy is only for use during development, to - confirm that we always pass the proper number of - arguments. We know that the C-level code will always use the - correct argument count. */ - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - - const opTimer = Object.create(null); - opTimer.op = undefined; - opTimer.start = undefined; - const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start - ); - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - /** - As of late 2022, only a single lock can be held on an OPFS - file. We have no way of checking whether any _other_ db - connection has a lock except by trying to obtain and (on - success) release a sync-handle for it, but doing so would - involve an inherent race condition. For the time being, - pending a better solution, we simply report whether the - given pFile instance has a lock. - */ - const f = __openFiles[pFile]; - wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); - return 0; - }, - xClose: function(pFile){ - mTimeStart('xClose'); - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - mTimeEnd(); - return rc; - }, - xDeviceCharacteristics: function(pFile){ - //debug("xDeviceCharacteristics(",pFile,")"); - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - mTimeStart('xFileControl'); - const rc = (capi.SQLITE_FCNTL_SYNC===opId) - ? opRun('xSync', pFile, 0) - : capi.SQLITE_NOTFOUND; - mTimeEnd(); - return rc; - }, - xFileSize: function(pFile,pSz64){ - mTimeStart('xFileSize'); - const rc = opRun('xFileSize', pFile); - if(0==rc){ - const sz = state.s11n.deserialize()[0]; - wasm.setMemValue(pSz64, sz, 'i64'); - } - mTimeEnd(); - return rc; - }, - xLock: function(pFile,lockType){ - mTimeStart('xLock'); - const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === f.lockType ) { - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - }else{ - f.lockType = lockType; - } - mTimeEnd(); - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - mTimeStart('xRead'); - const f = __openFiles[pFile]; - let rc; - try { - rc = opRun('xRead',pFile, n, Number(offset64)); - if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ - /** - Results get written to the SharedArrayBuffer f.sabView. - Because the heap is _not_ a SharedArrayBuffer, we have - to copy the results. TypedArray.set() seems to be the - fastest way to copy this. */ - wasm.heap8u().set(f.sabView.subarray(0, n), pDest); - } - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - mTimeEnd(); - return rc; - }, - xSync: function(pFile,flags){ - ++metrics.xSync.count; - return 0; // impl'd in xFileControl() - }, - xTruncate: function(pFile,sz64){ - mTimeStart('xTruncate'); - const rc = opRun('xTruncate', pFile, Number(sz64)); - mTimeEnd(); - return rc; - }, - xUnlock: function(pFile,lockType){ - mTimeStart('xUnlock'); - const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType - && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } - if( 0===rc ) f.lockType = lockType; - mTimeEnd(); - return rc; - }, - xWrite: function(pFile,pSrc,n,offset64){ - mTimeStart('xWrite'); - const f = __openFiles[pFile]; - let rc; - try { - f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n)); - rc = opRun('xWrite', pFile, n, Number(offset64)); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_WRITE; - } - mTimeEnd(); - return rc; - } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstringToJs(zName)); - wasm.setMemValue( pOut, (rc ? 0 : 1), 'i32' ); - mTimeEnd(); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - // TODO: confirm that this calculation is correct - wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - mTimeStart('xDelete'); - opRun('xDelete', wasm.cstringToJs(zName), doSyncDir, false); - /* We're ignoring errors because we cannot yet differentiate - between harmless and non-harmless failures. */ - mTimeEnd(); - return 0; - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return ipMethods is NULL. */ - if(fh.readOnly){ - wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); - } - __openFiles[pFile] = fh; - fh.sabView = state.sabFileBufView; - fh.sq3File = new sqlite3_file(pFile); - fh.sq3File.$pMethods = opfsIoMethods.pointer; - fh.lockType = capi.SQLITE_LOCK_NONE; - } - mTimeEnd(); - return rc; - }/*xOpen()*/ - }/*vfsSyncWrappers*/; - - if(dVfs){ - opfsVfs.$xRandomness = dVfs.$xRandomness; - opfsVfs.$xSleep = dVfs.$xSleep; - } - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - /* If we can inherit an xSleep() impl from the default VFS then - assume it's sane and use it, otherwise install a JS-based - one. */ - vfsSyncWrappers.xSleep = function(pVfs,ms){ - Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); - return 0; - }; - } - - /* Install the vfs/io_methods into their C-level shared instances... */ - for(let k of Object.keys(ioSyncWrappers)){ - installMethod(opfsIoMethods, k, ioSyncWrappers[k]); - } - for(let k of Object.keys(vfsSyncWrappers)){ - installMethod(opfsVfs, k, vfsSyncWrappers[k]); - } - - /** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg - is true, the result is returned as an array of path elements, - else an absolute path string is returned. - */ - opfsUtil.getResolvedPath = function(filename,splitIt){ - const p = new URL(filename, "file://irrelevant").pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //console.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Re-registers the OPFS VFS. This is intended only for odd use - cases which have to call sqlite3_shutdown() as part of their - initialization process, which will unregister the VFS - registered by installOpfsVfs(). If passed a truthy value, the - OPFS VFS is registered as the default VFS, else it is not made - the default. Returns the result of the the - sqlite3_vfs_register() call. - - Design note: the problem of having to re-register things after - a shutdown/initialize pair is more general. How to best plug - that in to the library is unclear. In particular, we cannot - hook in to any C-side calls to sqlite3_initialize(), so we - cannot add an after-initialize callback mechanism. - */ - opfsUtil.registerVfs = (asDefault=false)=>{ - return wasm.exports.sqlite3_vfs_register( - opfsVfs.pointer, asDefault ? 1 : 0 - ); - }; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which it will throw - are not documented (so we cannot tell you what they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each one. - The argument may be either a callback function or an options object - with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - - TODO: add an option which specifies whether to traverse - depth-first or breadth-first. We currently do depth-first but - an incremental file browsing widget would benefit more from - breadth-first. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - //TODO to support fiddle and worker1 db upload: - //opfsUtil.createFile = function(absName, content=undefined){...} - - if(sqlite3.oo1){ - opfsUtil.OpfsDb = function(...args){ - const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; - sqlite3.oo1.DB.dbCtorHelper.call(this, opt); - }; - opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( - opfsVfs.pointer, - [ - /* Truncate journal mode is faster than delete or wal for - this vfs, per speedtest1. */ - "pragma journal_mode=truncate;" - /* - This vfs benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We currently - rely on setting a non-default cache size when building - sqlite3.wasm. If that policy changes, the cache can - be set here. - */ - //"pragma cache_size=-8388608;" - ].join('') - ); - } - - /** - Potential TODOs: - - - Expose one or both of the Worker objects via opfsUtil and - publish an interface for proxying the higher-level OPFS - features like getting a directory listing. - */ - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.getMemValue(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.getMemValue(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.setMemValue(pOut,0,'i64'); - rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); - rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.setMemValue(readBuf+6,0); - let jRead = wasm.cstringToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.getMemValue(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-async-loaded': - /*Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async worker.*/ - W.postMessage({type: 'opfs-async-init',args: state}); - break; - case 'opfs-async-inited':{ - /*Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin...*/ - try { - const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0); - if(rc){ - toss("sqlite3_vfs_register(OPFS) failed with rc",rc); - } - if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ - toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); - } - capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - initS11n(); - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - navigator.storage.getDirectory().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(sqlite3); - }); - }catch(e){ - error(e); - promiseReject(e); - } - break; - } - default: - promiseReject(e); - error("Unexpected message from the async worker:",data); - break; - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; -}/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; -self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){ - return; - } - try{ - let proxyJs = installOpfsVfs.defaultProxyUri; - if(sqlite3.scriptInfo.sqlite3Dir){ - installOpfsVfs.defaultProxyUri = - sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //console.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); - } - return installOpfsVfs().catch((e)=>{ - console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); - }); - }catch(e){ - console.error("installOpfsVfs() exception:",e); - throw e; - } -}); -}/*sqlite3ApiBootstrap.initializers.push()*/); Index: ext/wasm/api/sqlite3-api-prologue.js ================================================================== --- ext/wasm/api/sqlite3-api-prologue.js +++ ext/wasm/api/sqlite3-api-prologue.js @@ -9,88 +9,34 @@ * May you share freely, never taking more than you give. *********************************************************************** This file is intended to be combined at build-time with other - related code, most notably a header and footer which wraps this whole - file into an Emscripten Module.postRun() handler which has a parameter - named "Module" (the Emscripten Module object). The exact requirements, - conventions, and build process are very much under construction and - will be (re)documented once they've stopped fluctuating so much. + related code, most notably a header and footer which wraps this + whole file into an Emscripten Module.postRun() handler which has a + parameter named "Module" (the Emscripten Module object). The sqlite3 + JS API has no hard requirements on Emscripten, and does not expose + any Emscripten APIs to clients. It is structured such that its build + can be tweaked to include it in arbitrary WASM environments which + supply the necessary underlying features (e.g. a POSIX file I/O + layer). - Project home page: https://sqlite.org + Main project home page: https://sqlite.org Documentation home page: https://sqlite.org/wasm - - Specific goals of this subproject: - - - Except where noted in the non-goals, provide a more-or-less - feature-complete wrapper to the sqlite3 C API, insofar as WASM - feature parity with C allows for. In fact, provide at least 4 - APIs... - - 1) 1-to-1 bindings as exported from WASM, with no automatic - type conversions between JS and C. - - 2) A binding of (1) which provides certain JS/C type conversions - to greatly simplify its use. - - 3) A higher-level API, more akin to sql.js and node.js-style - implementations. This one speaks directly to the low-level - API. This API must be used from the same thread as the - low-level API. - - 4) A second higher-level API which speaks to the previous APIs via - worker messages. This one is intended for use in the main - thread, with the lower-level APIs installed in a Worker thread, - and talking to them via Worker messages. Because Workers are - asynchronouns and have only a single message channel, some - acrobatics are needed here to feed async work results back to - the client (as we cannot simply pass around callbacks between - the main and Worker threads). - - - Insofar as possible, support client-side storage using JS - filesystem APIs. As of this writing, such things are still very - much under development. - - Specific non-goals of this project: - - - As WASM is a web-centric technology and UTF-8 is the King of - Encodings in that realm, there are no currently plans to support - the UTF16-related sqlite3 APIs. They would add a complication to - the bindings for no appreciable benefit. Though web-related - implementation details take priority, and the JavaScript - components of the API specifically focus on browser clients, the - lower-level WASM module "should" work in non-web WASM - environments. - - - Supporting old or niche-market platforms. WASM is built for a - modern web and requires modern platforms. - - - Though scalar User-Defined Functions (UDFs) may be created in - JavaScript, there are currently no plans to add support for - aggregate and window functions. - - Attribution: - - This project is endebted to the work of sql.js: - - https://github.com/sql-js/sql.js - - sql.js was an essential stepping stone in this code's development as - it demonstrated how to handle some of the WASM-related voodoo (like - handling pointers-to-pointers and adding JS implementations of - C-bound callback functions). These APIs have a considerably - different shape than sql.js's, however. */ /** sqlite3ApiBootstrap() is the only global symbol persistently exposed by this API. It is intended to be called one time at the end of the API amalgamation process, passed configuration details for the current environment, and then optionally be removed from the global object using `delete self.sqlite3ApiBootstrap`. + + This function is not intended for client-level use. It is intended + for use in creating bundles configured for specific WASM + environments. This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so that it can be used without any _direct_ dependency on Emscripten. (Note the default values for the config object!) The @@ -117,23 +63,27 @@ will throw exceptions if called without BigInt support, as BigInt is required for marshalling C-side int64 into and out of JS. - `allocExportName`: the name of the function, in `exports`, of the `malloc(3)`-compatible routine for the WASM environment. Defaults - to `"malloc"`. + to `"sqlite3_malloc"`. Beware that using any allocator other than + sqlite3_malloc() may require care in certain client-side code + regarding which allocator is uses. Notably, sqlite3_deserialize() + and sqlite3_serialize() can only safely use memory from different + allocators under very specific conditions. - `deallocExportName`: the name of the function, in `exports`, of the `free(3)`-compatible routine for the WASM - environment. Defaults to `"free"`. - - - `wasmfsOpfsDir`[^1]: if the environment supports persistent - storage, this directory names the "mount point" for that - directory. It must be prefixed by `/` and may contain only a - single directory-name part. Using the root directory name is not - supported by any current persistent backend. This setting is - only used in WASMFS-enabled builds. - + environment. Defaults to `"sqlite3_free"`. + + - `reallocExportName`: the name of the function, in `exports`, of + the `realloc(3)`-compatible routine for the WASM + environment. Defaults to `"sqlite3_realloc"`. + + - `wasmfsOpfsDir`[^1]: As of 2022-12-17, this feature does not + currently work due to incompatible Emscripten-side changes made + in the WASMFS+OPFS combination. This option is currently ignored. [^1] = This property may optionally be a function, in which case this function re-assigns it to the value returned from that function, enabling delayed evaluation. @@ -156,14 +106,26 @@ -sWASM_BIGINT=1, else it will not. */ return !!Module.HEAPU64; } return !!self.BigInt64Array; })(), - allocExportName: 'malloc', - deallocExportName: 'free', - wasmfsOpfsDir: '/opfs' + wasmfsOpfsDir: '/opfs', + /** + useStdAlloc is just for testing an allocator discrepancy. The + docs guarantee that this is false in the canonical builds. For + 99% of purposes it doesn't matter which allocators we use, but + it becomes significant with, e.g., sqlite3_deserialize() + and certain wasm.xWrap.resultAdapter()s. + */ + useStdAlloc: false }, apiConfig || {}); + + Object.assign(config, { + allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc', + deallocExportName: config.useStdAlloc ? 'free' : 'sqlite3_free', + reallocExportName: config.useStdAlloc ? 'realloc' : 'sqlite3_realloc' + }, config); [ // If any of these config options are functions, replace them with // the result of calling that function... 'exports', 'memory', 'wasmfsOpfsDir' @@ -170,10 +132,14 @@ ].forEach((k)=>{ if('function' === typeof config[k]){ config[k] = config[k](); } }); + config.wasmOpfsDir = + /* 2022-12-17: WASMFS+OPFS can no longer be activated from the + main thread (aborts via a failed assert() if it's attempted), + which eliminates any(?) benefit to supporting it. */ false; /** The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as @@ -189,15 +155,11 @@ C-side counterparts. */ const capi = Object.create(null); /** Holds state which are specific to the WASM-related - infrastructure and glue code. It is not expected that client - code will normally need these, but they're exposed here in case - it does. These APIs are _not_ to be considered an - official/stable part of the sqlite3 WASM API. They may change - as the developers' experience suggests appropriate changes. + infrastructure and glue code. Note that a number of members of this object are injected dynamically after the api object is fully constructed, so not all are documented in this file. */ @@ -221,32 +183,53 @@ */ class SQLite3Error extends Error { /** Constructs this object with a message depending on its arguments: - - If it's passed only a single integer argument, it is assumed - to be an sqlite3 C API result code. The message becomes the - result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns - falsy) a synthesized string which contains that integer. - - - If passed 2 arguments and the 2nd is a object, it bevaves - like the Error(string,object) constructor except that the first - argument is subject to the is-integer semantics from the - previous point. - - - Else all arguments are concatenated with a space between each - one, using args.join(' '), to create the error message. + If its first argument is an integer, it is assumed to be + an SQLITE_... result code and it is passed to + sqlite3.capi.sqlite3_js_rc_str() to stringify it. + + If called with exactly 2 arguments and the 2nd is an object, + that object is treated as the 2nd argument to the parent + constructor. + + The exception's message is created by concatenating its + arguments with a space between each, except for the + two-args-with-an-objec form and that the first argument will + get coerced to a string, as described above, if it's an + integer. + + If passed an integer first argument, the error object's + `resultCode` member will be set to the given integer value, + else it will be set to capi.SQLITE_ERROR. */ constructor(...args){ - if(1===args.length && __isInt(args[0])){ - super(__rcStr(args[0])); - }else if(2===args.length && 'object'===typeof args){ - if(__isInt(args[0])) super(__rcStr(args[0]), args[1]); - else super(...args); - }else{ - super(args.join(' ')); - } + let rc; + if(args.length){ + if(__isInt(args[0])){ + rc = args[0]; + if(1===args.length){ + super(__rcStr(args[0])); + }else{ + const rcStr = __rcStr(rc); + if('object'===typeof args[1]){ + super(rcStr,args[1]); + }else{ + args[0] = rcStr+':'; + super(args.join(' ')); + } + } + }else{ + if(2===args.length && 'object'===typeof args[1]){ + super(...args); + }else{ + super(args.join(' ')); + } + } + } + this.resultCode = rc || capi.SQLITE_ERROR; this.name = 'SQLite3Error'; } }; /** @@ -336,16 +319,21 @@ ? aTypedArray.slice(begin, end) : aTypedArray.subarray(begin, end); }; /** - Returns true if v appears to be one of our bind()-able - TypedArray types: Uint8Array or Int8Array. Support for - TypedArrays with element sizes >1 is TODO. + Returns v if v appears to be one of our bind()-able TypedArray + types: Uint8Array or Int8Array or ArrayBuffer. Support for + TypedArrays with element sizes >1 is a potential TODO just + waiting on a use case to justify them. Until then, their `buffer` + property can be used to pass them as an ArrayBuffer. If it's not + a bindable array type, a falsy value is returned. */ const isBindableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + return v && (v instanceof Uint8Array + || v instanceof Int8Array + || v instanceof ArrayBuffer); }; /** Returns true if v appears to be one of the TypedArray types which is legal for holding SQL code (as opposed to binary blobs). @@ -354,11 +342,13 @@ seems likely that we'll eventually want to add Uint32Array and friends to the isBindableTypedArray() list but not to the isSQLableTypedArray() list. */ const isSQLableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + return v && (v instanceof Uint8Array + || v instanceof Int8Array + || v instanceof ArrayBuffer); }; /** Returns true if isBindableTypedArray(v) does, else throws with a message that v is not a supported TypedArray value. */ const affirmBindableTypedArray = (v)=>{ @@ -381,17 +371,17 @@ }; /** If v is-a Array, its join("") result is returned. If isSQLableTypedArray(v) is true then typedArrayToString(v) is - returned. If it looks like a WASM pointer, wasm.cstringToJs(v) is + returned. If it looks like a WASM pointer, wasm.cstrToJs(v) is returned. Else v is returned as-is. */ const flexibleString = function(v){ if(isSQLableTypedArray(v)) return typedArrayToString(v); else if(Array.isArray(v)) return v.join(""); - else if(wasm.isPtr(v)) v = wasm.cstringToJs(v); + else if(wasm.isPtr(v)) v = wasm.cstrToJs(v); return v; }; /** An Error subclass specifically for reporting Wasm-level malloc() @@ -406,17 +396,18 @@ construct an error message string. As a special case, if called with no arguments then it uses a default error message. */ constructor(...args){ - if(2===args.length && 'object'===typeof args){ + if(2===args.length && 'object'===typeof args[1]){ super(...args); }else if(args.length){ super(args.join(' ')); }else{ super("Allocation failed."); } + this.resultCode = capi.SQLITE_NOMEM; this.name = 'WasmAllocError'; } }; /** Functionally equivalent to the WasmAllocError constructor but may @@ -429,25 +420,117 @@ WasmAllocError.toss = (...args)=>{ throw new WasmAllocError(...args); }; Object.assign(capi, { + /** + sqlite3_bind_blob() works exactly like its C counterpart unless + its 3rd argument is one of: + + - JS string: the 3rd argument is converted to a C string, the + 4th argument is ignored, and the C-string's length is used + in its place. + + - Array: converted to a string as defined for "flexible + strings" and then it's treated as a JS string. + + - Int8Array or Uint8Array: wasm.allocFromTypedArray() is used to + conver the memory to the WASM heap. If the 4th argument is + 0 or greater, it is used as-is, otherwise the array's byteLength + value is used. This is an exception to the C API's undefined + behavior for a negative 4th argument, but results are undefined + if the given 4th argument value is greater than the byteLength + of the input array. + + - If it's an ArrayBuffer, it gets wrapped in a Uint8Array and + treated as that type. + + In all of those cases, the final argument (destructor) is + ignored and capi.SQLITE_WASM_DEALLOC is assumed. + + A 3rd argument of `null` is treated as if it were a WASM pointer + of 0. + + If the 3rd argument is neither a WASM pointer nor one of the + above-described types, capi.SQLITE_MISUSE is returned. + + The first argument may be either an `sqlite3_stmt*` WASM + pointer or an sqlite3.oo1.Stmt instance. + + For consistency with the C API, it requires the same number of + arguments. It returns capi.SQLITE_MISUSE if passed any other + argument count. + */ + sqlite3_bind_blob: undefined/*installed later*/, + + /** + sqlite3_bind_text() works exactly like its C counterpart unless + its 3rd argument is one of: + + - JS string: the 3rd argument is converted to a C string, the + 4th argument is ignored, and the C-string's length is used + in its place. + + - Array: converted to a string as defined for "flexible + strings". The 4th argument is ignored and a value of -1 + is assumed. + + - Int8Array or Uint8Array: is assumed to contain UTF-8 text, is + converted to a string. The 4th argument is ignored, replaced + by the array's byteLength value. + + - If it's an ArrayBuffer, it gets wrapped in a Uint8Array and + treated as that type. + + In each of those cases, the final argument (text destructor) is + ignored and capi.SQLITE_WASM_DEALLOC is assumed. + + A 3rd argument of `null` is treated as if it were a WASM pointer + of 0. + + If the 3rd argument is neither a WASM pointer nor one of the + above-described types, capi.SQLITE_MISUSE is returned. + + The first argument may be either an `sqlite3_stmt*` WASM + pointer or an sqlite3.oo1.Stmt instance. + + For consistency with the C API, it requires the same number of + arguments. It returns capi.SQLITE_MISUSE if passed any other + argument count. + + If client code needs to bind partial strings, it needs to + either parcel the string up before passing it in here or it + must pass in a WASM pointer for the 3rd argument and a valid + 4th-argument value, taking care not to pass a value which + truncates a multi-byte UTF-8 character. When passing + WASM-format strings, it is important that the final argument be + valid or unexpected content can result can result, or even a + crash if the application reads past the WASM heap bounds. + */ + sqlite3_bind_text: undefined/*installed later*/, + /** sqlite3_create_function_v2() differs from its native counterpart only in the following ways: 1) The fourth argument (`eTextRep`) argument must not specify any encoding other than sqlite3.SQLITE_UTF8. The JS API does not currently support any other encoding and likely never will. This function does not replace that argument on its own - because it may contain other flags. + because it may contain other flags. As a special case, if + the bottom 4 bits of that argument are 0, SQLITE_UTF8 is + assumed. 2) Any of the four final arguments may be either WASM pointers (assumed to be function pointers) or JS Functions. In the latter case, each gets bound to WASM using sqlite3.capi.wasm.installFunction() and that wrapper is passed on to the native implementation. + + For consistency with the C API, it requires the same number of + arguments. It returns capi.SQLITE_MISUSE if passed any other + argument count. The semantics of JS functions are: xFunc: is passed `(pCtx, ...values)`. Its return value becomes the new SQL function's result. @@ -531,33 +614,33 @@ Maintenance reminder: the ability to add new WASM-accessible functions to the runtime requires that the WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` flag. */ - sqlite3_create_function_v2: function( + sqlite3_create_function_v2: ( pDb, funcName, nArg, eTextRep, pApp, xFunc, xStep, xFinal, xDestroy - ){/*installed later*/}, + )=>{/*installed later*/}, /** Equivalent to passing the same arguments to sqlite3_create_function_v2(), with 0 as the final argument. */ - sqlite3_create_function:function( + sqlite3_create_function: ( pDb, funcName, nArg, eTextRep, pApp, xFunc, xStep, xFinal - ){/*installed later*/}, + )=>{/*installed later*/}, /** The sqlite3_create_window_function() JS wrapper differs from its native implementation in the exact same way that sqlite3_create_function_v2() does. The additional function, xInverse(), is treated identically to xStep() by the wrapping layer. */ - sqlite3_create_window_function: function( + sqlite3_create_window_function: ( pDb, funcName, nArg, eTextRep, pApp, xStep, xFinal, xValue, xInverse, xDestroy - ){/*installed later*/}, + )=>{/*installed later*/}, /** The sqlite3_prepare_v3() binding handles two different uses with differing JS/WASM semantics: 1) sqlite3_prepare_v3(pDb, sqlString, -1, prepFlags, ppStmt , null) @@ -598,11 +681,11 @@ (pDb, sqlAsPointer, sqlByteLen, prepFlags, ppStmt, pzTail) It returns its result and compiled statement as documented in the C API. Fetching the output pointers (5th and 6th - parameters) requires using `capi.wasm.getMemValue()` (or + parameters) requires using `capi.wasm.peek()` (or equivalent) and the `pzTail` will point to an address relative to the `sqlAsPointer` value. If passed an invalid 2nd argument type, this function will return SQLITE_MISUSE and sqlite3_errmsg() will contain a string @@ -635,11 +718,11 @@ `callback(pVoid, colCount, listOfValues, listOfColNames)` If the callback is not a JS function then this binding performs no translation of the callback, but the sql argument is still converted to a WASM string for the call using the - "flexible-string" argument converter. + "string:flexible" argument converter. */ sqlite3_exec: (pDb, sql, callback, pVoid, pErrMsg)=>{}/*installed later*/, /** If passed a single argument which appears to be a byte-oriented @@ -668,15 +751,18 @@ affirmBindableTypedArray, flexibleString, bigIntFits32, bigIntFits64, bigIntFitsDouble, isBindableTypedArray, isInt32, isSQLableTypedArray, isTypedArray, typedArrayToString, - isUIThread: ()=>'undefined'===typeof WorkerGlobalScope, + isUIThread: ()=>(self.window===self && !!self.document), + // is this true for ESM?: 'undefined'===typeof WorkerGlobalScope isSharedTypedArray, + toss: function(...args){throw new Error(args.join(' '))}, + toss3, typedArrayPart }; - + Object.assign(wasm, { /** Emscripten APIs have a deep-seated assumption that all pointers are 32 bits. We'll remain optimistic that that won't always be the case and will use this constant in places where we might @@ -684,13 +770,11 @@ */ ptrSizeof: config.wasmPtrSizeof || 4, /** The WASM IR (Intermediate Representation) value for pointer-type values. It MUST refer to a value type of the - size described by this.ptrSizeof _or_ it may be any value - which ends in '*', which Emscripten's glue code internally - translates to i32. + size described by this.ptrSizeof. */ ptrIR: config.wasmPtrIR || "i32", /** True if BigInt support was enabled via (e.g.) the Emscripten -sWASM_BIGINT flag, else false. When @@ -715,16 +799,16 @@ || toss3("API config object requires a WebAssembly.Memory object", "in either config.exports.memory (exported)", "or config.memory (imported)."), /** - The API's one single point of access to the WASM-side memory - allocator. Works like malloc(3) (and is likely bound to - malloc()) but throws an WasmAllocError if allocation fails. It is - important that any code which might pass through the sqlite3 C - API NOT throw and must instead return SQLITE_NOMEM (or - equivalent, depending on the context). + The API's primary point of access to the WASM-side memory + allocator. Works like sqlite3_malloc() but throws a + WasmAllocError if allocation fails. It is important that any + code which might pass through the sqlite3 C API NOT throw and + must instead return SQLITE_NOMEM (or equivalent, depending on + the context). Very few cases in the sqlite3 JS APIs can result in client-defined functions propagating exceptions via the C-style API. Most notably, this applies to WASM-bound JS functions which are created directly by clients and passed on _as WASM @@ -732,11 +816,11 @@ sqlite3_create_function_v2(). Such bindings created transparently by this API will automatically use wrappers which catch exceptions and convert them to appropriate error codes. For cases where non-throwing allocation is required, use - sqlite3.wasm.alloc.impl(), which is direct binding of the + this.alloc.impl(), which is direct binding of the underlying C-level allocator. Design note: this function is not named "malloc" primarily because Emscripten uses that name and we wanted to avoid any confusion early on in this code's development, when it still @@ -743,13 +827,31 @@ had close ties to Emscripten's glue code. */ alloc: undefined/*installed later*/, /** - The API's one single point of access to the WASM-side memory - deallocator. Works like free(3) (and is likely bound to - free()). + Rarely necessary in JS code, this routine works like + sqlite3_realloc(M,N), where M is either NULL or a pointer + obtained from this function or this.alloc() and N is the number + of bytes to reallocate the block to. Returns a pointer to the + reallocated block or 0 if allocation fails. + + If M is NULL and N is positive, this behaves like + this.alloc(N). If N is 0, it behaves like this.dealloc(). + Results are undefined if N is negative (sqlite3_realloc() + treats that as 0, but if this code is built with a different + allocator it may misbehave with negative values). + + Like this.alloc.impl(), this.realloc.impl() is a direct binding + to the underlying realloc() implementation which does not throw + exceptions, instead returning 0 on allocation error. + */ + realloc: undefined/*installed later*/, + + /** + The API's primary point of access to the WASM-side memory + deallocator. Works like sqlite3_free(). Design note: this function is not named "free" for the same reason that this.alloc() is not called this.malloc(). */ dealloc: undefined/*installed later*/ @@ -761,49 +863,58 @@ wasm.alloc()'s srcTypedArray.byteLength bytes, populates them with the values from the source TypedArray, and returns the pointer to that memory. The returned pointer must eventually be passed to wasm.dealloc() to clean it up. + + The argument may be a Uint8Array, Int8Array, or ArrayBuffer, + and it throws if passed any other type. As a special case, to avoid further special cases where this is used, if srcTypedArray.byteLength is 0, it allocates a single byte and sets it to the value 0. Even in such cases, calls must behave as if the allocated memory has exactly srcTypedArray.byteLength bytes. - - ACHTUNG: this currently only works for Uint8Array and - Int8Array types and will throw if srcTypedArray is of - any other type. */ wasm.allocFromTypedArray = function(srcTypedArray){ + if(srcTypedArray instanceof ArrayBuffer){ + srcTypedArray = new Uint8Array(srcTypedArray); + } affirmBindableTypedArray(srcTypedArray); const pRet = wasm.alloc(srcTypedArray.byteLength || 1); wasm.heapForSize(srcTypedArray.constructor).set( srcTypedArray.byteLength ? srcTypedArray : [0], pRet ); return pRet; }; - const keyAlloc = config.allocExportName || 'malloc', - keyDealloc = config.deallocExportName || 'free'; - for(const key of [keyAlloc, keyDealloc]){ - const f = wasm.exports[key]; - if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); - } - - wasm.alloc = function f(n){ - const m = f.impl(n); - if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); - return m; - }; - wasm.alloc.impl = wasm.exports[keyAlloc]; - wasm.dealloc = wasm.exports[keyDealloc]; + { + // Set up allocators... + const keyAlloc = config.allocExportName, + keyDealloc = config.deallocExportName, + keyRealloc = config.reallocExportName; + for(const key of [keyAlloc, keyDealloc, keyRealloc]){ + const f = wasm.exports[key]; + if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); + } + + wasm.alloc = function f(n){ + return f.impl(n) || WasmAllocError.toss("Failed to allocate",n," bytes."); + }; + wasm.alloc.impl = wasm.exports[keyAlloc]; + wasm.realloc = function f(m,n){ + const m2 = f.impl(m,n); + return n ? (m2 || WasmAllocError.toss("Failed to reallocate",n," bytes.")) : 0; + }; + wasm.realloc.impl = wasm.exports[keyRealloc]; + wasm.dealloc = wasm.exports[keyDealloc]; + } /** Reports info about compile-time options using - sqlite_compileoption_get() and sqlite3_compileoption_used(). It + sqlite3_compileoption_get() and sqlite3_compileoption_used(). It has several distinct uses: If optName is an array then it is expected to be a list of compilation options and this function returns an object which maps each such option to true or false, indicating @@ -836,11 +947,11 @@ f._rxInt = /^-?\d+$/; f._opt = function(opt, rv){ const m = f._rx.exec(opt); rv[0] = (m ? m[1] : opt); rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; - }; + }; } const rc = {}, ov = [0,0]; let i = 0, k; while((k = capi.sqlite3_compileoption_get(i++))){ f._opt(k,ov); @@ -861,169 +972,10 @@ } return ( 'string'===typeof optName ) ? !!capi.sqlite3_compileoption_used(optName) : false; }/*compileOptionUsed()*/; - - /** - Signatures for the WASM-exported C-side functions. Each entry - is an array with 2+ elements: - - [ "c-side name", - "result type" (wasm.xWrap() syntax), - [arg types in xWrap() syntax] - // ^^^ this needn't strictly be an array: it can be subsequent - // elements instead: [x,y,z] is equivalent to x,y,z - ] - - Note that support for the API-specific data types in the - result/argument type strings gets plugged in at a later phase in - the API initialization process. - */ - wasm.bindingSignatures = [ - // Please keep these sorted by function name! - ["sqlite3_aggregate_context","void*", "sqlite3_context*", "int"], - ["sqlite3_bind_blob","int", "sqlite3_stmt*", "int", "*", "int", "*" - /* TODO: we should arguably write a custom wrapper which knows - how to handle Blob, TypedArrays, and JS strings. */ - ], - ["sqlite3_bind_double","int", "sqlite3_stmt*", "int", "f64"], - ["sqlite3_bind_int","int", "sqlite3_stmt*", "int", "int"], - ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], - ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], - ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], - ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int" - /* We should arguably create a hand-written binding of - bind_text() which does more flexible text conversion, along - the lines of sqlite3_prepare_v3(). The slightly problematic - part is the final argument (text destructor). */ - ], - ["sqlite3_close_v2", "int", "sqlite3*"], - ["sqlite3_changes", "int", "sqlite3*"], - ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], - ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], - ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], - ["sqlite3_column_count", "int", "sqlite3_stmt*"], - ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], - ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], - ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], - ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], - ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], - ["sqlite3_compileoption_get", "string", "int"], - ["sqlite3_compileoption_used", "int", "string"], - /* sqlite3_create_function(), sqlite3_create_function_v2(), and - sqlite3_create_window_function() use hand-written bindings to - simplify handling of their function-type arguments. */ - ["sqlite3_data_count", "int", "sqlite3_stmt*"], - ["sqlite3_db_filename", "string", "sqlite3*", "string"], - ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], - ["sqlite3_db_name", "string", "sqlite3*", "int"], - ["sqlite3_deserialize", "int", "sqlite3*", "string", "*", "i64", "i64", "int"] - /* Careful! Short version: de/serialize() are problematic because they - might use a different allocator than the user for managing the - deserialized block. de/serialize() are ONLY safe to use with - sqlite3_malloc(), sqlite3_free(), and its 64-bit variants. */, - ["sqlite3_errmsg", "string", "sqlite3*"], - ["sqlite3_error_offset", "int", "sqlite3*"], - ["sqlite3_errstr", "string", "int"], - /*["sqlite3_exec", "int", "sqlite3*", "string", "*", "*", "**" - Handled seperately to perform translation of the callback - into a WASM-usable one. ],*/ - ["sqlite3_expanded_sql", "string", "sqlite3_stmt*"], - ["sqlite3_extended_errcode", "int", "sqlite3*"], - ["sqlite3_extended_result_codes", "int", "sqlite3*", "int"], - ["sqlite3_file_control", "int", "sqlite3*", "string", "int", "*"], - ["sqlite3_finalize", "int", "sqlite3_stmt*"], - ["sqlite3_free", undefined,"*"], - ["sqlite3_initialize", undefined], - /*["sqlite3_interrupt", undefined, "sqlite3*" - ^^^ we cannot actually currently support this because JS is - single-threaded and we don't have a portable way to access a DB - from 2 SharedWorkers concurrently. ],*/ - ["sqlite3_libversion", "string"], - ["sqlite3_libversion_number", "int"], - ["sqlite3_malloc", "*","int"], - ["sqlite3_open", "int", "string", "*"], - ["sqlite3_open_v2", "int", "string", "*", "int", "string"], - /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled - separately due to us requiring two different sets of semantics - for those, depending on how their SQL argument is provided. */ - /* sqlite3_randomness() uses a hand-written wrapper to extend - the range of supported argument types. */ - ["sqlite3_realloc", "*","*","int"], - ["sqlite3_reset", "int", "sqlite3_stmt*"], - ["sqlite3_result_blob",undefined, "*", "*", "int", "*"], - ["sqlite3_result_double",undefined, "*", "f64"], - ["sqlite3_result_error",undefined, "*", "string", "int"], - ["sqlite3_result_error_code", undefined, "*", "int"], - ["sqlite3_result_error_nomem", undefined, "*"], - ["sqlite3_result_error_toobig", undefined, "*"], - ["sqlite3_result_int",undefined, "*", "int"], - ["sqlite3_result_null",undefined, "*"], - ["sqlite3_result_text",undefined, "*", "string", "int", "*"], - ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], - ["sqlite3_shutdown", undefined], - ["sqlite3_sourceid", "string"], - ["sqlite3_sql", "string", "sqlite3_stmt*"], - ["sqlite3_step", "int", "sqlite3_stmt*"], - ["sqlite3_strglob", "int", "string","string"], - ["sqlite3_strlike", "int", "string","string","int"], - ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], - ["sqlite3_total_changes", "int", "sqlite3*"], - ["sqlite3_uri_boolean", "int", "string", "string", "int"], - ["sqlite3_uri_key", "string", "string", "int"], - ["sqlite3_uri_parameter", "string", "string", "string"], - ["sqlite3_user_data","void*", "sqlite3_context*"], - ["sqlite3_value_blob", "*", "sqlite3_value*"], - ["sqlite3_value_bytes","int", "sqlite3_value*"], - ["sqlite3_value_double","f64", "sqlite3_value*"], - ["sqlite3_value_int","int", "sqlite3_value*"], - ["sqlite3_value_text", "string", "sqlite3_value*"], - ["sqlite3_value_type", "int", "sqlite3_value*"], - ["sqlite3_vfs_find", "*", "string"], - ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], - ["sqlite3_vfs_unregister", "int", "sqlite3_vfs*"] - ]/*wasm.bindingSignatures*/; - - if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ - /* ^^^ "the problem" is that this is an option feature and the - build-time function-export list does not currently take - optional features into account. */ - wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); - } - - /** - Functions which require BigInt (int64) support are separated from - the others because we need to conditionally bind them or apply - dummy impls, depending on the capabilities of the environment. - */ - wasm.bindingSignatures.int64 = [ - ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], - ["sqlite3_changes64","i64", ["sqlite3*"]], - ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], - ["sqlite3_malloc64", "*","i64"], - ["sqlite3_msize", "i64", "*"], - ["sqlite3_realloc64", "*","*", "i64"], - ["sqlite3_result_int64",undefined, "*", "i64"], - ["sqlite3_total_changes64", "i64", ["sqlite3*"]], - ["sqlite3_uri_int64", "i64", ["string", "string", "i64"]], - ["sqlite3_value_int64","i64", "sqlite3_value*"], - ]; - - /** - Functions which are intended solely for API-internal use by the - WASM components, not client code. These get installed into - sqlite3.wasm. - */ - wasm.bindingSignatures.wasm = [ - ["sqlite3_wasm_db_reset", "int", "sqlite3*"], - ["sqlite3_wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"], - ["sqlite3_wasm_vfs_create_file", "int", - "sqlite3_vfs*","string","*", "int"], - ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] - ]; - /** sqlite3.wasm.pstack (pseudo-stack) holds a special-case stack-style allocator intended only for use with _small_ data of not more than (in total) a few kb in size, managed as if it were @@ -1050,11 +1002,11 @@ It operates from a static range of memory which lives outside of space managed by Emscripten's stack-management, so does not collide with Emscripten-provided stack allocation APIs. The memory lives in the WASM heap and can be used with routines such - as wasm.setMemValue() and any wasm.heap8u().slice(). + as wasm.poke() and any wasm.heap8u().slice(). */ wasm.pstack = Object.assign(Object.create(null),{ /** Sets the current pstack position to the given pointer. Results are undefined if the passed-in value did not come from @@ -1065,35 +1017,48 @@ Attempts to allocate the given number of bytes from the pstack. On success, it zeroes out a block of memory of the given size, adjusts the pstack pointer, and returns a pointer to the memory. On error, returns throws a WasmAllocError. The memory must eventually be released using restore(). + + If n is a string, it must be a WASM "IR" value in the set + accepted by wasm.sizeofIR(), which is mapped to the size of + that data type. If passed a string not in that set, it throws a + WasmAllocError. This method always adjusts the given value to be a multiple of 8 bytes because failing to do so can lead to incorrect results when reading and writing 64-bit values from/to the WASM heap. Similarly, the returned address is always 8-byte aligned. */ - alloc: (n)=>{ + alloc: function(n){ + if('string'===typeof n && !(n = wasm.sizeofIR(n))){ + WasmAllocError.toss("Invalid value for pstack.alloc(",arguments[0],")"); + } return wasm.exports.sqlite3_wasm_pstack_alloc(n) || WasmAllocError.toss("Could not allocate",n, "bytes from the pstack."); }, /** alloc()'s n chunks, each sz bytes, as a single memory block and returns the addresses as an array of n element, each holding the address of one chunk. + sz may optionally be an IR string accepted by wasm.sizeofIR(). + Throws a WasmAllocError if allocation fails. Example: ``` const [p1, p2, p3] = wasm.pstack.allocChunks(3,4); ``` */ - allocChunks: (n,sz)=>{ + allocChunks: function(n,sz){ + if('string'===typeof sz && !(sz = wasm.sizeofIR(sz))){ + WasmAllocError.toss("Invalid size value for allocChunks(",arguments[1],")"); + } const mem = wasm.pstack.alloc(n * sz); const rc = []; let i = 0, offset = 0; for(; i < n; offset = (sz * ++i)){ rc.push(mem + offset); @@ -1109,11 +1074,11 @@ argument: if it's 1, it returns a single pointer value. If it's more than 1, it returns the same as allocChunks(). When a returned pointers will refer to a 64-bit value, e.g. a double or int64, and that value must be written or fetched, - e.g. using wasm.setMemValue() or wasm.getMemValue(), it is + e.g. using wasm.poke() or wasm.peek(), it is important that the pointer in question be aligned to an 8-byte boundary or else it will not be fetched or written properly and will corrupt or read neighboring memory. However, when all pointers involved point to "small" data, it @@ -1193,10 +1158,17 @@ }; /** State for sqlite3_wasmfs_opfs_dir(). */ let __wasmfsOpfsDir = undefined; /** + 2022-12-17: incompatible WASMFS changes have made WASMFS+OPFS + unavailable from the main thread, which eliminates the most + significant benefit of supporting WASMFS. This function is now a + no-op which always returns a falsy value. Before that change, + this function behaved as documented below (and how it will again + if we can find a compelling reason to support it). + If the wasm environment has a WASMFS/OPFS-backed persistent storage directory, its path is returned by this function. If it does not then it returns "" (noting that "" is a falsy value). The first time this is called, this function inspects the current @@ -1209,10 +1181,12 @@ */ capi.sqlite3_wasmfs_opfs_dir = function(){ if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; // If we have no OPFS, there is no persistent dir const pdir = config.wasmfsOpfsDir; + console.error("sqlite3_wasmfs_opfs_dir() can no longer work due "+ + "to incompatible WASMFS changes. It will be removed."); if(!pdir || !self.FileSystemHandle || !self.FileSystemDirectoryHandle || !self.FileSystemFileHandle){ return __wasmfsOpfsDir = ""; @@ -1296,53 +1270,65 @@ capi.sqlite3_js_vfs_list = function(){ const rc = []; let pVfs = capi.sqlite3_vfs_find(0); while(pVfs){ const oVfs = new capi.sqlite3_vfs(pVfs); - rc.push(wasm.cstringToJs(oVfs.$zName)); + rc.push(wasm.cstrToJs(oVfs.$zName)); pVfs = oVfs.$pNext; oVfs.dispose(); } return rc; }; /** - Serializes the given `sqlite3*` pointer to a Uint8Array, as per - sqlite3_serialize(). On success it returns a Uint8Array. On - error it throws with a description of the problem. + A convenience wrapper around sqlite3_serialize() which serializes + the given `sqlite3*` pointer to a Uint8Array. The first argument + may be either an `sqlite3*` or an sqlite3.oo1.DB instance. + + On success it returns a Uint8Array. If the schema is empty, an + empty array is returned. + + `schema` is the schema to serialize. It may be a WASM C-string + pointer or a JS string. If it is falsy, it defaults to `"main"`. + + On error it throws with a description of the problem. */ - capi.sqlite3_js_db_export = function(pDb){ + capi.sqlite3_js_db_export = function(pDb, schema=0){ + pDb = wasm.xWrap.testConvertArg('sqlite3*', pDb); if(!pDb) toss3('Invalid sqlite3* argument.'); if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.'); - const stack = wasm.pstack.pointer; + const scope = wasm.scopedAllocPush(); let pOut; try{ - const pSize = wasm.pstack.alloc(8/*i64*/ + wasm.ptrSizeof); + const pSize = wasm.scopedAlloc(8/*i64*/ + wasm.ptrSizeof); const ppOut = pSize + 8; /** Maintenance reminder, since this cost a full hour of grief and confusion: if the order of pSize/ppOut are reversed in that memory block, fetching the value of pSize after the export reads a garbage size because it's not on an 8-byte memory boundary! */ + const zSchema = schema + ? (wasm.isPtr(schema) ? schema : wasm.scopedAllocCString(''+schema)) + : 0; let rc = wasm.exports.sqlite3_wasm_db_serialize( - pDb, ppOut, pSize, 0 + pDb, zSchema, ppOut, pSize, 0 ); if(rc){ toss3("Database serialization failed with code", sqlite3.capi.sqlite3_js_rc_str(rc)); } - pOut = wasm.getPtrValue(ppOut); - const nOut = wasm.getMemValue(pSize, 'i64'); + pOut = wasm.peekPtr(ppOut); + const nOut = wasm.peek(pSize, 'i64'); rc = nOut ? wasm.heap8u().slice(pOut, pOut + Number(nOut)) : new Uint8Array(); return rc; }finally{ if(pOut) wasm.exports.sqlite3_free(pOut); - wasm.pstack.restore(stack); + wasm.scopedAllocPop(scope); } }; /** Given a `sqlite3*` and a database name (JS string or WASM @@ -1364,11 +1350,91 @@ return capi.sqlite3_aggregate_context(pCtx, n) || (n ? WasmAllocError.toss("Cannot allocate",n, "bytes for sqlite3_aggregate_context()") : 0); }; - + + /** + Creates a file using the storage appropriate for the given + sqlite3_vfs. The first argument may be a VFS name (JS string + only, NOT a WASM C-string), WASM-managed `sqlite3_vfs*`, or + a capi.sqlite3_vfs instance. Pass 0 (a NULL pointer) to use the + default VFS. If passed a string which does not resolve using + sqlite3_vfs_find(), an exception is thrown. (Note that a WASM + C-string is not accepted because it is impossible to + distinguish from a C-level `sqlite3_vfs*`.) + + The second argument, the filename, must be a JS or WASM C-string. + + The 3rd may either be falsy, a valid WASM memory pointer, an + ArrayBuffer, or a Uint8Array. The 4th must be the length, in + bytes, of the data array to copy. If the 3rd argument is an + ArrayBuffer or Uint8Array and the 4th is not a positive integer + then the 4th defaults to the array's byteLength value. + + If data is falsy then a file is created with dataLen bytes filled + with uninitialized data (whatever truncate() leaves there). If + data is not falsy then a file is created or truncated and it is + filled with the first dataLen bytes of the data source. + + Throws if any arguments are invalid or if creating or writing to + the file fails. + + Note that most VFSes do _not_ automatically create directory + parts of filenames, nor do all VFSes have a concept of + directories. If the given filename is not valid for the given + VFS, an exception will be thrown. This function exists primarily + to assist in implementing file-upload capability, with the caveat + that clients must have some idea of the VFS into which they want + to upload and that VFS must support the operation. + + VFS-specific notes: + + - "memdb": results are undefined. + + - "kvvfs": will fail with an I/O error due to strict internal + requirments of that VFS's xTruncate(). + + - "unix" and related: will use the WASM build's equivalent of the + POSIX I/O APIs. This will work so long as neither a specific + VFS nor the WASM environment imposes requirements which break it. + + - "opfs": uses OPFS storage and creates directory parts of the + filename. + */ + capi.sqlite3_js_vfs_create_file = function(vfs, filename, data, dataLen){ + let pData; + if(data){ + if(wasm.isPtr(data)){ + pData = data; + }else if(data instanceof ArrayBuffer){ + data = new Uint8Array(data); + } + if(data instanceof Uint8Array){ + pData = wasm.allocFromTypedArray(data); + if(arguments.length<4 || !util.isInt32(dataLen) || dataLen<0){ + dataLen = data.byteLength; + } + }else{ + SQLite3Error.toss("Invalid 3rd argument type for sqlite3_js_vfs_create_file()."); + } + }else{ + pData = 0; + } + if(!util.isInt32(dataLen) || dataLen<0){ + wasm.dealloc(pData); + SQLite3Error.toss("Invalid 4th argument for sqlite3_js_vfs_create_file()."); + } + try{ + const rc = wasm.sqlite3_wasm_vfs_create_file(vfs, filename, pData, dataLen); + if(rc) SQLite3Error.toss("Creation of file failed with sqlite3 result code", + capi.sqlite3_js_rc_str(rc)); + }finally{ + wasm.dealloc(pData); + } + }; + if( util.isUIThread() ){ /* Features specific to the main window thread... */ /** Internal helper for sqlite3_js_kvvfs_clear() and friends. @@ -1445,10 +1511,271 @@ return sz * 2 /* because JS uses 2-byte char encoding */; }; }/* main-window-only bits */ + /** + Wraps all known variants of the C-side variadic + sqlite3_db_config(). + + Full docs: https://sqlite.org/c3ref/db_config.html + + Returns capi.SQLITE_MISUSE if op is not a valid operation ID. + */ + capi.sqlite3_db_config = function f(pDb, op, ...args){ + if(!this.s){ + this.s = wasm.xWrap('sqlite3_wasm_db_config_s','int', + ['sqlite3*', 'int', 'string:static'] + /* MAINDBNAME requires a static string */); + this.pii = wasm.xWrap('sqlite3_wasm_db_config_pii', 'int', + ['sqlite3*', 'int', '*','int', 'int']); + this.ip = wasm.xWrap('sqlite3_wasm_db_config_ip','int', + ['sqlite3*', 'int', 'int','*']); + } + const c = capi; + switch(op){ + case c.SQLITE_DBCONFIG_ENABLE_FKEY: + case c.SQLITE_DBCONFIG_ENABLE_TRIGGER: + case c.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: + case c.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: + case c.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: + case c.SQLITE_DBCONFIG_ENABLE_QPSG: + case c.SQLITE_DBCONFIG_TRIGGER_EQP: + case c.SQLITE_DBCONFIG_RESET_DATABASE: + case c.SQLITE_DBCONFIG_DEFENSIVE: + case c.SQLITE_DBCONFIG_WRITABLE_SCHEMA: + case c.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: + case c.SQLITE_DBCONFIG_DQS_DML: + case c.SQLITE_DBCONFIG_DQS_DDL: + case c.SQLITE_DBCONFIG_ENABLE_VIEW: + case c.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: + case c.SQLITE_DBCONFIG_TRUSTED_SCHEMA: + return this.ip(pDb, op, args[0], args[1] || 0); + case c.SQLITE_DBCONFIG_LOOKASIDE: + return this.pii(pDb, op, args[0], args[1], args[2]); + case c.SQLITE_DBCONFIG_MAINDBNAME: + return this.s(pDb, op, args[0]); + default: + return c.SQLITE_MISUSE; + } + }.bind(Object.create(null)); + + /** + Given a (sqlite3_value*), this function attempts to convert it + to an equivalent JS value with as much fidelity as feasible and + return it. + + By default it throws if it cannot determine any sensible + conversion. If passed a falsy second argument, it instead returns + `undefined` if no suitable conversion is found. Note that there + is no conversion from SQL to JS which results in the `undefined` + value, so `undefined` has an unambiguous meaning here. It will + always throw a WasmAllocError if allocating memory for a + conversion fails. + + Caveats: + + - It does not support sqlite3_value_to_pointer() conversions + because those require a type name string which this function + does not have and cannot sensibly be given at the level of the + API where this is used (e.g. automatically converting UDF + arguments). Clients using sqlite3_value_to_pointer(), and its + related APIs, will need to manage those themselves. + */ + capi.sqlite3_value_to_js = function(pVal,throwIfCannotConvert=true){ + let arg; + const valType = capi.sqlite3_value_type(pVal); + switch(valType){ + case capi.SQLITE_INTEGER: + if(wasm.bigIntEnabled){ + arg = capi.sqlite3_value_int64(pVal); + if(util.bigIntFitsDouble(arg)) arg = Number(arg); + } + else arg = capi.sqlite3_value_double(pVal)/*yes, double, for larger integers*/; + break; + case capi.SQLITE_FLOAT: + arg = capi.sqlite3_value_double(pVal); + break; + case capi.SQLITE_TEXT: + arg = capi.sqlite3_value_text(pVal); + break; + case capi.SQLITE_BLOB:{ + const n = capi.sqlite3_value_bytes(pVal); + const pBlob = capi.sqlite3_value_blob(pVal); + if(n && !pBlob) sqlite3.WasmAllocError.toss( + "Cannot allocate memory for blob argument of",n,"byte(s)" + ); + arg = n ? wasm.heap8u().slice(pBlob, pBlob + Number(n)) : null; + break; + } + case capi.SQLITE_NULL: + arg = null; break; + default: + if(throwIfCannotConvert){ + toss3(capi.SQLITE_MISMATCH, + "Unhandled sqlite3_value_type():",valType); + } + arg = undefined; + } + return arg; + }; + + /** + Requires a C-style array of `sqlite3_value*` objects and the + number of entries in that array. Returns a JS array containing + the results of passing each C array entry to + sqlite3_value_to_js(). The 3rd argument to this function is + passed on as the 2nd argument to that one. + */ + capi.sqlite3_values_to_js = function(argc,pArgv,throwIfCannotConvert=true){ + let i; + const tgt = []; + for(i = 0; i < argc; ++i){ + /** + Curiously: despite ostensibly requiring 8-byte + alignment, the pArgv array is parcelled into chunks of + 4 bytes (1 pointer each). The values those point to + have 8-byte alignment but the individual argv entries + do not. + */ + tgt.push(capi.sqlite3_value_to_js( + wasm.peekPtr(pArgv + (wasm.ptrSizeof * i)) + )); + } + return tgt; + }; + + /** + Calls either sqlite3_result_error_nomem(), if e is-a + WasmAllocError, or sqlite3_result_error(). In the latter case, + the second arugment is coerced to a string to create the error + message. + + The first argument is a (sqlite3_context*). Returns void. + Does not throw. + */ + capi.sqlite3_result_error_js = function(pCtx,e){ + if(e instanceof WasmAllocError){ + capi.sqlite3_result_error_nomem(pCtx); + }else{ + /* Maintenance reminder: ''+e, rather than e.message, + will prefix e.message with e.name, so it includes + the exception's type name in the result. */; + capi.sqlite3_result_error(pCtx, ''+e, -1); + } + }; + + /** + This function passes its 2nd argument to one of the + sqlite3_result_xyz() routines, depending on the type of that + argument: + + - If (val instanceof Error), this function passes it to + sqlite3_result_error_js(). + - `null`: `sqlite3_result_null()` + - `boolean`: `sqlite3_result_int()` with a value of 0 or 1. + - `number`: `sqlite3_result_int()`, `sqlite3_result_int64()`, or + `sqlite3_result_double()`, depending on the range of the number + and whether or not int64 support is enabled. + - `bigint`: similar to `number` but will trigger an error if the + value is too big to store in an int64. + - `string`: `sqlite3_result_text()` + - Uint8Array or Int8Array or ArrayBuffer: `sqlite3_result_blob()` + - `undefined`: is a no-op provided to simplify certain use cases. + + Anything else triggers `sqlite3_result_error()` with a + description of the problem. + + The first argument to this function is a `(sqlite3_context*)`. + Returns void. Does not throw. + */ + capi.sqlite3_result_js = function(pCtx,val){ + if(val instanceof Error){ + capi.sqlite3_result_error_js(pCtx, val); + return; + } + try{ + switch(typeof val) { + case 'undefined': + /* This is a no-op. This routine originated in the create_function() + family of APIs and in that context, passing in undefined indicated + that the caller was responsible for calling sqlite3_result_xxx() + (if needed). */ + break; + case 'boolean': + capi.sqlite3_result_int(pCtx, val ? 1 : 0); + break; + case 'bigint': + if(util.bigIntFits32(val)){ + capi.sqlite3_result_int(pCtx, Number(val)); + }else if(util.bigIntFitsDouble(val)){ + capi.sqlite3_result_double(pCtx, Number(val)); + }else if(wasm.bigIntEnabled){ + if(util.bigIntFits64(val)) capi.sqlite3_result_int64(pCtx, val); + else toss3("BigInt value",val.toString(),"is too BigInt for int64."); + }else{ + toss3("BigInt value",val.toString(),"is too BigInt."); + } + break; + case 'number': { + let f; + if(util.isInt32(val)){ + f = capi.sqlite3_result_int; + }else if(wasm.bigIntEnabled + && Number.isInteger(val) + && util.bigIntFits64(BigInt(val))){ + f = capi.sqlite3_result_int64; + }else{ + f = capi.sqlite3_result_double; + } + f(pCtx, val); + break; + } + case 'string': { + const [p, n] = wasm.allocCString(val,true); + capi.sqlite3_result_text(pCtx, p, n, capi.SQLITE_WASM_DEALLOC); + break; + } + case 'object': + if(null===val/*yes, typeof null === 'object'*/) { + capi.sqlite3_result_null(pCtx); + break; + }else if(util.isBindableTypedArray(val)){ + const pBlob = wasm.allocFromTypedArray(val); + capi.sqlite3_result_blob( + pCtx, pBlob, val.byteLength, + capi.SQLITE_WASM_DEALLOC + ); + break; + } + // else fall through + default: + toss3("Don't not how to handle this UDF result value:",(typeof val), val); + } + }catch(e){ + capi.sqlite3_result_error_js(pCtx, e); + } + }; + + /** + Returns the result sqlite3_column_value(pStmt,iCol) passed to + sqlite3_value_to_js(). The 3rd argument of this function is + ignored by this function except to pass it on as the second + argument of sqlite3_value_to_js(). If the sqlite3_column_value() + returns NULL (e.g. because the column index is out of range), + this function returns `undefined`, regardless of the 3rd + argument. If the 3rd argument is falsy and conversion fails, + `undefined` will be returned. + + Note that sqlite3_column_value() returns an "unprotected" value + object, but in a single-threaded environment (like this one) + there is no distinction between protected and unprotected values. + */ + capi.sqlite3_column_js = function(pStmt, iCol, throwIfCannotConvert=true){ + const v = capi.sqlite3_column_value(pStmt, iCol); + return (0===v) ? undefined : capi.sqlite3_value_to_js(v, throwIfCannotConvert); + }; /* The remainder of the API will be set up in later steps. */ const sqlite3 = { WasmAllocError: WasmAllocError, SQLite3Error: SQLite3Error, @@ -1465,10 +1792,19 @@ This object is initially a placeholder which gets replaced by a build-generated object. */ version: Object.create(null), + + /** + The library reserves the 'client' property for client-side use + and promises to never define a property with this name nor to + ever rely on specific contents of it. It makes no such guarantees + for other properties. + */ + client: undefined, + /** Performs any optional asynchronous library-level initialization which might be required. This function returns a Promise which resolves to the sqlite3 namespace object. Any error in the async init will be fatal to the init as a whole, but init @@ -1494,24 +1830,43 @@ */ asyncPostInit: async function(){ let lip = sqlite3ApiBootstrap.initializersAsync; delete sqlite3ApiBootstrap.initializersAsync; if(!lip || !lip.length) return Promise.resolve(sqlite3); - // Is it okay to resolve these in parallel or do we need them - // to resolve in order? We currently only have 1, so it - // makes no difference. lip = lip.map((f)=>{ const p = (f instanceof Promise) ? f : f(sqlite3); return p.catch((e)=>{ console.error("an async sqlite3 initializer failed:",e); throw e; }); }); - //let p = lip.shift(); - //while(lip.length) p = p.then(lip.shift()); - //return p.then(()=>sqlite3); - return Promise.all(lip).then(()=>sqlite3); + const postInit = ()=>{ + if(!sqlite3.__isUnderTest){ + /* Delete references to internal-only APIs which are used by + some initializers. Retain them when running in test mode + so that we can add tests for them. */ + delete sqlite3.util; + /* It's conceivable that we might want to expose + StructBinder to client-side code, but it's only useful if + clients build their own sqlite3.wasm which contains their + one C struct types. */ + delete sqlite3.StructBinder; + } + return sqlite3; + }; + if(1){ + /* Run all initializers in sequence. The advantage is that it + allows us to have post-init cleanup defined outside of this + routine at the end of the list and have it run at a + well-defined time. */ + let p = lip.shift(); + while(lip.length) p = p.then(lip.shift()); + return p.then(postInit); + }else{ + /* Run them in an arbitrary order. */ + return Promise.all(lip).then(postInit); + } }, /** scriptInfo ideally gets injected into this object by the infrastructure which assembles the JS/WASM module. It contains state which must be collected before sqlite3ApiBootstrap() can Index: ext/wasm/api/sqlite3-api-worker1.js ================================================================== --- ext/wasm/api/sqlite3-api-worker1.js +++ ext/wasm/api/sqlite3-api-worker1.js @@ -156,19 +156,10 @@ version: sqlite3.version object bigIntEnabled: bool. True if BigInt support is enabled. - wasmfsOpfsDir: path prefix, if any, _intended_ for use with - WASMFS OPFS persistent storage. - - wasmfsOpfsEnabled: true if persistent storage is enabled in the - current environment. Only files stored under wasmfsOpfsDir - will persist using that mechanism, however. It is legal to use - the non-WASMFS OPFS VFS to open a database via a URI-style - db filename. - vfsList: result of sqlite3.capi.sqlite3_js_vfs_list() } } ``` @@ -447,11 +438,10 @@ const oargs = Object.create(null), args = (ev.args || Object.create(null)); if(args.simulateError){ // undocumented internal testing option toss("Throwing because of simulateError flag."); } const rc = Object.create(null); - const pDir = sqlite3.capi.sqlite3_wasmfs_opfs_dir(); let byteArray, pVfs; oargs.vfs = args.vfs; if(isSpecialDbFilename(args.filename)){ oargs.filename = args.filename || ""; }else{ @@ -473,19 +463,18 @@ }catch(e){ throw new sqlite3.SQLite3Error( e.name+' creating '+args.filename+": "+e.message, { cause: e } - ); + ); }finally{ if(pMem) sqlite3.wasm.dealloc(pMem); } } const db = wState.open(oargs); rc.filename = db.filename; - rc.persistent = (!!pDir && db.filename.startsWith(pDir+'/')) - || !!sqlite3.capi.sqlite3_js_db_uses_vfs(db.pointer, "opfs"); + rc.persistent = !!sqlite3.capi.sqlite3_js_db_uses_vfs(db.pointer, "opfs"); rc.dbId = getDbId(db); rc.vfs = db.dbVfsName(); return rc; }, @@ -556,15 +545,14 @@ }/*exec()*/, 'config-get': function(){ const rc = Object.create(null), src = sqlite3.config; [ - 'wasmfsOpfsDir', 'bigIntEnabled' + 'bigIntEnabled' ].forEach(function(k){ if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k]; }); - rc.wasmfsOpfsEnabled = !!sqlite3.capi.sqlite3_wasmfs_opfs_dir(); rc.version = sqlite3.version; rc.vfsList = sqlite3.capi.sqlite3_js_vfs_list(); rc.opfsEnabled = !!sqlite3.opfs; return rc; }, Index: ext/wasm/api/sqlite3-opfs-async-proxy.js ================================================================== --- ext/wasm/api/sqlite3-opfs-async-proxy.js +++ ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -11,11 +11,11 @@ *********************************************************************** A Worker which manages asynchronous OPFS handles on behalf of a synchronous API which controls it via a combination of Worker messages, SharedArrayBuffer, and Atomics. It is the asynchronous - counterpart of the API defined in sqlite3-api-opfs.js. + counterpart of the API defined in sqlite3-vfs-opfs.js. Highly indebted to: https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js @@ -27,804 +27,871 @@ initialization process. This file represents an implementation detail of a larger piece of code, and not a public interface. Its details may change at any time and are not intended to be used by any client-level code. -*/ -"use strict"; -const toss = function(...args){throw new Error(args.join(' '))}; -if(self.window === self){ - toss("This code cannot run from the main thread.", - "Load it as a Worker from a separate Worker."); -}else if(!navigator.storage.getDirectory){ - toss("This API requires navigator.storage.getDirectory."); -} - -/** - Will hold state copied to this object from the syncronous side of - this API. -*/ -const state = Object.create(null); - -/** - verbose: - - 0 = no logging output - 1 = only errors - 2 = warnings and errors - 3 = debug, warnings, and errors -*/ -state.verbose = 2; - -const loggers = { - 0:console.error.bind(console), - 1:console.warn.bind(console), - 2:console.log.bind(console) -}; -const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); -}; -const log = (...args)=>logImpl(2, ...args); -const warn = (...args)=>logImpl(1, ...args); -const error = (...args)=>logImpl(0, ...args); -const metrics = Object.create(null); -metrics.reset = ()=>{ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; -}; -metrics.dump = ()=>{ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - } - console.log(self.location.href, - "metrics for",self.location.href,":\n", - metrics, - "\nTotal of",n,"op(s) for",t,"ms", - "approx",w,"ms spent waiting on OPFS APIs."); - console.log("Serialization metrics:",metrics.s11n); -}; - -/** - __openFiles is a map of sqlite3_file pointers (integers) to - metadata related to a given OPFS file handles. The pointers are, in - this side of the interface, opaque file handle IDs provided by the - synchronous part of this constellation. Each value is an object - with a structure demonstrated in the xOpen() impl. -*/ -const __openFiles = Object.create(null); -/** - __autoLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. - - Maintenance reminder: if we relinquish auto-locks at the end of the - operation which acquires them, we pay a massive performance - penalty: speedtest1 benchmarks take up to 4x as long. By delaying - the lock release until idle time, the hit is negligible. -*/ -const __autoLocks = new Set(); - -/** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg is - true, the result is returned as an array of path elements, else an - absolute path string is returned. -*/ -const getResolvedPath = function(filename,splitIt){ - const p = new URL( - filename, 'file://irrelevant' - ).pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; -}; - -/** - Takes the absolute path to a filesystem element. Returns an array - of [handleOfContainingDir, filename]. If the 2nd argument is truthy - then each directory element leading to the file is created along - the way. Throws if any creation or resolution fails. -*/ -const getDirForFilename = async function f(absFilename, createDirs = false){ - const path = getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = state.rootDir; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; -}; - -/** - An error class specifically for use with getSyncHandle(), the goal - of which is to eventually be able to distinguish unambiguously - between locking-related failures and other types, noting that we - cannot currently do so because createSyncAccessHandle() does not - define its exceptions in the required level of detail. -*/ -class GetSyncHandleError extends Error { - constructor(errorObject, ...msg){ - super(); - this.error = errorObject; - this.message = [ - ...msg, ': Original exception ['+errorObject.name+']:', - errorObject.message - ].join(' '); - this.name = 'GetSyncHandleError'; - } -}; - -/** - Returns the sync access handle associated with the given file - handle object (which must be a valid handle object, as created by - xOpen()), lazily opening it if needed. - - In order to help alleviate cross-tab contention for a dabase, - if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to 3 times. If acquisition - still fails at that point it will give up and propagate the - exception. -*/ -const getSyncHandle = async (fh)=>{ - if(!fh.syncHandle){ - const t = performance.now(); - log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 4, msBase = 300; - let i = 1, ms = msBase; - for(; true; ms = msBase * ++i){ - try { - //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); - //TODO? A config option which tells it to throw here - //randomly every now and then, for testing purposes. - fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); - break; - }catch(e){ - if(i === maxTries){ - throw new GetSyncHandleError( - e, "Error getting sync handle.",maxTries, - "attempts failed.",fh.filenameAbs - ); - } - warn("Error getting sync handle. Waiting",ms, - "ms and trying again.",fh.filenameAbs,e); - Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); - } - } - log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); - if(!fh.xLock){ - __autoLocks.add(fh.fid); - log("Auto-locked",fh.fid,fh.filenameAbs); - } - } - return fh.syncHandle; -}; - -/** - If the given file-holding object has a sync handle attached to it, - that handle is remove and asynchronously closed. Though it may - sound sensible to continue work as soon as the close() returns - (noting that it's asynchronous), doing so can cause operations - performed soon afterwards, e.g. a call to getSyncHandle() to fail - because they may happen out of order from the close(). OPFS does - not guaranty that the actual order of operations is retained in - such cases. i.e. always "await" on the result of this function. -*/ -const closeSyncHandle = async (fh)=>{ - if(fh.syncHandle){ - log("Closing sync handle for",fh.filenameAbs); - const h = fh.syncHandle; - delete fh.syncHandle; - delete fh.xLock; - __autoLocks.delete(fh.fid); - return h.close(); - } -}; - -/** - A proxy for closeSyncHandle() which is guaranteed to not throw. - - This function is part of a lock/unlock step in functions which - require a sync access handle but may be called without xLock() - having been called first. Such calls need to release that - handle to avoid locking the file for all of time. This is an - _attempt_ at reducing cross-tab contention but it may prove - to be more of a problem than a solution and may need to be - removed. -*/ -const closeSyncHandleNoThrow = async (fh)=>{ - try{await closeSyncHandle(fh)} - catch(e){ - warn("closeSyncHandleNoThrow() ignoring:",e,fh); - } -}; - -/** - Stores the given value at state.sabOPView[state.opIds.rc] and then - Atomics.notify()'s it. -*/ -const storeAndNotify = (opName, value)=>{ - log(opName+"() => notify(",value,")"); - Atomics.store(state.sabOPView, state.opIds.rc, value); - Atomics.notify(state.sabOPView, state.opIds.rc); -}; - -/** - Throws if fh is a file-holding object which is flagged as read-only. -*/ -const affirmNotRO = function(opName,fh){ - if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); -}; -const affirmLocked = function(opName,fh){ - //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); - /** - Currently a no-op, as speedtest1 triggers xRead() without a - lock (that seems like a bug but it's currently uninvestigated). - This means, however, that some OPFS VFS routines may trigger - acquisition of a lock but never let it go until xUnlock() is - called (which it likely won't be if xLock() was not called). - */ -}; - -/** - We track 2 different timers: the "metrics" timer records how much - time we spend performing work. The "wait" timer records how much - time we spend waiting on the underlying OPFS timer. See the calls - to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() - throughout this file to see how they're used. -*/ -const __mTimer = Object.create(null); -__mTimer.op = undefined; -__mTimer.start = undefined; -const mTimeStart = (op)=>{ - __mTimer.start = performance.now(); - __mTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - ++metrics[op].count; -}; -const mTimeEnd = ()=>( - metrics[__mTimer.op].time += performance.now() - __mTimer.start -); -const __wTimer = Object.create(null); -__wTimer.op = undefined; -__wTimer.start = undefined; -const wTimeStart = (op)=>{ - __wTimer.start = performance.now(); - __wTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); -}; -const wTimeEnd = ()=>( - metrics[__wTimer.op].wait += performance.now() - __wTimer.start -); - -/** - Gets set to true by the 'opfs-async-shutdown' command to quit the - wait loop. This is only intended for debugging purposes: we cannot - inspect this file's state while the tight waitLoop() is running and - need a way to stop that loop for introspection purposes. -*/ -let flagAsyncShutdown = false; - - -/** - Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods - methods, as well as helpers like mkdir(). Maintenance reminder: - members are in alphabetical order to simplify finding them. -*/ -const vfsAsyncImpls = { - 'opfs-async-metrics': async ()=>{ - mTimeStart('opfs-async-metrics'); - metrics.dump(); - storeAndNotify('opfs-async-metrics', 0); - mTimeEnd(); - }, - 'opfs-async-shutdown': async ()=>{ - flagAsyncShutdown = true; - storeAndNotify('opfs-async-shutdown', 0); - }, - mkdir: async (dirname)=>{ - mTimeStart('mkdir'); - let rc = 0; - wTimeStart('mkdir'); - try { - await getDirForFilename(dirname+"/filepart", true); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); - } - storeAndNotify('mkdir', rc); - mTimeEnd(); - }, - xAccess: async (filename)=>{ - mTimeStart('xAccess'); - /* OPFS cannot support the full range of xAccess() queries sqlite3 - calls for. We can essentially just tell if the file is - accessible, but if it is it's automatically writable (unless - it's locked, which we cannot(?) know without trying to open - it). OPFS does not have the notion of read-only. - - The return semantics of this function differ from sqlite3's - xAccess semantics because we are limited in what we can - communicate back to our synchronous communication partner: 0 = - accessible, non-0 means not accessible. - */ - let rc = 0; - wTimeStart('xAccess'); - try{ - const [dh, fn] = await getDirForFilename(filename); - await dh.getFileHandle(fn); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); - } - storeAndNotify('xAccess', rc); - mTimeEnd(); - }, - xClose: async function(fid/*sqlite3_file pointer*/){ - const opName = 'xClose'; - mTimeStart(opName); - __autoLocks.delete(fid); - const fh = __openFiles[fid]; - let rc = 0; - wTimeStart(opName); - if(fh){ - delete __openFiles[fid]; - await closeSyncHandle(fh); - if(fh.deleteOnClose){ - try{ await fh.dirHandle.removeEntry(fh.filenamePart) } - catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } - } - }else{ - state.s11n.serialize(); - rc = state.sq3Codes.SQLITE_NOTFOUND; - } - wTimeEnd(); - storeAndNotify(opName, rc); - mTimeEnd(); - }, - xDelete: async function(...args){ - mTimeStart('xDelete'); - const rc = await vfsAsyncImpls.xDeleteNoWait(...args); - storeAndNotify('xDelete', rc); - mTimeEnd(); - }, - xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ - /* The syncDir flag is, for purposes of the VFS API's semantics, - ignored here. However, if it has the value 0x1234 then: after - deleting the given file, recursively try to delete any empty - directories left behind in its wake (ignoring any errors and - stopping at the first failure). - - That said: we don't know for sure that removeEntry() fails if - the dir is not empty because the API is not documented. It has, - however, a "recursive" flag which defaults to false, so - presumably it will fail if the dir is not empty and that flag - is false. - */ - let rc = 0; - wTimeStart('xDelete'); - try { - while(filename){ - const [hDir, filenamePart] = await getDirForFilename(filename, false); - if(!filenamePart) break; - await hDir.removeEntry(filenamePart, {recursive}); - if(0x1234 !== syncDir) break; - recursive = false; - filename = getResolvedPath(filename, true); - filename.pop(); - filename = filename.join('/'); - } - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_DELETE; - } - wTimeEnd(); - return rc; - }, - xFileSize: async function(fid/*sqlite3_file pointer*/){ - mTimeStart('xFileSize'); - const fh = __openFiles[fid]; - let rc; - wTimeStart('xFileSize'); - try{ - affirmLocked('xFileSize',fh); - rc = await (await getSyncHandle(fh)).getSize(); - state.s11n.serialize(Number(rc)); - rc = 0; - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR; - } - wTimeEnd(); - storeAndNotify('xFileSize', rc); - mTimeEnd(); - }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xLock'); - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - wTimeStart('xLock'); - try { - await getSyncHandle(fh); - __autoLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_LOCK; - fh.xLock = oldLockType; - } - wTimeEnd(); - } - storeAndNotify('xLock',rc); - mTimeEnd(); - }, - xOpen: async function(fid/*sqlite3_file pointer*/, filename, - flags/*SQLITE_OPEN_...*/){ - const opName = 'xOpen'; - mTimeStart(opName); - const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); - const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); - wTimeStart('xOpen'); - try{ - let hDir, filenamePart; - try { - [hDir, filenamePart] = await getDirForFilename(filename, !!create); - }catch(e){ - state.s11n.storeException(1,e); - storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); - mTimeEnd(); - wTimeEnd(); - return; - } - const hFile = await hDir.getFileHandle(filenamePart, {create}); - /** - wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the syncHandle prop of the file state - object, but only for certain cases and it's unclear why it - places that limitation on it. - */ - wTimeEnd(); - __openFiles[fid] = Object.assign(Object.create(null),{ - fid: fid, - filenameAbs: filename, - filenamePart: filenamePart, - dirHandle: hDir, - fileHandle: hFile, - sabView: state.sabFileBufView, - readOnly: create - ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), - deleteOnClose: deleteOnClose - }); - storeAndNotify(opName, 0); - }catch(e){ - wTimeEnd(); - error(opName,e); - state.s11n.storeException(1,e); - storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); - } - mTimeEnd(); - }, - xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xRead'); - let rc = 0, nRead; - const fh = __openFiles[fid]; - try{ - affirmLocked('xRead',fh); - wTimeStart('xRead'); - nRead = (await getSyncHandle(fh)).read( - fh.sabView.subarray(0, n), - {at: Number(offset64)} - ); - wTimeEnd(); - if(nRead < n){/* Zero-fill remaining bytes */ - fh.sabView.fill(0, nRead, n); - rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; - } - }catch(e){ - if(undefined===nRead) wTimeEnd(); - error("xRead() failed",e,fh); - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_READ; - } - storeAndNotify('xRead',rc); - mTimeEnd(); - }, - xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ - mTimeStart('xSync'); - const fh = __openFiles[fid]; - let rc = 0; - if(!fh.readOnly && fh.syncHandle){ - try { - wTimeStart('xSync'); - await fh.syncHandle.flush(); - }catch(e){ - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_FSYNC; - } - wTimeEnd(); - } - storeAndNotify('xSync',rc); - mTimeEnd(); - }, - xTruncate: async function(fid/*sqlite3_file pointer*/,size){ - mTimeStart('xTruncate'); - let rc = 0; - const fh = __openFiles[fid]; - wTimeStart('xTruncate'); - try{ - affirmLocked('xTruncate',fh); - affirmNotRO('xTruncate', fh); - await (await getSyncHandle(fh)).truncate(size); - }catch(e){ - error("xTruncate():",e,fh); - state.s11n.storeException(2,e); - rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; - } - wTimeEnd(); - storeAndNotify('xTruncate',rc); - mTimeEnd(); - }, - xUnlock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xUnlock'); - let rc = 0; - const fh = __openFiles[fid]; - if( state.sq3Codes.SQLITE_LOCK_NONE===lockType - && fh.syncHandle ){ - wTimeStart('xUnlock'); - try { await closeSyncHandle(fh) } - catch(e){ - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; - } - wTimeEnd(); - } - storeAndNotify('xUnlock',rc); - mTimeEnd(); - }, - xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xWrite'); - let rc; - const fh = __openFiles[fid]; - wTimeStart('xWrite'); - try{ - affirmLocked('xWrite',fh); - affirmNotRO('xWrite', fh); - rc = ( - n === (await getSyncHandle(fh)) - .write(fh.sabView.subarray(0, n), - {at: Number(offset64)}) - ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; - }catch(e){ - error("xWrite():",e,fh); - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_WRITE; - } - wTimeEnd(); - storeAndNotify('xWrite',rc); - mTimeEnd(); - } -}/*vfsAsyncImpls*/; - -const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); - } - }) - : ()=>{}; - - return state.s11n; -}/*initS11n()*/; - -const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; - } - /** - waitTime is how long (ms) to wait for each Atomics.wait(). - We need to wake up periodically to give the thread a chance - to do other things. - */ - const waitTime = 500; - while(!flagAsyncShutdown){ - try { - if('timed-out'===Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, waitTime - )){ - if(__autoLocks.size){ - /* Release all auto-locks. */ - for(const fid of __autoLocks){ - const fh = __openFiles[fid]; - await closeSyncHandleNoThrow(fh); - log("Auto-unlocked",fid,fh.filenameAbs); - } - } - continue; - } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); - const args = state.s11n.deserialize( - true /* clear s11n to keep the caller from confusing this with - an exception string written by the upcoming - operation */ - ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); - }catch(e){ - error('in waitLoop():',e); - } - } -}; - -navigator.storage.getDirectory().then(function(d){ - const wMsg = (type)=>postMessage({type}); - state.rootDir = d; - self.onmessage = function({data}){ - switch(data.type){ - case 'opfs-async-init':{ - /* Receive shared state from synchronous partner */ - const opt = data.args; - state.littleEndian = opt.littleEndian; - state.asyncS11nExceptions = opt.asyncS11nExceptions; - state.verbose = opt.verbose ?? 2; - state.fileBufferSize = opt.fileBufferSize; - state.sabS11nOffset = opt.sabS11nOffset; - state.sabS11nSize = opt.sabS11nSize; - state.sabOP = opt.sabOP; - state.sabOPView = new Int32Array(state.sabOP); - state.sabIO = opt.sabIO; - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.opIds = opt.opIds; - state.sq3Codes = opt.sq3Codes; - Object.keys(vfsAsyncImpls).forEach((k)=>{ - if(!Number.isFinite(state.opIds[k])){ - toss("Maintenance required: missing state.opIds[",k,"]"); - } - }); - initS11n(); - metrics.reset(); - log("init state",state); - wMsg('opfs-async-inited'); - waitLoop(); - break; - } - case 'opfs-async-restart': - if(flagAsyncShutdown){ - warn("Restarting after opfs-async-shutdown. Might or might not work."); - flagAsyncShutdown = false; - waitLoop(); - } - break; - case 'opfs-async-metrics': - metrics.dump(); - break; - } - }; - wMsg('opfs-async-loaded'); -}).catch((e)=>error("error initializing OPFS asyncer:",e)); + + 2022-11-27: Chrome v108 changes some async methods to synchronous, as + documented at: + + https://developer.chrome.com/blog/sync-methods-for-accesshandles/ + + We cannot change to the sync forms at this point without breaking + clients who use Chrome v104-ish or higher. truncate(), getSize(), + flush(), and close() are now (as of v108) synchronous. Calling them + with an "await", as we have to for the async forms, is still legal + with the sync forms but is superfluous. Calling the async forms with + theFunc().then(...) is not compatible with the change to + synchronous, but we do do not use those APIs that way. i.e. we don't + _need_ to change anything for this, but at some point (after Chrome + versions (approximately) 104-107 are extinct) should change our + usage of those methods to remove the "await". +*/ +"use strict"; +const wPost = (type,...args)=>postMessage({type, payload:args}); +const installAsyncProxy = function(self){ + const toss = function(...args){throw new Error(args.join(' '))}; + if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); + }else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); + } + + /** + Will hold state copied to this object from the syncronous side of + this API. + */ + const state = Object.create(null); + + /** + verbose: + + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors + */ + state.verbose = 1; + + const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) + }; + const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const metrics = Object.create(null); + metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + }; + metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":\n", + metrics, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",metrics.s11n); + }; + + /** + __openFiles is a map of sqlite3_file pointers (integers) to + metadata related to a given OPFS file handles. The pointers are, in + this side of the interface, opaque file handle IDs provided by the + synchronous part of this constellation. Each value is an object + with a structure demonstrated in the xOpen() impl. + */ + const __openFiles = Object.create(null); + /** + __implicitLocks is a Set of sqlite3_file pointers (integers) which were + "auto-locked". i.e. those for which we obtained a sync access + handle without an explicit xLock() call. Such locks will be + released during db connection idle time, whereas a sync access + handle obtained via xLock(), or subsequently xLock()'d after + auto-acquisition, will not be released until xUnlock() is called. + + Maintenance reminder: if we relinquish auto-locks at the end of the + operation which acquires them, we pay a massive performance + penalty: speedtest1 benchmarks take up to 4x as long. By delaying + the lock release until idle time, the hit is negligible. + */ + const __implicitLocks = new Set(); + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. + */ + const getResolvedPath = function(filename,splitIt){ + const p = new URL( + filename, 'file://irrelevant' + ).pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. + */ + const getDirForFilename = async function f(absFilename, createDirs = false){ + const path = getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. + */ + const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + log("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + delete fh.xLock; + __implicitLocks.delete(fh.fid); + return h.close(); + } + }; + + /** + A proxy for closeSyncHandle() which is guaranteed to not throw. + + This function is part of a lock/unlock step in functions which + require a sync access handle but may be called without xLock() + having been called first. Such calls need to release that + handle to avoid locking the file for all of time. This is an + _attempt_ at reducing cross-tab contention but it may prove + to be more of a problem than a solution and may need to be + removed. + */ + const closeSyncHandleNoThrow = async (fh)=>{ + try{await closeSyncHandle(fh)} + catch(e){ + warn("closeSyncHandleNoThrow() ignoring:",e,fh); + } + }; + + /* Release all auto-locks. */ + const releaseImplicitLocks = async ()=>{ + if(__implicitLocks.size){ + /* Release all auto-locks. */ + for(const fid of __implicitLocks){ + const fh = __openFiles[fid]; + await closeSyncHandleNoThrow(fh); + log("Auto-unlocked",fid,fh.filenameAbs); + } + } + }; + + /** + An experiment in improving concurrency by freeing up implicit locks + sooner. This is known to impact performance dramatically but it has + also shown to improve concurrency considerably. + + If fh.releaseImplicitLocks is truthy and fh is in __implicitLocks, + this routine returns closeSyncHandleNoThrow(), else it is a no-op. + */ + const releaseImplicitLock = async (fh)=>{ + if(fh.releaseImplicitLocks && __implicitLocks.has(fh.fid)){ + return closeSyncHandleNoThrow(fh); + } + }; + + /** + An error class specifically for use with getSyncHandle(), the goal + of which is to eventually be able to distinguish unambiguously + between locking-related failures and other types, noting that we + cannot currently do so because createSyncAccessHandle() does not + define its exceptions in the required level of detail. + + 2022-11-29: according to: + + https://github.com/whatwg/fs/pull/21 + + NoModificationAllowedError will be the standard exception thrown + when acquisition of a sync access handle fails due to a locking + error. As of this writing, that error type is not visible in the + dev console in Chrome v109, nor is it documented in MDN, but an + error with that "name" property is being thrown from the OPFS + layer. + */ + class GetSyncHandleError extends Error { + constructor(errorObject, ...msg){ + super([ + ...msg, ': '+errorObject.name+':', + errorObject.message + ].join(' '), { + cause: errorObject + }); + this.name = 'GetSyncHandleError'; + } + }; + GetSyncHandleError.convertRc = (e,rc)=>{ + if(1){ + return ( + e instanceof GetSyncHandleError + && ((e.cause.name==='NoModificationAllowedError') + /* Inconsistent exception.name from Chrome/ium with the + same exception.message text: */ + || (e.cause.name==='DOMException' + && 0===e.cause.message.indexOf('Access Handles cannot'))) + ) ? ( + /*console.warn("SQLITE_BUSY",e),*/ + state.sq3Codes.SQLITE_BUSY + ) : rc; + }else{ + return rc; + } + } + /** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object, as created by + xOpen()), lazily opening it if needed. + + In order to help alleviate cross-tab contention for a dabase, if + an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to some fixed number of + times. If acquisition still fails at that point it will give up + and propagate the exception. Client-level code will see that as + an I/O error. + */ + const getSyncHandle = async (fh,opName)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 6, + msBase = state.asyncIdleWaitTime * 2; + let i = 1, ms = msBase; + for(; true; ms = msBase * ++i){ + try { + //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + throw new GetSyncHandleError( + e, "Error getting sync handle for",opName+"().",maxTries, + "attempts failed.",fh.filenameAbs + ); + } + warn("Error getting sync handle for",opName+"(). Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); + } + } + log("Got",opName+"() sync handle for",fh.filenameAbs, + 'in',performance.now() - t,'ms'); + if(!fh.xLock){ + __implicitLocks.add(fh.fid); + log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); + } + } + return fh.syncHandle; + }; + + /** + Stores the given value at state.sabOPView[state.opIds.rc] and then + Atomics.notify()'s it. + */ + const storeAndNotify = (opName, value)=>{ + log(opName+"() => notify(",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); + }; + + /** + Throws if fh is a file-holding object which is flagged as read-only. + */ + const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); + }; + + /** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. + */ + const __mTimer = Object.create(null); + __mTimer.op = undefined; + __mTimer.start = undefined; + const mTimeStart = (op)=>{ + __mTimer.start = performance.now(); + __mTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[__mTimer.op].time += performance.now() - __mTimer.start + ); + const __wTimer = Object.create(null); + __wTimer.op = undefined; + __wTimer.start = undefined; + const wTimeStart = (op)=>{ + __wTimer.start = performance.now(); + __wTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + }; + const wTimeEnd = ()=>( + metrics[__wTimer.op].wait += performance.now() - __wTimer.start + ); + + /** + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. + */ + let flagAsyncShutdown = false; + + /** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const vfsAsyncImpls = { + 'opfs-async-metrics': async ()=>{ + mTimeStart('opfs-async-metrics'); + metrics.dump(); + storeAndNotify('opfs-async-metrics', 0); + mTimeEnd(); + }, + 'opfs-async-shutdown': async ()=>{ + flagAsyncShutdown = true; + storeAndNotify('opfs-async-shutdown', 0); + }, + mkdir: async (dirname)=>{ + mTimeStart('mkdir'); + let rc = 0; + wTimeStart('mkdir'); + try { + await getDirForFilename(dirname+"/filepart", true); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('mkdir', rc); + mTimeEnd(); + }, + xAccess: async (filename)=>{ + mTimeStart('xAccess'); + /* OPFS cannot support the full range of xAccess() queries + sqlite3 calls for. We can essentially just tell if the file + is accessible, but if it is then it's automatically writable + (unless it's locked, which we cannot(?) know without trying + to open it). OPFS does not have the notion of read-only. + + The return semantics of this function differ from sqlite3's + xAccess semantics because we are limited in what we can + communicate back to our synchronous communication partner: 0 = + accessible, non-0 means not accessible. + */ + let rc = 0; + wTimeStart('xAccess'); + try{ + const [dh, fn] = await getDirForFilename(filename); + await dh.getFileHandle(fn); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('xAccess', rc); + mTimeEnd(); + }, + xClose: async function(fid/*sqlite3_file pointer*/){ + const opName = 'xClose'; + mTimeStart(opName); + __implicitLocks.delete(fid); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart(opName); + if(fh){ + delete __openFiles[fid]; + await closeSyncHandle(fh); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + }else{ + state.s11n.serialize(); + rc = state.sq3Codes.SQLITE_NOTFOUND; + } + wTimeEnd(); + storeAndNotify(opName, rc); + mTimeEnd(); + }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, + xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ + /* The syncDir flag is, for purposes of the VFS API's semantics, + ignored here. However, if it has the value 0x1234 then: after + deleting the given file, recursively try to delete any empty + directories left behind in its wake (ignoring any errors and + stopping at the first failure). + + That said: we don't know for sure that removeEntry() fails if + the dir is not empty because the API is not documented. It has, + however, a "recursive" flag which defaults to false, so + presumably it will fail if the dir is not empty and that flag + is false. + */ + let rc = 0; + wTimeStart('xDelete'); + try { + while(filename){ + const [hDir, filenamePart] = await getDirForFilename(filename, false); + if(!filenamePart) break; + await hDir.removeEntry(filenamePart, {recursive}); + if(0x1234 !== syncDir) break; + recursive = false; + filename = getResolvedPath(filename, true); + filename.pop(); + filename = filename.join('/'); + } + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_DELETE; + } + wTimeEnd(); + return rc; + }, + xFileSize: async function(fid/*sqlite3_file pointer*/){ + mTimeStart('xFileSize'); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart('xFileSize'); + try{ + const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); + state.s11n.serialize(Number(sz)); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xFileSize', rc); + mTimeEnd(); + }, + xLock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + wTimeStart('xLock'); + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } + wTimeEnd(); + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, + xOpen: async function(fid/*sqlite3_file pointer*/, filename, + flags/*SQLITE_OPEN_...*/, + opfsFlags/*OPFS_...*/){ + const opName = 'xOpen'; + mTimeStart(opName); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); + try{ + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForFilename(filename, !!create); + }catch(e){ + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); + mTimeEnd(); + wTimeEnd(); + return; + } + const hFile = await hDir.getFileHandle(filenamePart, {create}); + wTimeEnd(); + const fh = Object.assign(Object.create(null),{ + fid: fid, + filenameAbs: filename, + filenamePart: filenamePart, + dirHandle: hDir, + fileHandle: hFile, + sabView: state.sabFileBufView, + readOnly: create + ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + deleteOnClose: !!(state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags) + }); + fh.releaseImplicitLocks = + (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) + || state.opfsFlags.defaultUnlockAsap; + if(0 /* this block is modelled after something wa-sqlite + does but it leads to immediate contention on journal files. + Update: this approach reportedly only works for DELETE journal + mode. */ + && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ + /* sqlite does not lock these files, so go ahead and grab an OPFS + lock. */ + fh.xLock = "xOpen"/* Truthy value to keep entry from getting + flagged as auto-locked. String value so + that we can easily distinguish is later + if needed. */; + await getSyncHandle(fh,'xOpen'); + } + __openFiles[fid] = fh; + storeAndNotify(opName, 0); + }catch(e){ + wTimeEnd(); + error(opName,e); + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); + } + mTimeEnd(); + }, + xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xRead'); + let rc = 0, nRead; + const fh = __openFiles[fid]; + try{ + wTimeStart('xRead'); + nRead = (await getSyncHandle(fh,'xRead')).read( + fh.sabView.subarray(0, n), + {at: Number(offset64)} + ); + wTimeEnd(); + if(nRead < n){/* Zero-fill remaining bytes */ + fh.sabView.fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + if(undefined===nRead) wTimeEnd(); + error("xRead() failed",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); + } + await releaseImplicitLock(fh); + storeAndNotify('xRead',rc); + mTimeEnd(); + }, + xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ + mTimeStart('xSync'); + const fh = __openFiles[fid]; + let rc = 0; + if(!fh.readOnly && fh.syncHandle){ + try { + wTimeStart('xSync'); + await fh.syncHandle.flush(); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + } + wTimeEnd(); + } + storeAndNotify('xSync',rc); + mTimeEnd(); + }, + xTruncate: async function(fid/*sqlite3_file pointer*/,size){ + mTimeStart('xTruncate'); + let rc = 0; + const fh = __openFiles[fid]; + wTimeStart('xTruncate'); + try{ + affirmNotRO('xTruncate', fh); + await (await getSyncHandle(fh,'xTruncate')).truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + state.s11n.storeException(2,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xTruncate',rc); + mTimeEnd(); + }, + xUnlock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + wTimeStart('xUnlock'); + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + wTimeEnd(); + } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, + xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xWrite'); + let rc; + const fh = __openFiles[fid]; + wTimeStart('xWrite'); + try{ + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh,'xWrite')) + .write(fh.sabView.subarray(0, n), + {at: Number(offset64)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); + } + await releaseImplicitLock(fh); + wTimeEnd(); + storeAndNotify('xWrite',rc); + mTimeEnd(); + } + }/*vfsAsyncImpls*/; + + const initS11n = ()=>{ + /** + ACHTUNG: this code is 100% duplicated in the other half of this + proxy! The documentation is maintained in the "synchronous half". + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; + + return state.s11n; + }/*initS11n()*/; + + const waitLoop = async function f(){ + const opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } + while(!flagAsyncShutdown){ + try { + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime + )){ + await releaseImplicitLocks(); + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const args = state.s11n.deserialize( + true /* clear s11n to keep the caller from confusing this with + an exception string written by the upcoming + operation */ + ) || []; + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); + }catch(e){ + error('in waitLoop():',e); + } + } + }; + + navigator.storage.getDirectory().then(function(d){ + state.rootDir = d; + self.onmessage = function({data}){ + switch(data.type){ + case 'opfs-async-init':{ + /* Receive shared state from synchronous partner */ + const opt = data.args; + for(const k in opt) state[k] = opt[k]; + state.verbose = opt.verbose ?? 1; + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); + } + }); + initS11n(); + metrics.reset(); + log("init state",state); + wPost('opfs-async-inited'); + waitLoop(); + break; + } + case 'opfs-async-restart': + if(flagAsyncShutdown){ + warn("Restarting after opfs-async-shutdown. Might or might not work."); + flagAsyncShutdown = false; + waitLoop(); + } + break; + case 'opfs-async-metrics': + metrics.dump(); + break; + } + }; + wPost('opfs-async-loaded'); + }).catch((e)=>error("error initializing OPFS asyncer:",e)); +}/*installAsyncProxy()*/; +if(!self.SharedArrayBuffer){ + wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", + "The server must emit the COOP/COEP response headers to enable that."); +}else if(!self.Atomics){ + wPost('opfs-unavailable', "Missing Atomics API.", + "The server must emit the COOP/COEP response headers to enable that."); +}else if(!self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + wPost('opfs-unavailable',"Missing required OPFS APIs."); +}else{ + installAsyncProxy(self); +} ADDED ext/wasm/api/sqlite3-v-helper.js Index: ext/wasm/api/sqlite3-v-helper.js ================================================================== --- /dev/null +++ ext/wasm/api/sqlite3-v-helper.js @@ -0,0 +1,714 @@ +/* +** 2022-11-30 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ + +/** + This file installs sqlite3.vfs, and object which exists to assist + in the creation of JavaScript implementations of sqlite3_vfs, along + with its virtual table counterpart, sqlite3.vtab. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; + const vfs = Object.create(null), vtab = Object.create(null); + + sqlite3.vfs = vfs; + sqlite3.vtab = vtab; + + const sii = capi.sqlite3_index_info; + /** + If n is >=0 and less than this.$nConstraint, this function + returns either a WASM pointer to the 0-based nth entry of + this.$aConstraint (if passed a truthy 2nd argument) or an + sqlite3_index_info.sqlite3_index_constraint object wrapping that + address (if passed a falsy value or no 2nd argument). Returns a + falsy value if n is out of range. + */ + sii.prototype.nthConstraint = function(n, asPtr=false){ + if(n<0 || n>=this.$nConstraint) return false; + const ptr = this.$aConstraint + ( + sii.sqlite3_index_constraint.structInfo.sizeof * n + ); + return asPtr ? ptr : new sii.sqlite3_index_constraint(ptr); + }; + + /** + Works identically to nthConstraint() but returns state from + this.$aConstraintUsage, so returns an + sqlite3_index_info.sqlite3_index_constraint_usage instance + if passed no 2nd argument or a falsy 2nd argument. + */ + sii.prototype.nthConstraintUsage = function(n, asPtr=false){ + if(n<0 || n>=this.$nConstraint) return false; + const ptr = this.$aConstraintUsage + ( + sii.sqlite3_index_constraint_usage.structInfo.sizeof * n + ); + return asPtr ? ptr : new sii.sqlite3_index_constraint_usage(ptr); + }; + + /** + If n is >=0 and less than this.$nOrderBy, this function + returns either a WASM pointer to the 0-based nth entry of + this.$aOrderBy (if passed a truthy 2nd argument) or an + sqlite3_index_info.sqlite3_index_orderby object wrapping that + address (if passed a falsy value or no 2nd argument). Returns a + falsy value if n is out of range. + */ + sii.prototype.nthOrderBy = function(n, asPtr=false){ + if(n<0 || n>=this.$nOrderBy) return false; + const ptr = this.$aOrderBy + ( + sii.sqlite3_index_orderby.structInfo.sizeof * n + ); + return asPtr ? ptr : new sii.sqlite3_index_orderby(ptr); + }; + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error, e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc. + + As a special case, if the given function is a pointer, then + `wasm.functionEntry()` is used to validate that it is a known + function. If so, it is used as-is with no extra level of proxying + or cleanup, else an exception is thrown. It is legal to pass a + value of 0, indicating a NULL pointer, with the caveat that 0 + _is_ a legal function pointer in WASM but it will not be accepted + as such _here_. (Justification: the function at address zero must + be one which initially came from the WASM module, not a method we + want to bind to a virtual table or VFS.) + + This function returns a proxy for itself which is bound to tgt + and takes 2 args (name,func). That function returns the same + thing as this one, permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + + ACHTUNG: because we cannot generically know how to transform JS + exceptions into result codes, the installed functions do no + automatic catching of exceptions. It is critical, to avoid + undefined behavior in the C layer, that methods mapped via + this function do not throw. The exception, as it were, to that + rule is... + + If applyArgcCheck is true then each JS function (as opposed to + function pointers) gets wrapped in a proxy which asserts that it + is passed the expected number of arguments, throwing if the + argument count does not match expectations. That is only intended + for dev-time usage for sanity checking, and will leave the C + environment in an undefined state. + */ + const installMethod = function callee( + tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck + ){ + if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + }else if(!(func instanceof Function) && !wasm.isPtr(func)){ + toss("Usage errror: expecting a Function or WASM pointer to one."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + } + if(!callee.argcProxy){ + callee.argcProxy = function(tgt, funcName, func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch for", + tgt.structInfo.name+"::"+funcName + +": Native signature is:",sig); + } + return func.apply(this, args); + } + }; + /* An ondispose() callback for use with + sqlite3.StructBinder-created types. */ + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name,"does not have a function pointer signature:",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = (applyArgcCheck && !wasm.isPtr(func)) + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(tgt, memKey, func, sigN) + : func; + if(wasm.isPtr(fProxy)){ + if(fProxy && !wasm.functionEntry(fProxy)){ + toss("Pointer",fProxy,"is not a WASM function table entry."); + } + tgt[memKey] = fProxy; + }else{ + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ + tgt.addOnDispose('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + } + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + }/*installMethod*/; + installMethod.installMethodArgcCheck = false; + + /** + Installs methods into the given StructType-type instance. Each + entry in the given methods object must map to a known member of + the given StructType, else an exception will be triggered. See + installMethod() for more details, including the semantics of the + 3rd argument. + + As an exception to the above, if any two or more methods in the + 2nd argument are the exact same function, installMethod() is + _not_ called for the 2nd and subsequent instances, and instead + those instances get assigned the same method pointer which is + created for the first instance. This optimization is primarily to + accommodate special handling of sqlite3_module::xConnect and + xCreate methods. + + On success, returns its first argument. Throws on error. + */ + const installMethods = function( + structInstance, methods, applyArgcCheck = installMethod.installMethodArgcCheck + ){ + const seen = new Map /* map of */; + for(const k of Object.keys(methods)){ + const m = methods[k]; + const prior = seen.get(m); + if(prior){ + const mkey = structInstance.memberKey(k); + structInstance[mkey] = structInstance[structInstance.memberKey(prior)]; + }else{ + installMethod(structInstance, k, m, applyArgcCheck); + seen.set(m, k); + } + } + return structInstance; + }; + + /** + Equivalent to calling installMethod(this,...arguments) with a + first argument of this object. If called with 1 or 2 arguments + and the first is an object, it's instead equivalent to calling + installMethods(this,...arguments). + */ + sqlite3.StructBinder.StructType.prototype.installMethod = function callee( + name, func, applyArgcCheck = installMethod.installMethodArgcCheck + ){ + return (arguments.length < 3 && name && 'object'===typeof name) + ? installMethods(this, ...arguments) + : installMethod(this, ...arguments); + }; + + /** + Equivalent to calling installMethods() with a first argument + of this object. + */ + sqlite3.StructBinder.StructType.prototype.installMethods = function( + methods, applyArgcCheck = installMethod.installMethodArgcCheck + ){ + return installMethods(this, methods, applyArgcCheck); + }; + + /** + Uses sqlite3_vfs_register() to register this + sqlite3.capi.sqlite3_vfs. This object must have already been + filled out properly. If the first argument is truthy, the VFS is + registered as the default VFS, else it is not. + + On success, returns this object. Throws on error. + */ + capi.sqlite3_vfs.prototype.registerVfs = function(asDefault=false){ + if(!(this instanceof sqlite3.capi.sqlite3_vfs)){ + toss("Expecting a sqlite3_vfs-type argument."); + } + const rc = capi.sqlite3_vfs_register(this, asDefault ? 1 : 0); + if(rc){ + toss("sqlite3_vfs_register(",this,") failed with rc",rc); + } + if(this.pointer !== capi.sqlite3_vfs_find(this.$zName)){ + toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", + this); + } + return this; + }; + + /** + A wrapper for installMethods() or registerVfs() to reduce + installation of a VFS and/or its I/O methods to a single + call. + + Accepts an object which contains the properties "io" and/or + "vfs", each of which is itself an object with following properties: + + - `struct`: an sqlite3.StructType-type struct. This must be a + populated (except for the methods) object of type + sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the + "vfs" entry). + + - `methods`: an object mapping sqlite3_io_methods method names + (e.g. 'xClose') to JS implementations of those methods. The JS + implementations must be call-compatible with their native + counterparts. + + For each of those object, this function passes its (`struct`, + `methods`, (optional) `applyArgcCheck`) properties to + installMethods(). + + If the `vfs` entry is set then: + + - Its `struct` property's registerVfs() is called. The + `vfs` entry may optionally have an `asDefault` property, which + gets passed as the argument to registerVfs(). + + - If `struct.$zName` is falsy and the entry has a string-type + `name` property, `struct.$zName` is set to the C-string form of + that `name` value before registerVfs() is called. + + On success returns this object. Throws on error. + */ + vfs.installVfs = function(opt){ + let count = 0; + const propList = ['io','vfs']; + for(const key of propList){ + const o = opt[key]; + if(o){ + ++count; + installMethods(o.struct, o.methods, !!o.applyArgcCheck); + if('vfs'===key){ + if(!o.struct.$zName && 'string'===typeof o.name){ + o.struct.addOnDispose( + o.struct.$zName = wasm.allocCString(o.name) + ); + } + o.struct.registerVfs(!!o.asDefault); + } + } + } + if(!count) toss("Misuse: installVfs() options object requires at least", + "one of:", propList); + return this; + }; + + /** + Internal factory function for xVtab and xCursor impls. + */ + const __xWrapFactory = function(methodName,StructType){ + return function(ptr,removeMapping=false){ + if(0===arguments.length) ptr = new StructType; + if(ptr instanceof StructType){ + //T.assert(!this.has(ptr.pointer)); + this.set(ptr.pointer, ptr); + return ptr; + }else if(!wasm.isPtr(ptr)){ + sqlite3.SQLite3Error.toss("Invalid argument to",methodName+"()"); + } + let rc = this.get(ptr); + if(removeMapping) this.delete(ptr); + return rc; + }.bind(new Map); + }; + + /** + A factory function which implements a simple lifetime manager for + mappings between C struct pointers and their JS-level wrappers. + The first argument must be the logical name of the manager + (e.g. 'xVtab' or 'xCursor'), which is only used for error + reporting. The second must be the capi.XYZ struct-type value, + e.g. capi.sqlite3_vtab or capi.sqlite3_vtab_cursor. + + Returns an object with 4 methods: create(), get(), unget(), and + dispose(), plus a StructType member with the value of the 2nd + argument. The methods are documented in the body of this + function. + */ + const StructPtrMapper = function(name, StructType){ + const __xWrap = __xWrapFactory(name,StructType); + /** + This object houses a small API for managing mappings of (`T*`) + to StructType objects, specifically within the lifetime + requirements of sqlite3_module methods. + */ + return Object.assign(Object.create(null),{ + /** The StructType object for this object's API. */ + StructType, + /** + Creates a new StructType object, writes its `pointer` + value to the given output pointer, and returns that + object. Its intended usage depends on StructType: + + sqlite3_vtab: to be called from sqlite3_module::xConnect() + or xCreate() implementations. + + sqlite3_vtab_cursor: to be called from xOpen(). + + This will throw if allocation of the StructType instance + fails or if ppOut is not a pointer-type value. + */ + create: (ppOut)=>{ + const rc = __xWrap(); + wasm.pokePtr(ppOut, rc.pointer); + return rc; + }, + /** + Returns the StructType object previously mapped to the + given pointer using create(). Its intended usage depends + on StructType: + + sqlite3_vtab: to be called from sqlite3_module methods which + take a (sqlite3_vtab*) pointer _except_ for + xDestroy()/xDisconnect(), in which case unget() or dispose(). + + sqlite3_vtab_cursor: to be called from any sqlite3_module methods + which take a `sqlite3_vtab_cursor*` argument except xClose(), + in which case use unget() or dispose(). + + Rule to remember: _never_ call dispose() on an instance + returned by this function. + */ + get: (pCObj)=>__xWrap(pCObj), + /** + Identical to get() but also disconnects the mapping between the + given pointer and the returned StructType object, such that + future calls to this function or get() with the same pointer + will return the undefined value. Its intended usage depends + on StructType: + + sqlite3_vtab: to be called from sqlite3_module::xDisconnect() or + xDestroy() implementations or in error handling of a failed + xCreate() or xConnect(). + + sqlite3_vtab_cursor: to be called from xClose() or during + cleanup in a failed xOpen(). + + Calling this method obligates the caller to call dispose() on + the returned object when they're done with it. + */ + unget: (pCObj)=>__xWrap(pCObj,true), + /** + Works like unget() plus it calls dispose() on the + StructType object. + */ + dispose: (pCObj)=>{ + const o = __xWrap(pCObj,true); + if(o) o.dispose(); + } + }); + }; + + /** + A lifetime-management object for mapping `sqlite3_vtab*` + instances in sqlite3_module methods to capi.sqlite3_vtab + objects. + + The API docs are in the API-internal StructPtrMapper(). + */ + vtab.xVtab = StructPtrMapper('xVtab', capi.sqlite3_vtab); + + /** + A lifetime-management object for mapping `sqlite3_vtab_cursor*` + instances in sqlite3_module methods to capi.sqlite3_vtab_cursor + objects. + + The API docs are in the API-internal StructPtrMapper(). + */ + vtab.xCursor = StructPtrMapper('xCursor', capi.sqlite3_vtab_cursor); + + /** + Convenience form of creating an sqlite3_index_info wrapper, + intended for use in xBestIndex implementations. Note that the + caller is expected to call dispose() on the returned object + before returning. Though not _strictly_ required, as that object + does not own the pIdxInfo memory, it is nonetheless good form. + */ + vtab.xIndexInfo = (pIdxInfo)=>new capi.sqlite3_index_info(pIdxInfo); + + /** + Given an error object, this function returns + sqlite3.capi.SQLITE_NOMEM if (e instanceof + sqlite3.WasmAllocError), else it returns its + second argument. Its intended usage is in the methods + of a sqlite3_vfs or sqlite3_module: + + ``` + try{ + let rc = ... + return rc; + }catch(e){ + return sqlite3.vtab.exceptionToRc(e, sqlite3.capi.SQLITE_XYZ); + // where SQLITE_XYZ is some call-appropriate result code. + } + ``` + */ + /**vfs.exceptionToRc = vtab.exceptionToRc = + (e, defaultRc=capi.SQLITE_ERROR)=>( + (e instanceof sqlite3.WasmAllocError) + ? capi.SQLITE_NOMEM + : defaultRc + );*/ + + /** + Given an sqlite3_module method name and error object, this + function returns sqlite3.capi.SQLITE_NOMEM if (e instanceof + sqlite3.WasmAllocError), else it returns its second argument. Its + intended usage is in the methods of a sqlite3_vfs or + sqlite3_module: + + ``` + try{ + let rc = ... + return rc; + }catch(e){ + return sqlite3.vtab.xError( + 'xColumn', e, sqlite3.capi.SQLITE_XYZ); + // where SQLITE_XYZ is some call-appropriate result code. + } + ``` + + If no 3rd argument is provided, its default depends on + the error type: + + - An sqlite3.WasmAllocError always resolves to capi.SQLITE_NOMEM. + + - If err is an SQLite3Error then its `resultCode` property + is used. + + - If all else fails, capi.SQLITE_ERROR is used. + + If xError.errorReporter is a function, it is called in + order to report the error, else the error is not reported. + If that function throws, that exception is ignored. + */ + vtab.xError = function f(methodName, err, defaultRc){ + if(f.errorReporter instanceof Function){ + try{f.errorReporter("sqlite3_module::"+methodName+"(): "+err.message);} + catch(e){/*ignored*/} + } + let rc; + if(err instanceof sqlite3.WasmAllocError) rc = capi.SQLITE_NOMEM; + else if(arguments.length>2) rc = defaultRc; + else if(err instanceof sqlite3.SQLite3Error) rc = err.resultCode; + return rc || capi.SQLITE_ERROR; + }; + vtab.xError.errorReporter = 1 ? console.error.bind(console) : false; + + /** + "The problem" with this is that it introduces an outer function with + a different arity than the passed-in method callback. That means we + cannot do argc validation on these. Additionally, some methods (namely + xConnect) may have call-specific error handling. It would be a shame to + hard-coded that per-method support in this function. + */ + /** vtab.methodCatcher = function(methodName, method, defaultErrRc=capi.SQLITE_ERROR){ + return function(...args){ + try { method(...args); } + }catch(e){ return vtab.xError(methodName, e, defaultRc) } + }; + */ + + /** + A helper for sqlite3_vtab::xRowid() and xUpdate() + implementations. It must be passed the final argument to one of + those methods (an output pointer to an int64 row ID) and the + value to store at the output pointer's address. Returns the same + as wasm.poke() and will throw if the 1st or 2nd arguments + are invalid for that function. + + Example xRowid impl: + + ``` + const xRowid = (pCursor, ppRowid64)=>{ + const c = vtab.xCursor(pCursor); + vtab.xRowid(ppRowid64, c.myRowId); + return 0; + }; + ``` + */ + vtab.xRowid = (ppRowid64, value)=>wasm.poke(ppRowid64, value, 'i64'); + + /** + A helper to initialize and set up an sqlite3_module object for + later installation into individual databases using + sqlite3_create_module(). Requires an object with the following + properties: + + - `methods`: an object containing a mapping of properties with + the C-side names of the sqlite3_module methods, e.g. xCreate, + xBestIndex, etc., to JS implementations for those functions. + Certain special-case handling is performed, as described below. + + - `catchExceptions` (default=false): if truthy, the given methods + are not mapped as-is, but are instead wrapped inside wrappers + which translate exceptions into result codes of SQLITE_ERROR or + SQLITE_NOMEM, depending on whether the exception is an + sqlite3.WasmAllocError. In the case of the xConnect and xCreate + methods, the exception handler also sets the output error + string to the exception's error string. + + - OPTIONAL `struct`: a sqlite3.capi.sqlite3_module() instance. If + not set, one will be created automatically. If the current + "this" is-a sqlite3_module then it is unconditionally used in + place of `struct`. + + - OPTIONAL `iVersion`: if set, it must be an integer value and it + gets assigned to the `$iVersion` member of the struct object. + If it's _not_ set, and the passed-in `struct` object's `$iVersion` + is 0 (the default) then this function attempts to define a value + for that property based on the list of methods it has. + + If `catchExceptions` is false, it is up to the client to ensure + that no exceptions escape the methods, as doing so would move + them through the C API, leading to undefined + behavior. (vtab.xError() is intended to assist in reporting + such exceptions.) + + Certain methods may refer to the same implementation. To simplify + the definition of such methods: + + - If `methods.xConnect` is `true` then the value of + `methods.xCreate` is used in its place, and vice versa. sqlite + treats xConnect/xCreate functions specially if they are exactly + the same function (same pointer value). + + - If `methods.xDisconnect` is true then the value of + `methods.xDestroy` is used in its place, and vice versa. + + This is to facilitate creation of those methods inline in the + passed-in object without requiring the client to explicitly get a + reference to one of them in order to assign it to the other + one. + + The `catchExceptions`-installed handlers will account for + identical references to the above functions and will install the + same wrapper function for both. + + The given methods are expected to return integer values, as + expected by the C API. If `catchExceptions` is truthy, the return + value of the wrapped function will be used as-is and will be + translated to 0 if the function returns a falsy value (e.g. if it + does not have an explicit return). If `catchExceptions` is _not_ + active, the method implementations must explicitly return integer + values. + + Throws on error. On success, returns the sqlite3_module object + (`this` or `opt.struct` or a new sqlite3_module instance, + depending on how it's called). + */ + vtab.setupModule = function(opt){ + let createdMod = false; + const mod = (this instanceof capi.sqlite3_module) + ? this : (opt.struct || (createdMod = new capi.sqlite3_module())); + try{ + const methods = opt.methods || toss("Missing 'methods' object."); + for(const e of Object.entries({ + // -----^ ==> [k,v] triggers a broken code transformation in + // some versions of the emsdk toolchain. + xConnect: 'xCreate', xDisconnect: 'xDestroy' + })){ + // Remap X=true to X=Y for certain X/Y combinations + const k = e[0], v = e[1]; + if(true === methods[k]) methods[k] = methods[v]; + else if(true === methods[v]) methods[v] = methods[k]; + } + if(opt.catchExceptions){ + const fwrap = function(methodName, func){ + if(['xConnect','xCreate'].indexOf(methodName) >= 0){ + return function(pDb, pAux, argc, argv, ppVtab, pzErr){ + try{return func(...arguments) || 0} + catch(e){ + if(!(e instanceof sqlite3.WasmAllocError)){ + wasm.dealloc(wasm.peekPtr(pzErr)); + wasm.pokePtr(pzErr, wasm.allocCString(e.message)); + } + return vtab.xError(methodName, e); + } + }; + }else{ + return function(...args){ + try{return func(...args) || 0} + catch(e){ + return vtab.xError(methodName, e); + } + }; + } + }; + const mnames = [ + 'xCreate', 'xConnect', 'xBestIndex', 'xDisconnect', + 'xDestroy', 'xOpen', 'xClose', 'xFilter', 'xNext', + 'xEof', 'xColumn', 'xRowid', 'xUpdate', + 'xBegin', 'xSync', 'xCommit', 'xRollback', + 'xFindFunction', 'xRename', 'xSavepoint', 'xRelease', + 'xRollbackTo', 'xShadowName' + ]; + const remethods = Object.create(null); + for(const k of mnames){ + const m = methods[k]; + if(!(m instanceof Function)) continue; + else if('xConnect'===k && methods.xCreate===m){ + remethods[k] = methods.xCreate; + }else if('xCreate'===k && methods.xConnect===m){ + remethods[k] = methods.xConnect; + }else{ + remethods[k] = fwrap(k, m); + } + } + installMethods(mod, remethods, false); + }else{ + // No automatic exception handling. Trust the client + // to not throw. + installMethods( + mod, methods, !!opt.applyArgcCheck/*undocumented option*/ + ); + } + if(0===mod.$iVersion){ + let v; + if('number'===typeof opt.iVersion) v = opt.iVersion; + else if(mod.$xShadowName) v = 3; + else if(mod.$xSavePoint || mod.$xRelease || mod.$xRollbackTo) v = 2; + else v = 1; + mod.$iVersion = v; + } + }catch(e){ + if(createdMod) createdMod.dispose(); + throw e; + } + return mod; + }/*setupModule()*/; + + /** + Equivalent to calling vtab.setupModule() with this sqlite3_module + object as the call's `this`. + */ + capi.sqlite3_module.prototype.setupModule = function(opt){ + return vtab.setupModule.call(this, opt); + }; +}/*sqlite3ApiBootstrap.initializers.push()*/); ADDED ext/wasm/api/sqlite3-vfs-opfs.c-pp.js Index: ext/wasm/api/sqlite3-vfs-opfs.c-pp.js ================================================================== --- /dev/null +++ ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -0,0 +1,1328 @@ +/* + 2022-09-18 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds the synchronous half of an sqlite3_vfs + implementation which proxies, in a synchronous fashion, the + asynchronous Origin-Private FileSystem (OPFS) APIs using a second + Worker, implemented in sqlite3-opfs-async-proxy.js. This file is + intended to be appended to the main sqlite3 JS deliverable somewhere + after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +/** + installOpfsVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializersAsync or an equivalent mechanism. + + The installed VFS uses the Origin-Private FileSystem API for + all file storage. On error it is rejected with an exception + explaining the problem. Reasons for rejection include, but are + not limited to: + + - The counterpart Worker (see below) could not be loaded. + + - The environment does not support OPFS. That includes when + this function is called from the main window thread. + + Significant notes and limitations: + + - As of this writing, OPFS is still very much in flux and only + available in bleeding-edge versions of Chrome (v102+, noting that + that number will increase as the OPFS API matures). + + - The OPFS features used here are only available in dedicated Worker + threads. This file tries to detect that case, resulting in a + rejected Promise if those features do not seem to be available. + + - It requires the SharedArrayBuffer and Atomics classes, and the + former is only available if the HTTP server emits the so-called + COOP and COEP response headers. These features are required for + proxying OPFS's synchronous API via the synchronous interface + required by the sqlite3_vfs API. + + - This function may only be called a single time. When called, this + function removes itself from the sqlite3 object. + + All arguments to this function are for internal/development purposes + only. They do not constitute a public API and may change at any + time. + + The argument may optionally be a plain object with the following + configuration options: + + - proxyUri: as described above + + - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables + logging of errors. 2 enables logging of warnings and errors. 3 + additionally enables debugging info. + + - sanityChecks (=false): if true, some basic sanity tests are + run on the OPFS VFS API after it's initialized, before the + returned Promise resolves. + + On success, the Promise resolves to the top-most sqlite3 namespace + object and that object gets a new object installed in its + `opfs` property, containing several OPFS-specific utilities. +*/ +const installOpfsVfs = function callee(options){ + if(!self.SharedArrayBuffer + || !self.Atomics){ + return Promise.reject( + new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ + "The server must emit the COOP/COEP response headers to enable those. "+ + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") + ); + }else if(self.window===self && self.document){ + return Promise.reject( + new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ + "because it requires Atomics.wait().") + ); + }else if(!self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + return Promise.reject( + new Error("Missing required OPFS APIs.") + ); + } + if(!options || 'object'!==typeof options){ + options = Object.create(null); + } + const urlParams = new URL(self.location.href).searchParams; + if(undefined===options.verbose){ + options.verbose = urlParams.has('opfs-verbose') + ? (+urlParams.get('opfs-verbose') || 2) : 1; + } + if(undefined===options.sanityChecks){ + options.sanityChecks = urlParams.has('opfs-sanity-check'); + } + if(undefined===options.proxyUri){ + options.proxyUri = callee.defaultProxyUri; + } + + //console.warn("OPFS options =",options,self.location); + + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) + }; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const toss = sqlite3.util.toss; + const capi = sqlite3.capi; + const wasm = sqlite3.wasm; + const sqlite3_vfs = capi.sqlite3_vfs; + const sqlite3_file = capi.sqlite3_file; + const sqlite3_io_methods = capi.sqlite3_io_methods; + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + ACHTUNG: do not rely on these APIs in client code. They are + experimental and subject to change or removal as the + OPFS-specific sqlite3_vfs evolves. + */ + const opfsUtil = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + const thisThreadHasOPFS = ()=>{ + return self.FileSystemHandle && + self.FileSystemDirectoryHandle && + self.FileSystemFileHandle && + self.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator.storage.getDirectory; + }; + + /** + Not part of the public API. Solely for internal/development + use. + */ + opfsUtil.metrics = { + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + console.log("Serialization metrics:",metrics.s11n); + W.postMessage({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + }/*metrics*/; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); + const promiseReject = function(err){ + opfsVfs.dispose(); + return promiseReject_(err); + }; + const W = +//#if target=es6-module + new Worker(new URL(options.proxyUri, import.meta.url)); +//#else + new Worker(options.proxyUri); +//#endif + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/*sure, why not?*/; + opfsVfs.$zName = wasm.allocCString("opfs"); + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + opfsVfs.ondispose = [ + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), + 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() + ]; + /** + Pedantic sidebar about opfsVfs.ondispose: the entries in that array + are items to clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the opfsVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + /** + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. + + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. + + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. + */ + const state = Object.create(null); + state.verbose = options.verbose; + state.littleEndian = (()=>{ + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 150; + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging. + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + state.opIds = Object.create(null); + const metrics = Object.create(null); + { + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes. */ + state.opIds.whichOp = i++; + /* Slot for storing return values. This worker listens to that + slot and the other worker writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to + the whichOp slot. The async-api worker uses Atomic.wait() + on the whichOp slot to figure out which operation to run + next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileControl = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++; + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. Though we could hypothetically use the xSleep slot + for that, doing so might lead to undesired side effects. */ + state.opIds.retry = i++; + state.sabOP = new SharedArrayBuffer( + i * 4/* ==sizeof int32, noting that Atomics.wait() and friends + can only function on Int32Array views of an SAB. */); + opfsUtil.metrics.reset(); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', + 'SQLITE_OPEN_READONLY' + ].forEach((k)=>{ + if(undefined === (state.sq3Codes[k] = capi[k])){ + toss("Maintenance required: not found:",k); + } + }); + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). "opfs-unlock-asap=1" enables + this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + If true, any async routine which implicitly acquires a sync + access handle (i.e. an OPFS lock) will release that locks at + the end of the call which acquires it. If false, such + "autolocks" are not released until the VFS is idle for some + brief amount of time. + + The benefit of enabling this is much higher concurrency. The + down-side is much-reduced performance (as much as a 4x decrease + in speedtest1). + */ + defaultUnlockAsap: false + }); + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The + 2nd and subsequent arguments must be the aruguments for the + async op. + */ + const opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + Atomics.wait(state.sabOPView, state.opIds.rc, -1) + /* When this wait() call returns, the async half will have + completed the operation and reported its results. */; + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + /** + Not part of the public API. Only for test/development use. + */ + opfsUtil.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; + + const initS11n = ()=>{ + /** + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ACHTUNG: this code is 100% duplicated in the other half of + this proxy! The documentation is maintained in the + "synchronous half". + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset]. Only one dataset is recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + return state.s11n; + }/*initS11n()*/; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = Object.create(null); + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + /** + As of late 2022, only a single lock can be held on an OPFS + file. We have no way of checking whether any _other_ db + connection has a lock except by trying to obtain and (on + success) release a sync-handle for it, but doing so would + involve an inherent race condition. For the time being, + pending a better solution, we simply report whether the + given pFile is open. + */ + const f = __openFiles[pFile]; + wasm.poke(pOut, f.lockType ? 1 : 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + //debug("xDeviceCharacteristics(",pFile,")"); + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + mTimeStart('xFileControl'); + const rc = (capi.SQLITE_FCNTL_SYNC===opId) + ? opRun('xSync', pFile, 0) + : capi.SQLITE_NOTFOUND; + mTimeEnd(); + return rc; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + let rc = opRun('xFileSize', pFile); + if(0==rc){ + try { + const sz = state.s11n.deserialize()[0]; + wasm.poke(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + mTimeEnd(); + return rc; + }, + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( !f.lockType ) { + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; + }else{ + f.lockType = lockType; + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), pDest); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + ++metrics.xSync.count; + return 0; // impl'd in xFileControl() + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); + } + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n)); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); + wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + // TODO: confirm that this calculation is correct + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); + /* We're ignoring errors because we cannot yet differentiate + between harmless and non-harmless failures. */ + mTimeEnd(); + return 0; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return ipMethods is NULL. */ + if(fh.readOnly){ + wasm.poke(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = fh; + fh.sabView = state.sabFileBufView; + fh.sq3File = new sqlite3_file(pFile); + fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; + } + mTimeEnd(); + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + if(dVfs){ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + assume it's sane and use it, otherwise install a JS-based + one. */ + vfsSyncWrappers.xSleep = function(pVfs,ms){ + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + return 0; + }; + } + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = opfsUtil.rootDirectory; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //console.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string, intended for use as a + temporary file name. Its argument is the length of the string, + defaulting to 16. + */ + opfsUtil.randomFilename = randomFilename; + + /** + Re-registers the OPFS VFS. This is intended only for odd use + cases which have to call sqlite3_shutdown() as part of their + initialization process, which will unregister the VFS + registered by installOpfsVfs(). If passed a truthy value, the + OPFS VFS is registered as the default VFS, else it is not made + the default. Returns the result of the the + sqlite3_vfs_register() call. + + Design note: the problem of having to re-register things after + a shutdown/initialize pair is more general. How to best plug + that in to the library is unclear. In particular, we cannot + hook in to any C-side calls to sqlite3_initialize(), so we + cannot add an after-initialize callback mechanism. + */ + opfsUtil.registerVfs = (asDefault=false)=>{ + return wasm.exports.sqlite3_vfs_register( + opfsVfs.pointer, asDefault ? 1 : 0 + ); + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + await doDir(opfsUtil.rootDirectory, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const dir = opfsUtil.rootDirectory, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each one. + The argument may be either a callback function or an options object + with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + + TODO: add an option which specifies whether to traverse + depth-first or breadth-first. We currently do depth-first but + an incremental file browsing widget would benefit more from + breadth-first. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: opfsUtil.rootDirectory + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + //TODO to support fiddle and worker1 db upload: + //opfsUtil.createFile = function(absName, content=undefined){...} + //We have sqlite3.wasm.sqlite3_wasm_vfs_create_file() for this + //purpose but its interface and name are still under + //consideration. + + if(sqlite3.oo1){ + const OpfsDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsDb = OpfsDb; + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); + sqlite3.capi.sqlite3_exec(oo1Db, [ + /* Truncate journal mode is faster than delete for + this vfs, per speedtest1. That gap seems to have closed with + Chrome version 108 or 109, but "persist" is very roughly 5-6% + faster than truncate in initial tests. */ + "pragma journal_mode=persist;", + /* + This vfs benefits hugely from cache on moderate/large + speedtest1 --size 50 and --size 100 workloads. We + currently rely on setting a non-default cache size when + building sqlite3.wasm. If that policy changes, the cache + can be set here. + */ + "pragma cache_size=-16384;" + ], 0, 0, 0); + } + ); + }/*extend sqlite3.oo1*/ + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /*Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async worker.*/ + W.postMessage({type: 'opfs-async-init',args: state}); + break; + case 'opfs-async-inited':{ + /*Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin...*/ + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsIoMethods, methods: ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + initS11n(); + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(thisThreadHasOPFS()){ + navigator.storage.getDirectory().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + sqlite3.opfs = opfsUtil; + opfsUtil.rootDirectory = d; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(sqlite3); + }).catch(promiseReject); + }else{ + promiseResolve(sqlite3); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: + promiseReject(e); + error("Unexpected message from the async worker:",data); + break; + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; +installOpfsVfs.defaultProxyUri = + "sqlite3-opfs-async-proxy.js"; +self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + try{ + let proxyJs = installOpfsVfs.defaultProxyUri; + if(sqlite3.scriptInfo.sqlite3Dir){ + installOpfsVfs.defaultProxyUri = + sqlite3.scriptInfo.sqlite3Dir + proxyJs; + //console.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + } + return installOpfsVfs().catch((e)=>{ + console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + }); + }catch(e){ + console.error("installOpfsVfs() exception:",e); + throw e; + } +}); +}/*sqlite3ApiBootstrap.initializers.push()*/); Index: ext/wasm/api/sqlite3-wasm.c ================================================================== --- ext/wasm/api/sqlite3-wasm.c +++ ext/wasm/api/sqlite3-wasm.c @@ -30,25 +30,20 @@ ** Threading and file locking: JS is single-threaded. Each Worker ** thread is a separate instance of the JS engine so can never access ** the same db handle as another thread, thus multi-threading support ** is unnecessary in the library. Because the filesystems are virtual ** and local to a given wasm runtime instance, two Workers can never -** access the same db file at once, with the exception of OPFS. As of -** this writing (2022-09-30), OPFS exclusively locks a file when -** opening it, so two Workers can never open the same OPFS-backed file -** at once. That situation will change if and when lower-level locking -** features are added to OPFS (as is currently planned, per folks -** involved with its development). +** access the same db file at once, with the exception of OPFS. ** -** Summary: except for the case of future OPFS, which supports -** locking, and any similar future filesystems, threading and file -** locking support are unnecessary in the wasm build. +** Summary: except for the case of OPFS, which supports locking using +** its own API, threading and file locking support are unnecessary in +** the wasm build. */ /* ** Undefine any SQLITE_... config flags which we specifically do not -** want undefined. Please keep these alphabetized. +** want defined. Please keep these alphabetized. */ #undef SQLITE_OMIT_DESERIALIZE #undef SQLITE_OMIT_MEMORYDB /* @@ -67,13 +62,21 @@ ** larger cache benefits the larger workloads. Speed differences ** between 2x and nearly 3x have been measured with ample page cache. */ # define SQLITE_DEFAULT_CACHE_SIZE -16384 #endif -#if 0 && !defined(SQLITE_DEFAULT_PAGE_SIZE) -/* TODO: experiment with this. */ -# define SQLITE_DEFAULT_PAGE_SIZE 8192 /*4096*/ +#if !defined(SQLITE_DEFAULT_PAGE_SIZE) +/* +** OPFS performance is improved by approx. 12% with a page size of 8kb +** instead of 4kb. Performance with 16kb is equivalent to 8kb. +** +** Performance difference of kvvfs with a page size of 8kb compared to +** 4kb, as measured by speedtest1 --size 4, is indeterminate: +** measurements are all over the place either way and not +** significantly different. +*/ +# define SQLITE_DEFAULT_PAGE_SIZE 8192 #endif #ifndef SQLITE_DEFAULT_UNIX_VFS # define SQLITE_DEFAULT_UNIX_VFS "unix-none" #endif #undef SQLITE_DQS @@ -94,22 +97,37 @@ # define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 #endif #ifndef SQLITE_ENABLE_FTS4 # define SQLITE_ENABLE_FTS4 1 #endif +#ifndef SQLITE_ENABLE_MATH_FUNCTIONS +# define SQLITE_ENABLE_MATH_FUNCTIONS 1 +#endif #ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC # define SQLITE_ENABLE_OFFSET_SQL_FUNC 1 #endif +#ifndef SQLITE_ENABLE_PREUPDATE_HOOK +# define SQLITE_ENABLE_PREUPDATE_HOOK 1 /*required by session extension*/ +#endif #ifndef SQLITE_ENABLE_RTREE # define SQLITE_ENABLE_RTREE 1 #endif +#ifndef SQLITE_ENABLE_SESSION +# define SQLITE_ENABLE_SESSION 1 +#endif #ifndef SQLITE_ENABLE_STMTVTAB # define SQLITE_ENABLE_STMTVTAB 1 #endif #ifndef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION # define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION #endif + +/**********************************************************************/ +/* SQLITE_M... */ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 0x1fffffff +#endif /**********************************************************************/ /* SQLITE_O... */ #ifndef SQLITE_OMIT_DEPRECATED # define SQLITE_OMIT_DEPRECATED 1 @@ -309,15 +327,17 @@ ** ** Returns err_code. */ SQLITE_WASM_KEEP int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ - if( 0!=zMsg ){ - const int nMsg = sqlite3Strlen30(zMsg); - sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); - }else{ - sqlite3ErrorWithMsg(db, err_code, NULL); + if( db!=0 ){ + if( 0!=zMsg ){ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + }else{ + sqlite3ErrorWithMsg(db, err_code, NULL); + } } return err_code; } #if SQLITE_WASM_TESTS @@ -357,15 +377,15 @@ ** buffer is not large enough for the generated JSON and needs to be ** increased. In debug builds that will trigger an assert(). */ SQLITE_WASM_KEEP const char * sqlite3_wasm_enum_json(void){ - static char aBuffer[1024 * 12] = {0} /* where the JSON goes */; + static char aBuffer[1024 * 20] = {0} /* where the JSON goes */; int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; char * zPos = &aBuffer[1] /* skip first byte for now to help protect - ** against a small race condition */; + ** against a small race condition */; char const * const zEnd = &aBuffer[0] + sizeof(aBuffer) /* one-past-the-end */; if(aBuffer[0]) return aBuffer; /* Leave aBuffer[0] at 0 until the end to help guard against a tiny ** race condition. If this is called twice concurrently, they might ** end up both writing to aBuffer, but they'll both write the same @@ -398,24 +418,150 @@ DefGroup(access){ DefInt(SQLITE_ACCESS_EXISTS); DefInt(SQLITE_ACCESS_READWRITE); DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; } _DefGroup; + + DefGroup(authorizer){ + DefInt(SQLITE_DENY); + DefInt(SQLITE_IGNORE); + DefInt(SQLITE_CREATE_INDEX); + DefInt(SQLITE_CREATE_TABLE); + DefInt(SQLITE_CREATE_TEMP_INDEX); + DefInt(SQLITE_CREATE_TEMP_TABLE); + DefInt(SQLITE_CREATE_TEMP_TRIGGER); + DefInt(SQLITE_CREATE_TEMP_VIEW); + DefInt(SQLITE_CREATE_TRIGGER); + DefInt(SQLITE_CREATE_VIEW); + DefInt(SQLITE_DELETE); + DefInt(SQLITE_DROP_INDEX); + DefInt(SQLITE_DROP_TABLE); + DefInt(SQLITE_DROP_TEMP_INDEX); + DefInt(SQLITE_DROP_TEMP_TABLE); + DefInt(SQLITE_DROP_TEMP_TRIGGER); + DefInt(SQLITE_DROP_TEMP_VIEW); + DefInt(SQLITE_DROP_TRIGGER); + DefInt(SQLITE_DROP_VIEW); + DefInt(SQLITE_INSERT); + DefInt(SQLITE_PRAGMA); + DefInt(SQLITE_READ); + DefInt(SQLITE_SELECT); + DefInt(SQLITE_TRANSACTION); + DefInt(SQLITE_UPDATE); + DefInt(SQLITE_ATTACH); + DefInt(SQLITE_DETACH); + DefInt(SQLITE_ALTER_TABLE); + DefInt(SQLITE_REINDEX); + DefInt(SQLITE_ANALYZE); + DefInt(SQLITE_CREATE_VTABLE); + DefInt(SQLITE_DROP_VTABLE); + DefInt(SQLITE_FUNCTION); + DefInt(SQLITE_SAVEPOINT); + //DefInt(SQLITE_COPY) /* No longer used */; + DefInt(SQLITE_RECURSIVE); + } _DefGroup; DefGroup(blobFinalizers) { /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as ** integers to avoid casting-related warnings. */ out("\"SQLITE_STATIC\":0, \"SQLITE_TRANSIENT\":-1"); } _DefGroup; + + DefGroup(changeset){ + DefInt(SQLITE_CHANGESETSTART_INVERT); + DefInt(SQLITE_CHANGESETAPPLY_NOSAVEPOINT); + DefInt(SQLITE_CHANGESETAPPLY_INVERT); + + DefInt(SQLITE_CHANGESET_DATA); + DefInt(SQLITE_CHANGESET_NOTFOUND); + DefInt(SQLITE_CHANGESET_CONFLICT); + DefInt(SQLITE_CHANGESET_CONSTRAINT); + DefInt(SQLITE_CHANGESET_FOREIGN_KEY); + + DefInt(SQLITE_CHANGESET_OMIT); + DefInt(SQLITE_CHANGESET_REPLACE); + DefInt(SQLITE_CHANGESET_ABORT); + } _DefGroup; + + DefGroup(config){ + DefInt(SQLITE_CONFIG_SINGLETHREAD); + DefInt(SQLITE_CONFIG_MULTITHREAD); + DefInt(SQLITE_CONFIG_SERIALIZED); + DefInt(SQLITE_CONFIG_MALLOC); + DefInt(SQLITE_CONFIG_GETMALLOC); + DefInt(SQLITE_CONFIG_SCRATCH); + DefInt(SQLITE_CONFIG_PAGECACHE); + DefInt(SQLITE_CONFIG_HEAP); + DefInt(SQLITE_CONFIG_MEMSTATUS); + DefInt(SQLITE_CONFIG_MUTEX); + DefInt(SQLITE_CONFIG_GETMUTEX); +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ + DefInt(SQLITE_CONFIG_LOOKASIDE); + DefInt(SQLITE_CONFIG_PCACHE); + DefInt(SQLITE_CONFIG_GETPCACHE); + DefInt(SQLITE_CONFIG_LOG); + DefInt(SQLITE_CONFIG_URI); + DefInt(SQLITE_CONFIG_PCACHE2); + DefInt(SQLITE_CONFIG_GETPCACHE2); + DefInt(SQLITE_CONFIG_COVERING_INDEX_SCAN); + DefInt(SQLITE_CONFIG_SQLLOG); + DefInt(SQLITE_CONFIG_MMAP_SIZE); + DefInt(SQLITE_CONFIG_WIN32_HEAPSIZE); + DefInt(SQLITE_CONFIG_PCACHE_HDRSZ); + DefInt(SQLITE_CONFIG_PMASZ); + DefInt(SQLITE_CONFIG_STMTJRNL_SPILL); + DefInt(SQLITE_CONFIG_SMALL_MALLOC); + DefInt(SQLITE_CONFIG_SORTERREF_SIZE); + DefInt(SQLITE_CONFIG_MEMDB_MAXSIZE); + } _DefGroup; DefGroup(dataTypes) { DefInt(SQLITE_INTEGER); DefInt(SQLITE_FLOAT); DefInt(SQLITE_TEXT); DefInt(SQLITE_BLOB); DefInt(SQLITE_NULL); } _DefGroup; + + DefGroup(dbConfig){ + DefInt(SQLITE_DBCONFIG_MAINDBNAME); + DefInt(SQLITE_DBCONFIG_LOOKASIDE); + DefInt(SQLITE_DBCONFIG_ENABLE_FKEY); + DefInt(SQLITE_DBCONFIG_ENABLE_TRIGGER); + DefInt(SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER); + DefInt(SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION); + DefInt(SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE); + DefInt(SQLITE_DBCONFIG_ENABLE_QPSG); + DefInt(SQLITE_DBCONFIG_TRIGGER_EQP); + DefInt(SQLITE_DBCONFIG_RESET_DATABASE); + DefInt(SQLITE_DBCONFIG_DEFENSIVE); + DefInt(SQLITE_DBCONFIG_WRITABLE_SCHEMA); + DefInt(SQLITE_DBCONFIG_LEGACY_ALTER_TABLE); + DefInt(SQLITE_DBCONFIG_DQS_DML); + DefInt(SQLITE_DBCONFIG_DQS_DDL); + DefInt(SQLITE_DBCONFIG_ENABLE_VIEW); + DefInt(SQLITE_DBCONFIG_LEGACY_FILE_FORMAT); + DefInt(SQLITE_DBCONFIG_TRUSTED_SCHEMA); + DefInt(SQLITE_DBCONFIG_MAX); + } _DefGroup; + + DefGroup(dbStatus){ + DefInt(SQLITE_DBSTATUS_LOOKASIDE_USED); + DefInt(SQLITE_DBSTATUS_CACHE_USED); + DefInt(SQLITE_DBSTATUS_SCHEMA_USED); + DefInt(SQLITE_DBSTATUS_STMT_USED); + DefInt(SQLITE_DBSTATUS_LOOKASIDE_HIT); + DefInt(SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE); + DefInt(SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL); + DefInt(SQLITE_DBSTATUS_CACHE_HIT); + DefInt(SQLITE_DBSTATUS_CACHE_MISS); + DefInt(SQLITE_DBSTATUS_CACHE_WRITE); + DefInt(SQLITE_DBSTATUS_DEFERRED_FKS); + DefInt(SQLITE_DBSTATUS_CACHE_USED_SHARED); + DefInt(SQLITE_DBSTATUS_CACHE_SPILL); + DefInt(SQLITE_DBSTATUS_MAX); + } _DefGroup; DefGroup(encodings) { /* Noting that the wasm binding only aims to support UTF-8. */ DefInt(SQLITE_UTF8); DefInt(SQLITE_UTF16LE); @@ -491,10 +637,38 @@ DefInt(SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN); DefInt(SQLITE_IOCAP_POWERSAFE_OVERWRITE); DefInt(SQLITE_IOCAP_IMMUTABLE); DefInt(SQLITE_IOCAP_BATCH_ATOMIC); } _DefGroup; + + DefGroup(limits) { + DefInt(SQLITE_MAX_ALLOCATION_SIZE); + DefInt(SQLITE_LIMIT_LENGTH); + DefInt(SQLITE_MAX_LENGTH); + DefInt(SQLITE_LIMIT_SQL_LENGTH); + DefInt(SQLITE_MAX_SQL_LENGTH); + DefInt(SQLITE_LIMIT_COLUMN); + DefInt(SQLITE_MAX_COLUMN); + DefInt(SQLITE_LIMIT_EXPR_DEPTH); + DefInt(SQLITE_MAX_EXPR_DEPTH); + DefInt(SQLITE_LIMIT_COMPOUND_SELECT); + DefInt(SQLITE_MAX_COMPOUND_SELECT); + DefInt(SQLITE_LIMIT_VDBE_OP); + DefInt(SQLITE_MAX_VDBE_OP); + DefInt(SQLITE_LIMIT_FUNCTION_ARG); + DefInt(SQLITE_MAX_FUNCTION_ARG); + DefInt(SQLITE_LIMIT_ATTACHED); + DefInt(SQLITE_MAX_ATTACHED); + DefInt(SQLITE_LIMIT_LIKE_PATTERN_LENGTH); + DefInt(SQLITE_MAX_LIKE_PATTERN_LENGTH); + DefInt(SQLITE_LIMIT_VARIABLE_NUMBER); + DefInt(SQLITE_MAX_VARIABLE_NUMBER); + DefInt(SQLITE_LIMIT_TRIGGER_DEPTH); + DefInt(SQLITE_MAX_TRIGGER_DEPTH); + DefInt(SQLITE_LIMIT_WORKER_THREADS); + DefInt(SQLITE_MAX_WORKER_THREADS); + } _DefGroup; DefGroup(openFlags) { /* Noting that not all of these will have any effect in ** WASM-space. */ DefInt(SQLITE_OPEN_READONLY); @@ -642,10 +816,40 @@ DefInt(SQLITE_DESERIALIZE_FREEONCLOSE); DefInt(SQLITE_DESERIALIZE_READONLY); DefInt(SQLITE_DESERIALIZE_RESIZEABLE); } _DefGroup; + DefGroup(session){ + DefInt(SQLITE_SESSION_CONFIG_STRMSIZE); + DefInt(SQLITE_SESSION_OBJCONFIG_SIZE); + } _DefGroup; + + DefGroup(sqlite3Status){ + DefInt(SQLITE_STATUS_MEMORY_USED); + DefInt(SQLITE_STATUS_PAGECACHE_USED); + DefInt(SQLITE_STATUS_PAGECACHE_OVERFLOW); + //DefInt(SQLITE_STATUS_SCRATCH_USED) /* NOT USED */; + //DefInt(SQLITE_STATUS_SCRATCH_OVERFLOW) /* NOT USED */; + DefInt(SQLITE_STATUS_MALLOC_SIZE); + DefInt(SQLITE_STATUS_PARSER_STACK); + DefInt(SQLITE_STATUS_PAGECACHE_SIZE); + //DefInt(SQLITE_STATUS_SCRATCH_SIZE) /* NOT USED */; + DefInt(SQLITE_STATUS_MALLOC_COUNT); + } _DefGroup; + + DefGroup(stmtStatus){ + DefInt(SQLITE_STMTSTATUS_FULLSCAN_STEP); + DefInt(SQLITE_STMTSTATUS_SORT); + DefInt(SQLITE_STMTSTATUS_AUTOINDEX); + DefInt(SQLITE_STMTSTATUS_VM_STEP); + DefInt(SQLITE_STMTSTATUS_REPREPARE); + DefInt(SQLITE_STMTSTATUS_RUN); + DefInt(SQLITE_STMTSTATUS_FILTER_MISS); + DefInt(SQLITE_STMTSTATUS_FILTER_HIT); + DefInt(SQLITE_STMTSTATUS_MEMUSED); + } _DefGroup; + DefGroup(syncFlags) { DefInt(SQLITE_SYNC_NORMAL); DefInt(SQLITE_SYNC_FULL); DefInt(SQLITE_SYNC_DATAONLY); } _DefGroup; @@ -654,10 +858,16 @@ DefInt(SQLITE_TRACE_STMT); DefInt(SQLITE_TRACE_PROFILE); DefInt(SQLITE_TRACE_ROW); DefInt(SQLITE_TRACE_CLOSE); } _DefGroup; + + DefGroup(txnState){ + DefInt(SQLITE_TXN_NONE); + DefInt(SQLITE_TXN_READ); + DefInt(SQLITE_TXN_WRITE); + } _DefGroup; DefGroup(udfFlags) { DefInt(SQLITE_DETERMINISTIC); DefInt(SQLITE_DIRECTONLY); DefInt(SQLITE_INNOCUOUS); @@ -666,11 +876,40 @@ DefGroup(version) { DefInt(SQLITE_VERSION_NUMBER); DefStr(SQLITE_VERSION); DefStr(SQLITE_SOURCE_ID); } _DefGroup; - + + DefGroup(vtab) { + DefInt(SQLITE_INDEX_SCAN_UNIQUE); + DefInt(SQLITE_INDEX_CONSTRAINT_EQ); + DefInt(SQLITE_INDEX_CONSTRAINT_GT); + DefInt(SQLITE_INDEX_CONSTRAINT_LE); + DefInt(SQLITE_INDEX_CONSTRAINT_LT); + DefInt(SQLITE_INDEX_CONSTRAINT_GE); + DefInt(SQLITE_INDEX_CONSTRAINT_MATCH); + DefInt(SQLITE_INDEX_CONSTRAINT_LIKE); + DefInt(SQLITE_INDEX_CONSTRAINT_GLOB); + DefInt(SQLITE_INDEX_CONSTRAINT_REGEXP); + DefInt(SQLITE_INDEX_CONSTRAINT_NE); + DefInt(SQLITE_INDEX_CONSTRAINT_ISNOT); + DefInt(SQLITE_INDEX_CONSTRAINT_ISNOTNULL); + DefInt(SQLITE_INDEX_CONSTRAINT_ISNULL); + DefInt(SQLITE_INDEX_CONSTRAINT_IS); + DefInt(SQLITE_INDEX_CONSTRAINT_LIMIT); + DefInt(SQLITE_INDEX_CONSTRAINT_OFFSET); + DefInt(SQLITE_INDEX_CONSTRAINT_FUNCTION); + DefInt(SQLITE_VTAB_CONSTRAINT_SUPPORT); + DefInt(SQLITE_VTAB_INNOCUOUS); + DefInt(SQLITE_VTAB_DIRECTONLY); + DefInt(SQLITE_ROLLBACK); + //DefInt(SQLITE_IGNORE); // Also used by sqlite3_authorizer() callback + DefInt(SQLITE_FAIL); + //DefInt(SQLITE_ABORT); // Also an error code + DefInt(SQLITE_REPLACE); + } _DefGroup; + #undef DefGroup #undef DefStr #undef DefInt #undef _DefGroup @@ -693,12 +932,12 @@ */ /** Macros for emitting StructBinder description. */ #define StructBinder__(TYPE) \ n = 0; \ - outf("%s{", (nStruct++ ? ", " : "")); \ - out("\"name\": \"" # TYPE "\","); \ + outf("%s{", (nStruct++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ out(",\"members\": {"); #define StructBinder_(T) StructBinder__(T) /** ^^^ indirection needed to expand CurrentStruct */ #define StructBinder StructBinder_(CurrentStruct) @@ -714,82 +953,204 @@ nStruct = 0; out(", \"structs\": ["); { #define CurrentStruct sqlite3_vfs StructBinder { - M(iVersion,"i"); - M(szOsFile,"i"); - M(mxPathname,"i"); - M(pNext,"p"); - M(zName,"s"); - M(pAppData,"p"); - M(xOpen,"i(pppip)"); - M(xDelete,"i(ppi)"); - M(xAccess,"i(ppip)"); - M(xFullPathname,"i(ppip)"); - M(xDlOpen,"p(pp)"); - M(xDlError,"p(pip)"); - M(xDlSym,"p()"); - M(xDlClose,"v(pp)"); - M(xRandomness,"i(pip)"); - M(xSleep,"i(pi)"); - M(xCurrentTime,"i(pp)"); - M(xGetLastError,"i(pip)"); - M(xCurrentTimeInt64,"i(pp)"); - M(xSetSystemCall,"i(ppp)"); - M(xGetSystemCall,"p(pp)"); - M(xNextSystemCall,"p(pp)"); + M(iVersion, "i"); + M(szOsFile, "i"); + M(mxPathname, "i"); + M(pNext, "p"); + M(zName, "s"); + M(pAppData, "p"); + M(xOpen, "i(pppip)"); + M(xDelete, "i(ppi)"); + M(xAccess, "i(ppip)"); + M(xFullPathname, "i(ppip)"); + M(xDlOpen, "p(pp)"); + M(xDlError, "p(pip)"); + M(xDlSym, "p()"); + M(xDlClose, "v(pp)"); + M(xRandomness, "i(pip)"); + M(xSleep, "i(pi)"); + M(xCurrentTime, "i(pp)"); + M(xGetLastError, "i(pip)"); + M(xCurrentTimeInt64, "i(pp)"); + M(xSetSystemCall, "i(ppp)"); + M(xGetSystemCall, "p(pp)"); + M(xNextSystemCall, "p(pp)"); } _StructBinder; #undef CurrentStruct #define CurrentStruct sqlite3_io_methods StructBinder { - M(iVersion,"i"); - M(xClose,"i(p)"); - M(xRead,"i(ppij)"); - M(xWrite,"i(ppij)"); - M(xTruncate,"i(pj)"); - M(xSync,"i(pi)"); - M(xFileSize,"i(pp)"); - M(xLock,"i(pi)"); - M(xUnlock,"i(pi)"); - M(xCheckReservedLock,"i(pp)"); - M(xFileControl,"i(pip)"); - M(xSectorSize,"i(p)"); - M(xDeviceCharacteristics,"i(p)"); - M(xShmMap,"i(piiip)"); - M(xShmLock,"i(piii)"); - M(xShmBarrier,"v(p)"); - M(xShmUnmap,"i(pi)"); - M(xFetch,"i(pjip)"); - M(xUnfetch,"i(pjp)"); + M(iVersion, "i"); + M(xClose, "i(p)"); + M(xRead, "i(ppij)"); + M(xWrite, "i(ppij)"); + M(xTruncate, "i(pj)"); + M(xSync, "i(pi)"); + M(xFileSize, "i(pp)"); + M(xLock, "i(pi)"); + M(xUnlock, "i(pi)"); + M(xCheckReservedLock, "i(pp)"); + M(xFileControl, "i(pip)"); + M(xSectorSize, "i(p)"); + M(xDeviceCharacteristics, "i(p)"); + M(xShmMap, "i(piiip)"); + M(xShmLock, "i(piii)"); + M(xShmBarrier, "v(p)"); + M(xShmUnmap, "i(pi)"); + M(xFetch, "i(pjip)"); + M(xUnfetch, "i(pjp)"); } _StructBinder; #undef CurrentStruct #define CurrentStruct sqlite3_file StructBinder { - M(pMethods,"p"); + M(pMethods, "p"); } _StructBinder; #undef CurrentStruct #define CurrentStruct sqlite3_kvvfs_methods StructBinder { - M(xRead,"i(sspi)"); - M(xWrite,"i(sss)"); - M(xDelete,"i(ss)"); - M(nKeySize,"i"); + M(xRead, "i(sspi)"); + M(xWrite, "i(sss)"); + M(xDelete, "i(ss)"); + M(nKeySize, "i"); + } _StructBinder; +#undef CurrentStruct + + +#define CurrentStruct sqlite3_vtab + StructBinder { + M(pModule, "p"); + M(nRef, "i"); + M(zErrMsg, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_vtab_cursor + StructBinder { + M(pVtab, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_module + StructBinder { + M(iVersion, "i"); + M(xCreate, "i(ppippp)"); + M(xConnect, "i(ppippp)"); + M(xBestIndex, "i(pp)"); + M(xDisconnect, "i(p)"); + M(xDestroy, "i(p)"); + M(xOpen, "i(pp)"); + M(xClose, "i(p)"); + M(xFilter, "i(pisip)"); + M(xNext, "i(p)"); + M(xEof, "i(p)"); + M(xColumn, "i(ppi)"); + M(xRowid, "i(pp)"); + M(xUpdate, "i(pipp)"); + M(xBegin, "i(p)"); + M(xSync, "i(p)"); + M(xCommit, "i(p)"); + M(xRollback, "i(p)"); + M(xFindFunction, "i(pispp)"); + M(xRename, "i(ps)"); + // ^^^ v1. v2+ follows... + M(xSavepoint, "i(pi)"); + M(xRelease, "i(pi)"); + M(xRollbackTo, "i(pi)"); + // ^^^ v2. v3+ follows... + M(xShadowName, "i(s)"); + } _StructBinder; +#undef CurrentStruct + + /** + ** Workaround: in order to map the various inner structs from + ** sqlite3_index_info, we have to uplift those into constructs we + ** can access by type name. These structs _must_ match their + ** in-sqlite3_index_info counterparts byte for byte. + */ + typedef struct { + int iColumn; + unsigned char op; + unsigned char usable; + int iTermOffset; + } sqlite3_index_constraint; + typedef struct { + int iColumn; + unsigned char desc; + } sqlite3_index_orderby; + typedef struct { + int argvIndex; + unsigned char omit; + } sqlite3_index_constraint_usage; + { /* Validate that the above struct sizeof()s match + ** expectations. We could improve upon this by + ** checking the offsetof() for each member. */ + const sqlite3_index_info siiCheck; +#define IndexSzCheck(T,M) \ + (sizeof(T) == sizeof(*siiCheck.M)) + if(!IndexSzCheck(sqlite3_index_constraint,aConstraint) + || !IndexSzCheck(sqlite3_index_orderby,aOrderBy) + || !IndexSzCheck(sqlite3_index_constraint_usage,aConstraintUsage)){ + assert(!"sizeof mismatch in sqlite3_index_... struct(s)"); + return 0; + } +#undef IndexSzCheck + } + +#define CurrentStruct sqlite3_index_constraint + StructBinder { + M(iColumn, "i"); + M(op, "C"); + M(usable, "C"); + M(iTermOffset, "i"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_index_orderby + StructBinder { + M(iColumn, "i"); + M(desc, "C"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_index_constraint_usage + StructBinder { + M(argvIndex, "i"); + M(omit, "C"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_index_info + StructBinder { + M(nConstraint, "i"); + M(aConstraint, "p"); + M(nOrderBy, "i"); + M(aOrderBy, "p"); + M(aConstraintUsage, "p"); + M(idxNum, "i"); + M(idxStr, "p"); + M(needToFreeIdxStr, "i"); + M(orderByConsumed, "i"); + M(estimatedCost, "d"); + M(estimatedRows, "j"); + M(idxFlags, "i"); + M(colUsed, "j"); } _StructBinder; #undef CurrentStruct #if SQLITE_WASM_TESTS #define CurrentStruct WasmTestStruct StructBinder { - M(v4,"i"); - M(cstr,"s"); - M(ppV,"p"); - M(v8,"j"); - M(xFunc,"v(p)"); + M(v4, "i"); + M(cstr, "s"); + M(ppV, "p"); + M(v8, "j"); + M(xFunc, "v(p)"); } _StructBinder; #undef CurrentStruct #endif } out( "]"/*structs*/); @@ -818,11 +1179,11 @@ ** zName is NULL, no default VFS is found, or it has no xDelete ** method, SQLITE_MISUSE is returned, else the result of the xDelete() ** call is returned. */ SQLITE_WASM_KEEP -int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char * zName){ +int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); if( zName && pVfs && pVfs->xDelete ){ rc = pVfs->xDelete(pVfs, zName, 1); } @@ -849,27 +1210,37 @@ ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** ** This function resets the given db pointer's database as described at ** -** https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase +** https://sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase +** +** But beware: virtual tables destroyed that way do not have their +** xDestroy() called, so will leak if they require that function for +** proper cleanup. ** ** Returns 0 on success, an SQLITE_xxx code on error. Returns ** SQLITE_MISUSE if pDb is NULL. */ SQLITE_WASM_KEEP -int sqlite3_wasm_db_reset(sqlite3*pDb){ +int sqlite3_wasm_db_reset(sqlite3 *pDb){ int rc = SQLITE_MISUSE; if( pDb ){ + sqlite3_table_column_metadata(pDb, "main", 0, 0, 0, 0, 0, 0, 0); rc = sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); - if( 0==rc ) rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); - sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + if( 0==rc ){ + rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + } } return rc; } /* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** ** Uses the given database's VFS xRead to stream the db file's ** contents out to the given callback. The callback gets a single ** chunk of size n (its 2nd argument) on each call and must return 0 ** on success, non-0 on error. This function returns 0 on success, ** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 @@ -905,38 +1276,43 @@ else if(0 == nSize % 1024) nBuf = 1024; else nBuf = 512; } for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos); - if(SQLITE_IOERR_SHORT_READ == rc){ + if( SQLITE_IOERR_SHORT_READ == rc ){ rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; } if( 0==rc ) rc = xCallback(buf, nBuf); } return rc; } /* -** A proxy for sqlite3_serialize() which serializes the "main" schema +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** A proxy for sqlite3_serialize() which serializes the schema zSchema ** of pDb, placing the serialized output in pOut and nOut. nOut may be -** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If -** allocation of the serialized copy fails, SQLITE_NOMEM is returned. -** On success, 0 is returned and `*pOut` will contain a pointer to the -** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the -** database has no contiguous memory representation, in which case -** `*pOut` will be NULL but 0 will be returned. +** NULL. If zSchema is NULL then "main" is assumed. If pDb or pOut are +** NULL then SQLITE_MISUSE is returned. If allocation of the +** serialized copy fails, SQLITE_NOMEM is returned. On success, 0 is +** returned and `*pOut` will contain a pointer to the memory unless +** mFlags includes SQLITE_SERIALIZE_NOCOPY and the database has no +** contiguous memory representation, in which case `*pOut` will be +** NULL but 0 will be returned. ** ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ SQLITE_WASM_KEEP -int sqlite3_wasm_db_serialize( sqlite3 *pDb, unsigned char **pOut, +int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, + unsigned char **pOut, sqlite3_int64 *nOut, unsigned int mFlags ){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; - if(nOut) *nOut = 0; - z = sqlite3_serialize(pDb, "main", nOut, mFlags); + if( nOut ) *nOut = 0; + z = sqlite3_serialize(pDb, zSchema ? zSchema : "main", nOut, mFlags); if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ *pOut = z; return 0; }else{ return SQLITE_NOMEM; @@ -946,17 +1322,20 @@ /* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** ** Creates a new file using the I/O API of the given VFS, containing -** the given number of bytes of the given data. If the file exists, -** it is truncated to the given length and populated with the given +** the given number of bytes of the given data. If the file exists, it +** is truncated to the given length and populated with the given ** data. ** ** This function exists so that we can implement the equivalent of ** Emscripten's FS.createDataFile() in a VFS-agnostic way. This ** functionality is intended for use in uploading database files. +** +** Not all VFSes support this functionality, e.g. the "kvvfs" does +** not. ** ** If pVfs is NULL, sqlite3_vfs_find(0) is used. ** ** If zFile is NULL, pVfs is NULL (and sqlite3_vfs_find(0) returns ** NULL), or nData is negative, SQLITE_MISUSE are returned. @@ -965,14 +1344,11 @@ ** with the fist nData bytes of pData. If pData is NULL, the file is ** created and/or truncated to nData bytes. ** ** Whether or not directory components of zFilename are created ** automatically or not is unspecified: that detail is left to the -** VFS. The "opfs" VFS, for example, create them. -** -** Not all VFSes support this functionality, e.g. the "kvvfs" does -** not. +** VFS. The "opfs" VFS, for example, creates them. ** ** If an error happens while populating or truncating the file, the ** target file will be deleted (if needed) if this function created ** it. If this function did not create it, it is not deleted but may ** be left in an undefined state. @@ -1000,15 +1376,22 @@ const int blockSize = 512 /* Because we are using pFile->pMethods->xWrite() for writing, and ** it may have a buffer limit related to sqlite3's pager size, we ** conservatively write in 512-byte blocks (smallest page ** size). */; - + //fprintf(stderr, "pVfs=%p, zFilename=%s, nData=%d\n", pVfs, zFilename, nData); if( !pVfs ) pVfs = sqlite3_vfs_find(0); if( !pVfs || !zFilename || nData<0 ) return SQLITE_MISUSE; pVfs->xAccess(pVfs, zFilename, SQLITE_ACCESS_EXISTS, &fileExisted); rc = sqlite3OsOpenMalloc(pVfs, zFilename, &pFile, openFlags, &flagsOut); +#if 0 +# define RC fprintf(stderr,"create_file(%s,%s) @%d rc=%d\n", \ + pVfs->zName, zFilename, __LINE__, rc); +#else +# define RC +#endif + RC; if(rc) return rc; pIo = pFile->pMethods; if( pIo->xLock ) { /* We need xLock() in order to accommodate the OPFS VFS, as it ** obtains a writeable handle via the lock operation and releases @@ -1015,30 +1398,41 @@ ** it in xUnlock(). If we don't do those here, we have to add code ** to the VFS to account check whether it was locked before ** xFileSize(), xTruncate(), and the like, and release the lock ** only if it was unlocked when the op was started. */ rc = pIo->xLock(pFile, SQLITE_LOCK_EXCLUSIVE); + RC; doUnlock = 0==rc; } - if( 0==rc) rc = pIo->xTruncate(pFile, nData); + if( 0==rc ){ + rc = pIo->xTruncate(pFile, nData); + RC; + } if( 0==rc && 0!=pData && nData>0 ){ while( 0==rc && nData>0 ){ const int n = nData>=blockSize ? blockSize : nData; rc = pIo->xWrite(pFile, pPos, n, (sqlite3_int64)(pPos - pData)); + RC; nData -= n; pPos += n; } if( 0==rc && nData>0 ){ assert( nDataxWrite(pFile, pPos, nData, (sqlite3_int64)(pPos - pData)); + rc = pIo->xWrite(pFile, pPos, nData, + (sqlite3_int64)(pPos - pData)); + RC; } } - if( pIo->xUnlock && doUnlock!=0 ) pIo->xUnlock(pFile, SQLITE_LOCK_NONE); + if( pIo->xUnlock && doUnlock!=0 ){ + pIo->xUnlock(pFile, SQLITE_LOCK_NONE); + } pIo->xClose(pFile); if( rc!=0 && 0==fileExisted ){ pVfs->xDelete(pVfs, zFilename, 1); } + RC; +#undef RC return rc; } /* ** This function is NOT part of the sqlite3 public API. It is strictly @@ -1071,10 +1465,133 @@ */ SQLITE_WASM_KEEP sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This is a proxy for the variadic sqlite3_vtab_config() which passes +** its argument on, or not, to sqlite3_vtab_config(), depending on the +** value of its 2nd argument. Returns the result of +** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a +** valid value. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ + switch(op){ + case SQLITE_VTAB_DIRECTONLY: + case SQLITE_VTAB_INNOCUOUS: + return sqlite3_vtab_config(pDb, op); + case SQLITE_VTAB_CONSTRAINT_SUPPORT: + return sqlite3_vtab_config(pDb, op, arg); + default: + return SQLITE_MISUSE; + } +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Wrapper for the variants of sqlite3_db_config() which take +** (int,int*) variadic args. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ + switch(op){ + case SQLITE_DBCONFIG_ENABLE_FKEY: + case SQLITE_DBCONFIG_ENABLE_TRIGGER: + case SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: + case SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: + case SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: + case SQLITE_DBCONFIG_ENABLE_QPSG: + case SQLITE_DBCONFIG_TRIGGER_EQP: + case SQLITE_DBCONFIG_RESET_DATABASE: + case SQLITE_DBCONFIG_DEFENSIVE: + case SQLITE_DBCONFIG_WRITABLE_SCHEMA: + case SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: + case SQLITE_DBCONFIG_DQS_DML: + case SQLITE_DBCONFIG_DQS_DDL: + case SQLITE_DBCONFIG_ENABLE_VIEW: + case SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: + case SQLITE_DBCONFIG_TRUSTED_SCHEMA: + return sqlite3_db_config(pDb, op, arg1, pArg2); + default: return SQLITE_MISUSE; + } +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Wrapper for the variants of sqlite3_db_config() which take +** (void*,int,int) variadic args. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ + switch(op){ + case SQLITE_DBCONFIG_LOOKASIDE: + return sqlite3_db_config(pDb, op, pArg1, arg2, arg3); + default: return SQLITE_MISUSE; + } +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Wrapper for the variants of sqlite3_db_config() which take +** (const char *) variadic args. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ + switch(op){ + case SQLITE_DBCONFIG_MAINDBNAME: + return sqlite3_db_config(pDb, op, zArg); + default: return SQLITE_MISUSE; + } +} + + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Binding for combinations of sqlite3_config() arguments which take +** a single integer argument. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_config_i(int op, int arg){ + return sqlite3_config(op, arg); +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Binding for combinations of sqlite3_config() arguments which take +** two int arguments. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ + return sqlite3_config(op, arg1, arg2); +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Binding for combinations of sqlite3_config() arguments which take +** a single i64 argument. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_config_j(int op, sqlite3_int64 arg){ + return sqlite3_config(op, arg); +} #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) #include /* @@ -1130,10 +1647,15 @@ SQLITE_WASM_KEEP int sqlite3_wasm_test_intptr(int * p){ return *p = *p * 2; } + +SQLITE_WASM_KEEP +void * sqlite3_wasm_test_voidptr(void * p){ + return p; +} SQLITE_WASM_KEEP int64_t sqlite3_wasm_test_int64_max(void){ return (int64_t)0x7fffffffffffffff; } @@ -1164,18 +1686,18 @@ SQLITE_WASM_KEEP void sqlite3_wasm_test_stack_overflow(int recurse){ if(recurse) sqlite3_wasm_test_stack_overflow(recurse); } -/* For testing the 'string-free' whwasmutil.xWrap() conversion. */ +/* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ SQLITE_WASM_KEEP char * sqlite3_wasm_test_str_hello(int fail){ - char * s = fail ? 0 : (char *)malloc(6); + char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ memcpy(s, "hello", 5); s[5] = 0; } return s; } #endif /* SQLITE_WASM_TESTS */ #undef SQLITE_WASM_KEEP Index: ext/wasm/api/sqlite3-worker1.js ================================================================== --- ext/wasm/api/sqlite3-worker1.js +++ ext/wasm/api/sqlite3-worker1.js @@ -39,11 +39,8 @@ theJs = urlParams.get('sqlite3.dir') + '/' + theJs; } //console.warn("worker1 theJs =",theJs); importScripts(theJs); sqlite3InitModule().then((sqlite3)=>{ - if(sqlite3.capi.sqlite3_wasmfs_opfs_dir){ - sqlite3.capi.sqlite3_wasmfs_opfs_dir(); - } sqlite3.initWorker1API(); }); })(); Index: ext/wasm/batch-runner.js ================================================================== --- ext/wasm/batch-runner.js +++ ext/wasm/batch-runner.js @@ -225,24 +225,24 @@ metrics.byteLength = sqlByteLen; let pSql = pSqlBegin; const pSqlEnd = pSqlBegin + sqlByteLen; t = performance.now(); wasm.heap8().set(sql, pSql); - wasm.setMemValue(pSql + sqlByteLen, 0); + wasm.poke(pSql + sqlByteLen, 0); metrics.strcpy = performance.now() - t; let breaker = 0; - while(pSql && wasm.getMemValue(pSql,'i8')){ - wasm.setPtrValue(ppStmt, 0); - wasm.setPtrValue(pzTail, 0); + while(pSql && wasm.peek(pSql,'i8')){ + wasm.pokePtr(ppStmt, 0); + wasm.pokePtr(pzTail, 0); t = performance.now(); let rc = capi.sqlite3_prepare_v3( db.handle, pSql, sqlByteLen, 0, ppStmt, pzTail ); metrics.prepTotal += performance.now() - t; checkSqliteRc(db.handle, rc); - pStmt = wasm.getPtrValue(ppStmt); - pSql = wasm.getPtrValue(pzTail); + pStmt = wasm.peekPtr(ppStmt); + pSql = wasm.peekPtr(pzTail); sqlByteLen = pSqlEnd - pSql; if(!pStmt) continue/*empty statement*/; ++metrics.stmtCount; t = performance.now(); rc = capi.sqlite3_step(pStmt); @@ -493,11 +493,11 @@ let pDb = 0; try{ const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; const ppDb = wasm.scopedAllocPtr(); const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, null); - pDb = wasm.getPtrValue(ppDb) + pDb = wasm.peekPtr(ppDb) if(rc) toss("sqlite3_open_v2() failed with code",rc); capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0); this.logHtml(dbId,"cache_size =",cacheSize); }catch(e){ if(pDb) capi.sqlite3_close_v2(pDb); ADDED ext/wasm/c-pp.c Index: ext/wasm/c-pp.c ================================================================== --- /dev/null +++ ext/wasm/c-pp.c @@ -0,0 +1,1525 @@ +/* +** 2022-11-12: +** +** In place of a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** +** The C-minus Preprocessor: a truly minimal C-like preprocessor. +** Why? Because C preprocessors _can_ process non-C code but generally make +** quite a mess of it. The purpose of this application is an extremely +** minimal preprocessor with only the most basic functionality of a C +** preprocessor, namely: +** +** - Limited `#if`, where its one argument is a macro name which +** resolves to true if it's defined, false if it's not. Likewise, +** `#ifnot` is the inverse. Includes `#else` and `#elif` and +** `#elifnot`. Such chains are terminated with `#endif`. +** +** - `#define` accepts one or more arguments, the names of +** macros. Each one is implicitly true. +** +** - `#undef` undefine one or more macros. +** +** - `#error` treats the rest of the line as a fatal error message. +** +** - `#include` treats its argument as a filename token (NOT quoted, +** though support for quoting may be added later). Some effort is +** made to prevent recursive inclusion, but that support is both +** somewhat fragile and possibly completely unnecessary. +** +** - `#pragma` is in place for adding "meta-commands", but it does not +** yet have any concrete list of documented commands. +** +* - `#stderr` outputs its file name, line number, and the remaininder +** of that line to stderr. +** +** - `#//` acts as a single-line comment, noting that there must be as +** space after the `//` part because `//` is (despite appearances) +** parsed like a keyword. +** +** Note that "#" above is symbolic. The keyword delimiter is +** configurable and defaults to "##". Define CMPP_DEFAULT_DELIM to a +** string when compiling to define the default at build-time. +** +** This preprocessor does no expansion of content except within the +** bounds of its `#keywords`. +** +** Design note: this code makes use of sqlite3. Though not _strictly_ +** needed in order to implement it, this tool was specifically created +** for potential use with the sqlite3 project's own JavaScript code, +** so there's no reason not to make use of it to do some of the heavy +** lifting. It does not require any cutting-edge sqlite3 features and +** should be usable with any version which supports `WITHOUT ROWID`. +** +** Author(s): +** +** - Stephan Beal +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "sqlite3.h" + +#if defined(_WIN32) || defined(WIN32) +# include +# include +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +#else +# include +#endif + +#ifndef CMPP_DEFAULT_DELIM +#define CMPP_DEFAULT_DELIM "##" +#endif + +#if 1 +# define CMPP_NORETURN __attribute__((noreturn)) +#else +# define CMPP_NORETURN +#endif + +/* Fatally exits the app with the given printf-style message. */ +static CMPP_NORETURN void fatalv(char const *zFmt, va_list); +static CMPP_NORETURN void fatal(char const *zFmt, ...); + +/** Proxy for free(), for symmetry with cmpp_realloc(). */ +static void cmpp_free(void *p); +/** A realloc() proxy which dies fatally on allocation error. */ +static void * cmpp_realloc(void * p, unsigned n); +#if 0 +/** A malloc() proxy which dies fatally on allocation error. */ +static void * cmpp_malloc(unsigned n); +#endif + +/* +** If p is stdin or stderr then this is a no-op, else it is a +** proxy for fclose(). This is a no-op if p is NULL. +*/ +static void FILE_close(FILE *p); +/* +** Works like fopen() but accepts the special name "-" to mean either +** stdin (if zMode indicates a real-only mode) or stdout. Fails +** fatally on error. +*/ +static FILE * FILE_open(char const *zName, const char * zMode); +/* +** Reads the entire contents of the given file, allocating it in a +** buffer which gets assigned to `*pOut`. `*nOut` gets assigned the +** length of the output buffer. Fails fatally on error. +*/ +static void FILE_slurp(FILE *pFile, unsigned char **pOut, + unsigned * nOut); + +/* +** Intended to be passed an sqlite3 result code. If it's non-0 +** then it emits a fatal error message which contains both the +** given string and the sqlite3_errmsg() from the application's +** database instance. +*/ +static void db_affirm_rc(int rc, const char * zMsg); + +/* +** Proxy for sqlite3_str_finish() which fails fatally if that +** routine returns NULL. +*/ +static char * db_str_finish(sqlite3_str *s, int * n); +/* +** Proxy for sqlite3_str_new() which fails fatally if that +** routine returns NULL. +*/ +static sqlite3_str * db_str_new(void); + +/* Proxy for sqlite3_finalize(). */ +static void db_finalize(sqlite3_stmt *pStmt); +/* +** Proxy for sqlite3_step() which fails fatally if the result +** is anything other than SQLITE_ROW or SQLITE_DONE. +*/ +static int db_step(sqlite3_stmt *pStmt); +/* +** Proxy for sqlite3_bind_int() which fails fatally on error. +*/ +static void db_bind_int(sqlite3_stmt *pStmt, int col, int val); +#if 0 +/* +** Proxy for sqlite3_bind_null() which fails fatally on error. +*/ +static void db_bind_null(sqlite3_stmt *pStmt, int col); +#endif +/* +** Proxy for sqlite3_bind_text() which fails fatally on error. +*/ +static void db_bind_text(sqlite3_stmt *pStmt, int col, const char * zStr); +/* +** Proxy for sqlite3_bind_text() which fails fatally on error. +*/ +static void db_bind_textn(sqlite3_stmt *pStmt, int col, const char * zStr, int len); +#if 0 +/* +** Proxy for sqlite3_bind_text() which fails fatally on error. It uses +** sqlite3_str_vappendf() so supports all of its formatting options. +*/ +static void db_bind_textv(sqlite3_stmt *pStmt, int col, const char * zFmt, ...); +#endif +/* +** Proxy for sqlite3_free(), to be passed any memory which is allocated +** by sqlite3_malloc(). +*/ +static void db_free(void *m); +/* +** Adds the given `#define` macro name to the list of macros, ignoring +** any duplicates. Fails fatally on error. +*/ +static void db_define_add(const char * zKey); +/* +** Returns true if the given key is already in the `#define` list, +** else false. Fails fatally on db error. +*/ +static int db_define_has(const char * zName); +/* +** Removes the given `#define` macro name from the list of +** macros. Fails fatally on error. +*/ +static void db_define_rm(const char * zKey); +/* +** Adds the given filename to the list of being-`#include`d files, +** using the given source file name and line number of error reporting +** purposes. If recursion is later detected. +*/ +static void db_including_add(const char * zKey, const char * zSrc, int srcLine); +/* +** Adds the given dir to the list of includes. They are checked in the +** order they are added. +*/ +static void db_include_dir_add(const char * zKey); +/* +** Returns a resolved path of PREFIX+'/'+zKey, where PREFIX is one of +** the `#include` dirs (db_include_dir_add()). If no file match is +** found, NULL is returned. Memory must eventually be passed to +** db_free() to free it. +*/ +static char * db_include_search(const char * zKey); +/* +** Removes the given key from the `#include` list. +*/ +static void db_include_rm(const char * zKey); +/* +** A proxy for sqlite3_prepare() which fails fatally on error. +*/ +static void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...); + +/* +** Opens the given file and processes its contents as c-pp, sending +** all output to the global c-pp output channel. Fails fatally on +** error. +*/ +static void cmpp_process_file(const char * zName); + +/* +** Returns the number newline characters between the given starting +** point and inclusive ending point. Results are undefined if zFrom is +** greater than zTo. +*/ +static unsigned count_lines(unsigned char const * zFrom, + unsigned char const *zTo); + +/* +** Wrapper around a FILE handle. +*/ +struct FileWrapper { + /* File's name. */ + char const *zName; + /* FILE handle. */ + FILE * pFile; + /* Where FileWrapper_slurp() stores the file's contents. */ + unsigned char * zContent; + /* Size of this->zContent, as set by FileWrapper_slurp(). */ + unsigned nContent; +}; +typedef struct FileWrapper FileWrapper; +#define FileWrapper_empty_m {0,0,0,0} +static const FileWrapper FileWrapper_empty = FileWrapper_empty_m; + +/* Proxy for FILE_close(). */ +static void FileWrapper_close(FileWrapper * p); +/* Proxy for FILE_open(). */ +static void FileWrapper_open(FileWrapper * p, const char * zName, const char *zMode); +/* Proxy for FILE_slurp(). */ +static void FileWrapper_slurp(FileWrapper * p); + +/* +** Outputs a printf()-formatted message to stderr. +*/ +static void g_stderr(char const *zFmt, ...); +/* +** Outputs a printf()-formatted message to stderr. +*/ +static void g_stderrv(char const *zFmt, va_list); +#define g_debug(lvl,pfexpr) \ + if(lvl<=g.doDebug) g_stderr("%s @ %s:%d: ",g.zArgv0,__FILE__,__LINE__); \ + if(lvl<=g.doDebug) g_stderr pfexpr + +void fatalv(char const *zFmt, va_list va){ + if(zFmt && *zFmt){ + vfprintf(stderr, zFmt, va); + } + fputc('\n', stderr); + exit(1); +} + +void fatal(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + fatalv(zFmt, va); + va_end(va); +} + +void cmpp_free(void *p){ + free(p); +} + +void * cmpp_realloc(void * p, unsigned n){ + void * const rc = realloc(p, n); + if(!rc) fatal("realloc(P,%u) failed", n); + return rc; +} + +#if 0 +void * cmpp_malloc(unsigned n){ + void * const rc = malloc(n); + if(!rc) fatal("malloc(%u) failed", n); + return rc; +} +#endif + +FILE * FILE_open(char const *zName, const char * zMode){ + FILE * p; + if('-'==zName[0] && 0==zName[1]){ + p = strstr(zMode,"w") ? stdout : stdin; + }else{ + p = fopen(zName, zMode); + if(!p) fatal("Cannot open file [%s] with mode [%s]", zName, zMode); + } + return p; +} + +void FILE_close(FILE *p){ + if(p && p!=stdout && p!=stderr){ + fclose(p); + } +} + +void FILE_slurp(FILE *pFile, unsigned char **pOut, + unsigned * nOut){ + unsigned char zBuf[1024 * 8]; + unsigned char * pDest = 0; + unsigned nAlloc = 0; + unsigned nOff = 0; + /* Note that this needs to be able to work on non-seekable streams, + ** thus we read in chunks instead of doing a single alloc and + ** filling it in one go. */ + while( !feof(pFile) ){ + size_t const n = fread(zBuf, 1, sizeof(zBuf), pFile); + if(n>0){ + if(nAlloc < nOff + n + 1){ + nAlloc = nOff + n + 1; + pDest = cmpp_realloc(pDest, nAlloc); + } + memcpy(pDest + nOff, zBuf, n); + nOff += n; + } + } + if(pDest) pDest[nOff] = 0; + *pOut = pDest; + *nOut = nOff; +} + +void FileWrapper_close(FileWrapper * p){ + if(p->pFile) FILE_close(p->pFile); + if(p->zContent) cmpp_free(p->zContent); + *p = FileWrapper_empty; +} + +void FileWrapper_open(FileWrapper * p, const char * zName, + const char * zMode){ + FileWrapper_close(p); + p->pFile = FILE_open(zName, zMode); + p->zName = zName; +} + +void FileWrapper_slurp(FileWrapper * p){ + assert(!p->zContent); + assert(p->pFile); + FILE_slurp(p->pFile, &p->zContent, &p->nContent); +} + +unsigned count_lines(unsigned char const * zFrom, unsigned char const *zTo){ + unsigned ln = 0; + unsigned char const *zPos = zFrom; + assert(zFrom && zTo); + assert(zFrom <= zTo); + for(; zPos < zTo; ++zPos){ + switch(*zPos){ + case (unsigned)'\n': ++ln; break; + default: break; + } + } + return ln; +} + +enum CmppParseState { +TS_Start = 1, +TS_If, +TS_IfPassed, +TS_Else, +TS_Error +}; +typedef enum CmppParseState CmppParseState; +enum CmppTokenType { +TT_Invalid = 0, +TT_Comment, +TT_Define, +TT_Elif, +TT_ElifNot, +TT_Else, +TT_EndIf, +TT_Error, +TT_If, +TT_IfNot, +TT_Include, +TT_Line, +TT_Pragma, +TT_Stderr, +TT_Undef +}; +typedef enum CmppTokenType CmppTokenType; + +struct CmppToken { + CmppTokenType ttype; + /* Line number of this token in the source file. */ + unsigned lineNo; + /* Start of the token. */ + unsigned char const * zBegin; + /* One-past-the-end byte of the token. */ + unsigned char const * zEnd; +}; +typedef struct CmppToken CmppToken; +#define CmppToken_empty_m {TT_Invalid,0,0,0} +static const CmppToken CmppToken_empty = CmppToken_empty_m; + +/* +** CmppLevel represents one "level" of tokenization, starting at the +** top of the main input, incrementing once for each level of `#if`, +** and decrementing for each `#endif`. +*/ +typedef struct CmppLevel CmppLevel; +struct CmppLevel { + unsigned short flags; + /* + ** Used for controlling which parts of an if/elif/...endif chain + ** should get output. + */ + unsigned short skipLevel; + /* The token which started this level (an 'if' or 'ifnot'). */ + CmppToken token; + CmppParseState pstate; +}; +#define CmppLevel_empty_m {0U,0U,CmppToken_empty_m,TS_Start} +static const CmppLevel CmppLevel_empty = CmppLevel_empty_m; +enum CmppLevel_Flags { +/* Max depth of nested `#if` constructs in a single tokenizer. */ +CmppLevel_Max = 10, +/* Max number of keyword arguments. */ +CmppArgs_Max = 10, +/* Flag indicating that output for a CmpLevel should be elided. */ +CmppLevel_F_ELIDE = 0x01, +/* +** Mask of CmppLevel::flags which are inherited when CmppLevel_push() +** is used. +*/ +CmppLevel_F_INHERIT_MASK = 0x01 +}; + +typedef struct CmppTokenizer CmppTokenizer; +typedef struct CmppKeyword CmppKeyword; +typedef void (*cmpp_keyword_f)(CmppKeyword const * pKw, CmppTokenizer * t); +struct CmppKeyword { + const char *zName; + unsigned nName; + int bTokenize; + CmppTokenType ttype; + cmpp_keyword_f xCall; +}; + +static CmppKeyword const * CmppKeyword_search(const char *zName); +static void cmpp_process_keyword(CmppTokenizer * const t); + +/* +** Tokenizer for c-pp input files. +*/ +struct CmppTokenizer { + const char * zName; /* Input (file) name for error reporting */ + unsigned const char * zBegin; /* start of input */ + unsigned const char * zEnd; /* one-after-the-end of input */ + unsigned const char * zAnchor; /* start of input or end point of + previous token */ + unsigned const char * zPos; /* current position */ + unsigned int lineNo; /* line # of current pos */ + CmppParseState pstate; + CmppToken token; /* current token result */ + struct { + unsigned ndx; + CmppLevel stack[CmppLevel_Max]; + } level; + /* Args for use in cmpp_keyword_f() impls. */ + struct { + CmppKeyword const * pKw; + int argc; + const unsigned char * argv[CmppArgs_Max]; + unsigned char lineBuf[1024]; + } args; +}; +#define CT_level(t) (t)->level.stack[(t)->level.ndx] +#define CT_pstate(t) CT_level(t).pstate +#define CT_skipLevel(t) CT_level(t).skipLevel +#define CLvl_skip(lvl) ((lvl)->skipLevel || ((lvl)->flags & CmppLevel_F_ELIDE)) +#define CT_skip(t) CLvl_skip(&CT_level(t)) +#define CmppTokenizer_empty_m { \ + 0,0,0,0,0,1U/*lineNo*/, \ + TS_Start, \ + CmppToken_empty_m, \ + {/*level*/0U,{CmppLevel_empty_m}}, \ + {/*args*/0,0,{0},{0}} \ + } +static const CmppTokenizer CmppTokenizer_empty = CmppTokenizer_empty_m; + +static void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n); +/*static void cmpp_t_outf(CmppTokenizer * t, char const *zFmt, ...);*/ + +/* +** Pushes a new level into the given tokenizer. Fails fatally if +** it's too deep. +*/ +static void CmppLevel_push(CmppTokenizer * const t); +/* +** Pops a level from the tokenizer. Fails fatally if the top +** level is popped. +*/ +static void CmppLevel_pop(CmppTokenizer * const t); +/* +** Returns the current level object. +*/ +static CmppLevel * CmppLevel_get(CmppTokenizer * const t); + +/* +** Global app state singleton. */ +static struct Global { + /* main()'s argv[0]. */ + const char * zArgv0; + /* + ** Bytes of the keyword delimiter/prefix. Owned + ** elsewhere. + */ + const char * zDelim; + /* Byte length of this->zDelim. */ + unsigned short nDelim; + /* If true, enables certain debugging output. */ + int doDebug; + /* App's db instance. */ + sqlite3 * db; + /* Output channel. */ + FileWrapper out; + struct { + sqlite3_stmt * defIns; + sqlite3_stmt * defDel; + sqlite3_stmt * defHas; + sqlite3_stmt * inclIns; + sqlite3_stmt * inclDel; + sqlite3_stmt * inclHas; + sqlite3_stmt * inclPathAdd; + sqlite3_stmt * inclSearch; + } stmt; +} g = { +"?", +CMPP_DEFAULT_DELIM/*zDelim*/, +(unsigned short) sizeof(CMPP_DEFAULT_DELIM)-1/*nDelim*/, +0/*doDebug*/, +0/*db*/, +FileWrapper_empty_m/*out*/, +{/*stmt*/ + 0/*defIns*/, 0/*defDel*/, 0/*defHas*/, + 0/*inclIns*/, 0/*inclDel*/, 0/*inclHas*/, + 0/*inclPathAdd*/ +} +}; + + +#if 0 +/* +** Outputs a printf()-formatted message to c-pp's global output +** channel. +*/ +static void g_outf(char const *zFmt, ...); +void g_outf(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + vfprintf(g.out.pFile, zFmt, va); + va_end(va); +} +#endif + +#if 0 +/* Outputs n bytes from z to c-pp's global output channel. */ +static void g_out(void const *z, unsigned int n); +void g_out(void const *z, unsigned int n){ + if(1!=fwrite(z, n, 1, g.out.pFile)){ + int const err = errno; + fatal("fwrite() output failed with errno #%d", err); + } +} +#endif + +void g_stderrv(char const *zFmt, va_list va){ + vfprintf(stderr, zFmt, va); +} + +void g_stderr(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + g_stderrv(zFmt, va); + va_end(va); +} + +#if 0 +void cmpp_t_outf(CmppTokenizer * t, char const *zFmt, ...){ + if(!CT_skip(t)){ + va_list va; + va_start(va, zFmt); + vfprintf(g.out.pFile, zFmt, va); + va_end(va); + } +} +#endif + +void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n){ + if(!CT_skip(t)){ + if(1!=fwrite(z, n, 1, g.out.pFile)){ + int const err = errno; + fatal("fwrite() output failed with errno #%d", err); + } + } +} + +void CmppLevel_push(CmppTokenizer * const t){ + CmppLevel * pPrev; + CmppLevel * p; + if(t->level.ndx+1 == (unsigned)CmppLevel_Max){ + fatal("%sif nesting level is too deep. Max=%d\n", + g.zDelim, CmppLevel_Max); + } + pPrev = &CT_level(t); + p = &t->level.stack[++t->level.ndx]; + *p = CmppLevel_empty; + p->token = t->token; + p->flags = (CmppLevel_F_INHERIT_MASK & pPrev->flags); + if(CLvl_skip(pPrev)) p->flags |= CmppLevel_F_ELIDE; +} + +void CmppLevel_pop(CmppTokenizer * const t){ + if(!t->level.ndx){ + fatal("Internal error: CmppLevel_pop() at the top of the stack"); + } + t->level.stack[t->level.ndx--] = CmppLevel_empty; +} + +CmppLevel * CmppLevel_get(CmppTokenizer * const t){ + return &t->level.stack[t->level.ndx]; +} + + +void db_affirm_rc(int rc, const char * zMsg){ + if(rc){ + fatal("Db error #%d %s: %s", rc, zMsg, sqlite3_errmsg(g.db)); + } +} + +void db_finalize(sqlite3_stmt *pStmt){ + sqlite3_finalize(pStmt); +} + +int db_step(sqlite3_stmt *pStmt){ + int const rc = sqlite3_step(pStmt); + if(SQLITE_ROW!=rc && SQLITE_DONE!=rc){ + db_affirm_rc(rc, "from db_step()"); + } + return rc; +} + +static sqlite3_str * db_str_new(void){ + sqlite3_str * rc = sqlite3_str_new(g.db); + if(!rc) fatal("Alloc failed for sqlite3_str_new()"); + return rc; +} + +static char * db_str_finish(sqlite3_str *s, int * n){ + int const rc = sqlite3_str_errcode(s); + if(rc) fatal("Error #%d from sqlite3_str_errcode()", rc); + if(n) *n = sqlite3_str_length(s); + char * z = sqlite3_str_finish(s); + if(!z) fatal("Alloc failed for sqlite3_str_new()"); + return z; +} + +void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...){ + int rc; + sqlite3_str * str = db_str_new(); + char * z = 0; + int n = 0; + va_list va; + if(!str) fatal("sqlite3_str_new() failed"); + va_start(va, zSql); + sqlite3_str_vappendf(str, zSql, va); + va_end(va); + rc = sqlite3_str_errcode(str); + if(rc) fatal("sqlite3_str_errcode() = %d", rc); + z = db_str_finish(str, &n); + rc = sqlite3_prepare_v2(g.db, z, n, pStmt, 0); + if(rc) fatal("Error #%d (%s) preparing: %s", + rc, sqlite3_errmsg(g.db), z); + sqlite3_free(z); +} + +void db_bind_int(sqlite3_stmt *pStmt, int col, int val){ + int const rc = sqlite3_bind_int(pStmt, col, val); + db_affirm_rc(rc,"from db_bind_int()"); +} + +#if 0 +void db_bind_null(sqlite3_stmt *pStmt, int col){ + int const rc = sqlite3_bind_null(pStmt, col); + db_affirm_rc(rc,"from db_bind_null()"); +} +#endif + +void db_bind_textn(sqlite3_stmt *pStmt, int col, + const char * zStr, int n){ + int const rc = zStr + ? sqlite3_bind_text(pStmt, col, zStr, n, SQLITE_TRANSIENT) + : sqlite3_bind_null(pStmt, col); + db_affirm_rc(rc,"from db_bind_textn()"); +} + +void db_bind_text(sqlite3_stmt *pStmt, int col, + const char * zStr){ + db_bind_textn(pStmt, col, zStr, -1); +} + +#if 0 +void db_bind_textv(sqlite3_stmt *pStmt, int col, + const char * zFmt, ...){ + int rc; + sqlite3_str * str = db_str_new(); + int n = 0; + char * z; + va_list va; + va_start(va,zFmt); + sqlite3_str_vappendf(str, zFmt, va); + va_end(va); + z = db_str_finish(str, &n); + rc = sqlite3_bind_text(pStmt, col, z, n, sqlite3_free); + db_affirm_rc(rc,"from db_bind_textv()"); +} +#endif + +void db_free(void *m){ + sqlite3_free(m); +} + +void db_define_add(const char * zKey){ + int rc; + if(!g.stmt.defIns){ + db_prepare(&g.stmt.defIns, + "INSERT OR REPLACE INTO def(k) VALUES(?)"); + } + db_bind_text(g.stmt.defIns, 1, zKey); + rc = db_step(g.stmt.defIns); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping INSERT on def"); + } + g_debug(2,("define: %s\n",zKey)); + sqlite3_reset(g.stmt.defIns); +} + +int db_define_has(const char * zName){ + int rc; + if(!g.stmt.defHas){ + db_prepare(&g.stmt.defHas, "SELECT 1 FROM def WHERE k=?"); + } + db_bind_text(g.stmt.defHas, 1, zName); + rc = db_step(g.stmt.defHas); + if(SQLITE_ROW == rc){ + rc = 1; + }else{ + assert(SQLITE_DONE==rc); + rc = 0; + } + g_debug(1,("define has [%s] = %d\n",zName, rc)); + sqlite3_clear_bindings(g.stmt.defHas); + sqlite3_reset(g.stmt.defHas); + return rc; +} + + +void db_define_rm(const char * zKey){ + int rc; + int n = 0; + const char *zPos = zKey; + if(!g.stmt.defDel){ + db_prepare(&g.stmt.defDel, "DELETE FROM def WHERE k=?"); + } + for( ; *zPos && '='!=*zPos; ++n, ++zPos) {} + db_bind_text(g.stmt.defDel, 1, zKey); + rc = db_step(g.stmt.defDel); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping DELETE on def"); + } + g_debug(2,("undefine: %.*s\n",n, zKey)); + sqlite3_clear_bindings(g.stmt.defDel); + sqlite3_reset(g.stmt.defDel); +} + +void db_including_add(const char * zKey, const char * zSrc, int srcLine){ + int rc; + if(!g.stmt.inclIns){ + db_prepare(&g.stmt.inclIns, + "INSERT OR FAIL INTO incl(file,srcFile,srcLine) VALUES(?,?,?)"); + } + db_bind_text(g.stmt.inclIns, 1, zKey); + db_bind_text(g.stmt.inclIns, 2, zSrc); + db_bind_int(g.stmt.inclIns, 3, srcLine); + rc = db_step(g.stmt.inclIns); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping INSERT on incl"); + } + g_debug(2,("inclpath add [%s] from [%s]:%d\n", zKey, zSrc, srcLine)); + sqlite3_clear_bindings(g.stmt.inclIns); + sqlite3_reset(g.stmt.inclIns); +} + +void db_include_rm(const char * zKey){ + int rc; + if(!g.stmt.inclDel){ + db_prepare(&g.stmt.inclDel, "DELETE FROM incl WHERE file=?"); + } + db_bind_text(g.stmt.inclDel, 1, zKey); + rc = db_step(g.stmt.inclDel); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping DELETE on incl"); + } + g_debug(2,("inclpath rm [%s]\n", zKey)); + sqlite3_clear_bindings(g.stmt.inclDel); + sqlite3_reset(g.stmt.inclDel); +} + +char * db_include_search(const char * zKey){ + char * zName = 0; + if(!g.stmt.inclSearch){ + db_prepare(&g.stmt.inclSearch, + "SELECT ?1 fn WHERE fileExists(fn) " + "UNION ALL SELECT * FROM (" + "SELECT replace(dir||'/'||?1, '//','/') AS fn " + "FROM inclpath WHERE fileExists(fn) ORDER BY seq" + ")"); + } + db_bind_text(g.stmt.inclSearch, 1, zKey); + if(SQLITE_ROW==db_step(g.stmt.inclSearch)){ + const unsigned char * z = sqlite3_column_text(g.stmt.inclSearch, 0); + zName = z ? sqlite3_mprintf("%s", z) : 0; + if(!zName) fatal("Alloc failed"); + } + sqlite3_clear_bindings(g.stmt.inclSearch); + sqlite3_reset(g.stmt.inclSearch); + return zName; +} + +static int db_including_has(const char * zName){ + int rc; + if(!g.stmt.inclHas){ + db_prepare(&g.stmt.inclHas, "SELECT 1 FROM incl WHERE file=?"); + } + db_bind_text(g.stmt.inclHas, 1, zName); + rc = db_step(g.stmt.inclHas); + if(SQLITE_ROW == rc){ + rc = 1; + }else{ + assert(SQLITE_DONE==rc); + rc = 0; + } + g_debug(2,("inclpath has [%s] = %d\n",zName, rc)); + sqlite3_clear_bindings(g.stmt.inclHas); + sqlite3_reset(g.stmt.inclHas); + return rc; +} + +#if 0 +/* +** Fails fatally if the `#include` list contains the given key. +*/ +static void db_including_check(const char * zKey); +void db_including_check(const char * zName){ + if(db_including_has(zName)){ + fatal("Recursive include detected: %s\n", zName); + } +} +#endif + +void db_include_dir_add(const char * zDir){ + static int seq = 0; + int rc; + if(!g.stmt.inclPathAdd){ + db_prepare(&g.stmt.inclPathAdd, + "INSERT OR FAIL INTO inclpath(seq,dir) VALUES(?,?)"); + } + db_bind_int(g.stmt.inclPathAdd, 1, ++seq); + db_bind_text(g.stmt.inclPathAdd, 2, zDir); + rc = db_step(g.stmt.inclPathAdd); + if(SQLITE_DONE != rc){ + db_affirm_rc(rc, "Stepping INSERT on inclpath"); + } + g_debug(2,("inclpath add #%d: %s\n",seq, zDir)); + sqlite3_clear_bindings(g.stmt.inclPathAdd); + sqlite3_reset(g.stmt.inclPathAdd); +} + +static void cmpp_atexit(void){ +#define FINI(M) if(g.stmt.M) sqlite3_finalize(g.stmt.M) + FINI(defIns); FINI(defDel); FINI(defHas); + FINI(inclIns); FINI(inclDel); FINI(inclHas); + FINI(inclPathAdd); FINI(inclSearch); +#undef FINI + FileWrapper_close(&g.out); + if(g.db) sqlite3_close(g.db); +} + +/* +** sqlite3 UDF which returns true if its argument refers to an +** accessible file, else false. +*/ +static void udf_file_exists( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + sqlite3_result_int(context, 0==access(zName, 0)); +} + +/* Initialize g.db, failing fatally on error. */ +static void cmpp_initdb(void){ + int rc; + char * zErr = 0; + const char * zSchema = + "CREATE TABLE def(" + "k TEXT PRIMARY KEY NOT NULL" + /*"v INTEGER DEFAULT 1"*/ + ") WITHOUT ROWID;" + /* ^^^ defines */ + "CREATE TABLE incl(" + "file TEXT PRIMARY KEY NOT NULL," + "srcFile TEXT DEFAULT NULL," + "srcLine INTEGER DEFAULT 0" + ") WITHOUT ROWID;" + /* ^^^ files currently being included */ + "CREATE TABLE inclpath(" + "seq INTEGER UNIQUE, " + "dir TEXT PRIMARY KEY NOT NULL ON CONFLICT IGNORE" + ")" + /* ^^^ include path */ + ; + assert(0==g.db); + if(g.db) return; + rc = sqlite3_open_v2(":memory:", &g.db, SQLITE_OPEN_READWRITE, 0); + if(rc) fatal("Error opening :memory: db."); + rc = sqlite3_exec(g.db, zSchema, 0, 0, &zErr); + if(rc) fatal("Error initializing database: %s", zErr); + rc = sqlite3_create_function(g.db, "fileExists", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + udf_file_exists, 0, 0); + db_affirm_rc(rc, "UDF registration failed."); +} + +/* +** For position zPos, which must be in the half-open range +** [zBegin,zEnd), returns g.nDelim if it is at the start of a line and +** starts with g.zDelim, else returns 0. +*/ +static unsigned short cmpp_is_delim(unsigned char const *zBegin, + unsigned char const *zEnd, + unsigned char const *zPos){ + assert(zEnd>zBegin); + assert(zPos=zBegin); + if(zPos>zBegin && + ('\n'!=*(zPos - 1) + || ((unsigned)(zEnd - zPos) <= g.nDelim))){ + return 0; + }else if(0==memcmp(zPos, g.zDelim, g.nDelim)){ + return g.nDelim; + }else{ + return 0; + } +} + +/* +** Scans t to the next keyword line, emitting all input before that +** which is _not_ a keyword line unless it's elided due to being +** inside a block which elides its content. Returns 0 if no keyword +** line was found, in which case the end of the input has been +** reached, else returns a truthy value and sets up t's state for use +** with cmpp_process_keyword(), which should then be called. +*/ +static int cmpp_next_keyword_line(CmppTokenizer * const t){ + unsigned char const * zStart; + unsigned char const * z; + CmppToken * const tok = &t->token; + unsigned short isDelim = 0; + + assert(t->zBegin); + assert(t->zEnd > t->zBegin); + if(!t->zPos) t->zPos = t->zBegin; + t->zAnchor = t->zPos; + zStart = z = t->zPos; + *tok = CmppToken_empty; + while(zzEnd + && 0==(isDelim = cmpp_is_delim(t->zBegin, t->zEnd, z))){ + ++z; + } + if(z>zStart){ + /* We passed up content */ + cmpp_t_out(t, zStart, (unsigned)(z - zStart)); + } + assert(isDelim==0 || isDelim==g.nDelim); + tok->lineNo = t->lineNo += count_lines(zStart, z); + if(isDelim){ + /* Handle backslash-escaped newlines */ + int isEsc = 0, atEol = 0; + tok->zBegin = z+isDelim; + for( ++z ; zzEnd && 0==atEol; ++z ){ + switch((int)*z){ + case (int)'\\': + isEsc = 0==isEsc; break; + case (int)'\n': + atEol = 0==isEsc; + isEsc = 0; + ++t->lineNo; + break; + default: + break; + } + } + tok->zEnd = atEol ? z-1 : z; + /* Strip leading spaces */ + while(tok->zBegin < tok->zEnd && isspace((char)(*tok->zBegin))){ + ++tok->zBegin; + } + tok->ttype = TT_Line; + g_debug(2,("Keyword @ line %u: [[[%.*s]]]\n", + tok->lineNo, + (int)(tok->zEnd-tok->zBegin), tok->zBegin)); + } + t->zPos = z; + if(isDelim){ + /* Split t->token into arguments for the line's keyword */ + int i, argc = 0, prevChar = 0; + const unsigned tokLen = (unsigned)(tok->zEnd - tok->zBegin); + unsigned char * zKwd; + unsigned char * zEsc; + unsigned char * zz; + + assert(TT_Line==tok->ttype); + if((unsigned)sizeof(t->args.lineBuf) < tokLen + 1){ + fatal("Keyword line is unreasonably long: %.*s", + tokLen, tok->zBegin); + }else if(!tokLen){ + fatal("Line #%u has no keyword after delimiter", tok->lineNo); + } + g_debug(2,("token @ line %u len=%u [[[%.*s]]]\n", + tok->lineNo, tokLen, tokLen, tok->zBegin)); + zKwd = &t->args.lineBuf[0]; + memcpy(zKwd, tok->zBegin, tokLen); + memset(zKwd + tokLen, 0, sizeof(t->args.lineBuf) - tokLen); + for( zEsc = 0, zz = zKwd; *zz; ++zz ){ + /* Convert backslash-escaped newlines to whitespace */ + switch((int)*zz){ + case (int)'\\': + if(zEsc) zEsc = 0; + else zEsc = zz; + break; + case (int)'\n': + assert(zEsc && "Should not have an unescaped newline?"); + if(zEsc==zz-1){ + *zEsc = (unsigned char)' '; + /* FIXME?: memmove() lnBuf content one byte to the left here + ** to collapse backslash and newline into a single + ** byte. Also consider collapsing all leading space on the + ** next line. */ + } + zEsc = 0; + *zz = (unsigned char)' '; + break; + default: + zEsc = 0; + break; + } + } + t->args.argv[argc++] = zKwd; + for( zz = zKwd; *zz; ++zz ){ + if(isspace(*zz)){ + *zz = 0; + break; + } + } + t->args.pKw = CmppKeyword_search((char const *)zKwd); + if(!t->args.pKw){ + fatal("Unknown keyword '%s' at line %u\n", (char const *)zKwd, + tok->lineNo); + } + for( ++zz ; *zz && isspace(*zz); ++zz ){} + if(t->args.pKw->bTokenize){ + for( ; *zz; prevChar = *zz, ++zz ){ + /* Split string into word-shaped tokens. + ** TODO ?= quoted strings, for the sake of the + ** #error keyword. */ + if(isspace(*zz)){ + assert(zz!=zKwd && "Leading space was stripped earlier."); + *zz = 0; + }else{ + if(argc == (int)CmppArgs_Max){ + fatal("Too many arguments @ line %u: %.*s", + tok->lineNo, tokLen, tok->zBegin); + }else if(zz>zKwd && !prevChar){ + t->args.argv[argc++] = zz; + } + } + } + }else{ + /* Treat rest of line as one token */ + if(*zz) t->args.argv[argc++] = zz; + } + tok->ttype = t->args.pKw->ttype; + if(g.doDebug>1){ + for(i = 0; i < argc; ++i){ + g_debug(0,("line %u arg #%d=%s\n", + tok->lineNo, i, + (char const *)t->args.argv[i])); + } + } + t->args.argc = argc; + }else{ + t->args.pKw = 0; + t->args.argc = 0; + } + return isDelim; +} + +static void cmpp_kwd__err_prefix(CmppKeyword const * pKw, CmppTokenizer *t, + char const *zPrefix){ + g_stderr("%s%s%s @ %s line %u: ", + zPrefix ? zPrefix : "", + zPrefix ? ": " : "", + pKw->zName, t->zName, t->token.lineNo); +} + +/* Internal error reporting helper for cmpp_keyword_f() impls. */ +static CMPP_NORETURN void cmpp_kwd__misuse(CmppKeyword const * pKw, + CmppTokenizer *t, + char const *zFmt, ...){ + va_list va; + cmpp_kwd__err_prefix(pKw, t, "Fatal error"); + va_start(va, zFmt); + fatalv(zFmt, va); + va_end(va); +} + +/* No-op cmpp_keyword_f() impl. */ +static void cmpp_kwd_noop(CmppKeyword const * pKw, CmppTokenizer *t){ + if(t || pKw){/*unused*/} +} + +/* #error impl. */ +static void cmpp_kwd_error(CmppKeyword const * pKw, CmppTokenizer *t){ + if(CT_skip(t)) return; + else{ + assert(t->args.argc < 3); + const char *zBegin = t->args.argc>1 + ? (const char *)t->args.argv[1] : 0; + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("%s", zBegin ? zBegin : "(no additional info)"); + } +} + +/* Impl. for #define, #undef */ +static void cmpp_kwd_define(CmppKeyword const * pKw, CmppTokenizer *t){ + if(CT_skip(t)) return; + if(t->args.argc<2){ + cmpp_kwd__misuse(pKw, t, "Expecting one or more arguments"); + }else{ + int i = 1; + void (*func)(const char *) = TT_Define==pKw->ttype + ? db_define_add : db_define_rm; + for( ; i < t->args.argc; ++i){ + func( (char const *)t->args.argv[i] ); + } + } +} + +/* Impl. for #if, #ifnot, #elif, #elifnot. */ +static void cmpp_kwd_if(CmppKeyword const * pKw, CmppTokenizer *t){ + int buul; + CmppParseState tmpState = TS_Start; + if(t->args.argc!=2){ + cmpp_kwd__misuse(pKw, t, "Expecting exactly 1 argument"); + } + /*g_debug(0,("%s %s level %u pstate=%d\n", pKw->zName, + (char const *)t->args.argv[1], + t->level.ndx, (int)CT_pstate(t)));*/ + switch(pKw->ttype){ + case TT_Elif: + case TT_ElifNot: + switch(CT_pstate(t)){ + case TS_If: break; + case TS_IfPassed: CT_level(t).flags |= CmppLevel_F_ELIDE; return; + default: goto misuse; + } + break; + case TT_If: + case TT_IfNot: + CmppLevel_push(t); + break; + default: + cmpp_kwd__misuse(pKw, t, "Unpexected keyword token type"); + break; + } + buul = db_define_has((char const *)t->args.argv[1]); + if(TT_IfNot==pKw->ttype || TT_ElifNot==pKw->ttype) buul = !buul; + if(buul){ + CT_pstate(t) = tmpState = TS_IfPassed; + CT_skipLevel(t) = 0; + }else{ + CT_pstate(t) = TS_If /* also for TT_IfNot, TT_Elif, TT_ElifNot */; + CT_skipLevel(t) = 1; + } + if(TT_If==pKw->ttype || TT_IfNot==pKw->ttype){ + unsigned const lvlIf = t->level.ndx; + CmppToken const lvlToken = CT_level(t).token; + while(cmpp_next_keyword_line(t)){ + cmpp_process_keyword(t); + if(lvlIf > t->level.ndx){ + assert(TT_EndIf == t->token.ttype); + break; + } + if(TS_IfPassed==tmpState){ + tmpState = TS_Start; + t->level.stack[lvlIf].flags |= CmppLevel_F_ELIDE; + } + } + if(lvlIf <= t->level.ndx){ + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("Input ended inside an unterminated %sif " + "opened at [%s] line %u", + g.zDelim, t->zName, lvlToken.lineNo); + } + } + return; + misuse: + cmpp_kwd__misuse(pKw, t, "'%s' used out of context", + pKw->zName); +} + +/* Impl. for #else. */ +static void cmpp_kwd_else(CmppKeyword const * pKw, CmppTokenizer *t){ + if(t->args.argc>1){ + cmpp_kwd__misuse(pKw, t, "Expecting no arguments"); + } + switch(CT_pstate(t)){ + case TS_IfPassed: CT_skipLevel(t) = 1; break; + case TS_If: CT_skipLevel(t) = 0; break; + default: + cmpp_kwd__misuse(pKw, t, "'%s' with no matching 'if'", + pKw->zName); + } + /*g_debug(0,("else flags=0x%02x skipLevel=%u\n", + CT_level(t).flags, CT_level(t).skipLevel));*/ + CT_pstate(t) = TS_Else; +} + +/* Impl. for #endif. */ +static void cmpp_kwd_endif(CmppKeyword const * pKw, CmppTokenizer *t){ + /* Maintenance reminder: we ignore all arguments after the endif + ** to allow for constructs like: + ** + ** #endif // foo + ** + ** in a manner which does not require a specific comment style */ + switch(CT_pstate(t)){ + case TS_Else: + case TS_If: + case TS_IfPassed: + break; + default: + cmpp_kwd__misuse(pKw, t, "'%s' with no matching 'if'", + pKw->zName); + } + CmppLevel_pop(t); +} + +/* Impl. for #include. */ +static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ + char const * zFile; + char * zResolved; + if(CT_skip(t)) return; + else if(t->args.argc!=2){ + cmpp_kwd__misuse(pKw, t, "Expecting exactly 1 filename argument"); + } + zFile = (const char *)t->args.argv[1]; + if(db_including_has(zFile)){ + /* Note that different spellings of the same filename + ** will elude this check, but that seems okay, as different + ** spellings means that we're not re-running the exact same + ** invocation. We might want some other form of multi-include + ** protection, rather than this, however. There may well be + ** sensible uses for recursion. */ + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("Recursive include of file: %s", zFile); + } + zResolved = db_include_search(zFile); + if(zResolved){ + db_including_add(zFile, t->zName, t->token.lineNo); + cmpp_process_file(zResolved); + db_include_rm(zFile); + db_free(zResolved); + }else{ + cmpp_kwd__err_prefix(pKw, t, NULL); + fatal("file not found: %s", zFile); + } +} + +/* Impl. for #pragma. */ +static void cmpp_kwd_pragma(CmppKeyword const * pKw, CmppTokenizer *t){ + const char * zArg; + if(CT_skip(t)) return; + else if(t->args.argc!=2){ + cmpp_kwd__misuse(pKw, t, "Expecting one argument"); + } + zArg = (const char *)t->args.argv[1]; +#define M(X) 0==strcmp(zArg,X) + if(M("defines")){ + sqlite3_stmt * q = 0; + db_prepare(&q, "SELECT k FROM def ORDER BY k"); + g_stderr("cmpp defines:\n"); + while(SQLITE_ROW==db_step(q)){ + int const n = sqlite3_column_bytes(q, 0); + const char * z = (const char *)sqlite3_column_text(q, 0); + g_stderr("\t%.*s\n", n, z); + } + db_finalize(q); + }else{ + cmpp_kwd__misuse(pKw, t, "Unknown pragma"); + } +#undef M +} + +/* #stder impl. */ +static void cmpp_kwd_stderr(CmppKeyword const * pKw, CmppTokenizer *t){ + if(CT_skip(t)) return; + else{ + const char *zBegin = t->args.argc>1 + ? (const char *)t->args.argv[1] : 0; + if(zBegin){ + g_stderr("%s:%u: %s\n", t->zName, t->token.lineNo, zBegin); + }else{ + g_stderr("%s:%u: (no %.*s%s argument)\n", + t->zName, t->token.lineNo, + g.nDelim, g.zDelim, pKw->zName); + } + } +} + +#if 0 +/* Impl. for dummy placeholder. */ +static void cmpp_kwd_todo(CmppKeyword const * pKw, CmppTokenizer *t){ + if(t){/*unused*/} + g_debug(0,("TODO: keyword handler for %s\n", pKw->zName)); +} +#endif + +CmppKeyword aKeywords[] = { +/* Keep these sorted by zName */ + {"//", 2, 0, TT_Comment, cmpp_kwd_noop}, + {"define", 6, 1, TT_Define, cmpp_kwd_define}, + {"elif", 4, 1, TT_Elif, cmpp_kwd_if}, + {"elifnot", 7, 1, TT_ElifNot, cmpp_kwd_if}, + {"else", 4, 1, TT_Else, cmpp_kwd_else}, + {"endif", 5, 0, TT_EndIf, cmpp_kwd_endif}, + {"error", 4, 0, TT_Error, cmpp_kwd_error}, + {"if", 2, 1, TT_If, cmpp_kwd_if}, + {"ifnot", 5, 1, TT_IfNot, cmpp_kwd_if}, + {"include", 7, 0, TT_Include, cmpp_kwd_include}, + {"pragma", 6, 1, TT_Pragma, cmpp_kwd_pragma}, + {"stderr", 6, 0, TT_Stderr, cmpp_kwd_stderr}, + {"undef", 5, 1, TT_Undef, cmpp_kwd_define}, + {0,0,TT_Invalid, 0} +}; + +static int cmp_CmppKeyword(const void *p1, const void *p2){ + char const * zName = (const char *)p1; + CmppKeyword const * kw = (CmppKeyword const *)p2; + return strcmp(zName, kw->zName); +} + +CmppKeyword const * CmppKeyword_search(const char *zName){ + return (CmppKeyword const *)bsearch(zName, &aKeywords[0], + sizeof(aKeywords)/sizeof(aKeywords[0]) - 1, + sizeof(aKeywords[0]), + cmp_CmppKeyword); +} + +void cmpp_process_keyword(CmppTokenizer * const t){ + assert(t->args.pKw); + assert(t->args.argc); + t->args.pKw->xCall(t->args.pKw, t); + t->args.pKw = 0; + t->args.argc = 0; +} + +void cmpp_process_file(const char * zName){ + FileWrapper fw = FileWrapper_empty; + CmppTokenizer ct = CmppTokenizer_empty; + + FileWrapper_open(&fw, zName, "r"); + FileWrapper_slurp(&fw); + g_debug(1,("Read %u byte(s) from [%s]\n", fw.nContent, fw.zName)); + ct.zName = zName; + ct.zBegin = fw.zContent; + ct.zEnd = fw.zContent + fw.nContent; + while(cmpp_next_keyword_line(&ct)){ + cmpp_process_keyword(&ct); + } + FileWrapper_close(&fw); + if(0!=ct.level.ndx){ + CmppLevel * const lv = CmppLevel_get(&ct); + fatal("Input ended inside an unterminated nested construct" + "opened at [%s] line %u", zName, lv->token.lineNo); + } +} + +static void usage(int isErr){ + FILE * const fOut = isErr ? stderr : stdout; + fprintf(fOut, + "Usage: %s [flags] [infile]\n" + "Flags:\n", + g.zArgv0); +#define arg(F,D) fprintf(fOut," %s\n %s\n",F, D) + arg("-f|--file FILE","Read input from FILE (default=- (stdin)).\n" + " Alternately, the first non-flag argument is assumed to " + "be the input file."); + arg("-o|--outfile FILE","Send output to FILE (default=- (stdout))"); + arg("-DXYZ","Define XYZ to true"); + arg("-UXYZ","Undefine XYZ (equivalent to false)"); + arg("-IXYZ","Add dir XYZ to include path"); + arg("-d|--delimiter VALUE", "Set keyword delimiter to VALUE " + "(default=" CMPP_DEFAULT_DELIM ")"); +#undef arg + fputs("",fOut); +} + +int main(int argc, char const * const * argv){ + int rc = 0; + int i; + int inclCount = 0; + const char * zInfile = 0; +#define M(X) (0==strcmp(X,zArg)) +#define ISFLAG(X) else if(M(X)) +#define ISFLAG2(X,Y) else if(M(X) || M(Y)) +#define ARGVAL \ + if(i+1>=argc) fatal("Missing value for flag '%s'", zArg); \ + zArg = argv[++i] + g.zArgv0 = argv[0]; + atexit(cmpp_atexit); + cmpp_initdb(); + for(i = 1; i < argc; ++i){ + char const * zArg = argv[i]; + while('-'==*zArg) ++zArg; + if(M("?") || M("help")) { + usage(0); + goto end; + }else if('D'==*zArg){ + ++zArg; + if(!*zArg) fatal("Missing key for -D"); + db_define_add(zArg); + }else if('U'==*zArg){ + ++zArg; + if(!*zArg) fatal("Missing key for -U"); + db_define_rm(zArg); + }else if('I'==*zArg){ + ++zArg; + if(!*zArg) fatal("Missing directory for -I"); + db_include_dir_add(zArg); + ++inclCount; + } + ISFLAG2("o","outfile"){ + ARGVAL; + if(g.out.zName) fatal("Cannot use -o more than once."); + g.out.zName = zArg; + } + ISFLAG2("f","file"){ + ARGVAL; + do_infile: + if(zInfile) fatal("Cannot use -i more than once."); + zInfile = zArg; + } + ISFLAG2("d","delimiter"){ + ARGVAL; + g.zDelim = zArg; + g.nDelim = (unsigned short)strlen(zArg); + if(!g.nDelim) fatal("Keyword delimiter may not be empty."); + } + ISFLAG("debug"){ + ++g.doDebug; + }else if(!zInfile){ + goto do_infile; + }else{ + fatal("Unhandled flag: %s", argv[i]); + } + } + if(!zInfile) zInfile = "-"; + if(!g.out.zName) g.out.zName = "-"; + if(!inclCount) db_include_dir_add("."); + FileWrapper_open(&g.out, g.out.zName, "w"); + cmpp_process_file(zInfile); + FileWrapper_close(&g.out); + end: + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} + +#undef CT_level +#undef CT_pstate +#undef CT_skipLevel +#undef CT_skip +#undef CLvl_skip Index: ext/wasm/common/emscripten.css ================================================================== --- ext/wasm/common/emscripten.css +++ ext/wasm/common/emscripten.css @@ -1,6 +1,6 @@ -/* emcscript-related styling, used during the module load/intialization processes... */ +/* emscripten-related styling, used during the module load/intialization processes... */ .emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } div.emscripten { text-align: center; } div.emscripten_border { border: 1px solid black; } #module-spinner { overflow: visible; } #module-spinner > * { Index: ext/wasm/common/testing.css ================================================================== --- ext/wasm/common/testing.css +++ ext/wasm/common/testing.css @@ -59,5 +59,11 @@ } #test-output.reverse { flex-direction: column-reverse; } label[for] { cursor: pointer } + +h1 { + border-radius: 0.25em; + padding: 0.15em 0.25em; +} +h1:first-of-type {margin: 0 0 0.5em 0;} Index: ext/wasm/common/whwasmutil.js ================================================================== --- ext/wasm/common/whwasmutil.js +++ ext/wasm/common/whwasmutil.js @@ -243,10 +243,29 @@ */ cache.scopedAlloc = []; cache.utf8Decoder = new TextDecoder(); cache.utf8Encoder = new TextEncoder('utf-8'); + + /** + For the given IR-like string in the set ('i8', 'i16', 'i32', + 'f32', 'float', 'i64', 'f64', 'double', '*'), or any string value + ending in '*', returns the sizeof for that value + (target.ptrSizeof in the latter case). For any other value, it + returns the undefined value. + */ + target.sizeofIR = (n)=>{ + switch(n){ + case 'i8': return 1; + case 'i16': return 2; + case 'i32': case 'f32': case 'float': return 4; + case 'i64': case 'f64': case 'double': return 8; + case '*': return ptrSizeof; + default: + return (''+n).endsWith('*') ? ptrSizeof : undefined; + } + }; /** If (cache.heapSize !== cache.memory.buffer.byteLength), i.e. if the heap has grown since the last call, updates cache.HEAPxyz. Returns the cache object. @@ -314,17 +333,15 @@ allocate. Instead, re-fetch the reference by calling this function again. Throws if passed an invalid n. - Pedantic side note: the name "heap" is a bit of a misnomer. In an - Emscripten environment, the memory managed via the stack - allocation API is in the same Memory object as the heap (which - makes sense because otherwise arbitrary pointer X would be - ambiguous: is it in the heap or the stack?). + Pedantic side note: the name "heap" is a bit of a misnomer. In a + WASM environment, the stack and heap memory are all accessed via + the same view(s) of the memory. */ - target.heapForSize = function(n,unsigned = false){ + target.heapForSize = function(n,unsigned = true){ let ctor; const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) ? cache : heapWrappers(); switch(n){ case Int8Array: return c.HEAP8; case Uint8Array: return c.HEAP8U; @@ -359,11 +376,12 @@ */ }; /** Given a function pointer, returns the WASM function table entry - if found, else returns a falsy value. + if found, else returns a falsy value: undefined if fptr is out of + range or null if it's in range but the table entry is empty. */ target.functionEntry = function(fptr){ const ft = target.functionTable(); return fptr < ft.length ? ft.get(fptr) : undefined; }; @@ -446,11 +464,11 @@ letterType: (x)=>f._.sigTypes[x] || toss("Invalid signature letter:",x), /** Returns an object describing the result type and parameter type(s) of the given function signature, or throws if the signature is invalid. */ /******** // only valid for use with the WebAssembly.Function ctor, which - // is not yet documented on MDN. + // is not yet documented on MDN. sigToWasm: function(sig){ const rc = {parameters:[], results: []}; if('v'!==sig[0]) rc.results.push(f.sigTypes(sig[0])); for(const x of f._.sigParams(sig)){ rc.parameters.push(f._.typeCodes(x)); @@ -494,11 +512,74 @@ return (new WebAssembly.Instance( new WebAssembly.Module(new Uint8Array(wasmCode)), { e: { f: func } })).exports['f']; }/*jsFuncToWasm()*/; - + + /** + Documented as target.installFunction() except for the 3rd + argument: if truthy, the newly-created function pointer + is stashed in the current scoped-alloc scope and will be + cleaned up at the matching scopedAllocPop(), else it + is not stashed there. + */ + const __installFunction = function f(func, sig, scoped){ + if(scoped && !cache.scopedAlloc.length){ + toss("No scopedAllocPush() scope is active."); + } + if('string'===typeof func){ + const x = sig; + sig = func; + func = x; + } + if('string'!==typeof sig || !(func instanceof Function)){ + toss("Invalid arguments: expecting (function,signature) "+ + "or (signature,function)."); + } + const ft = target.functionTable(); + const oldLen = ft.length; + let ptr; + while(cache.freeFuncIndexes.length){ + ptr = cache.freeFuncIndexes.pop(); + if(ft.get(ptr)){ /* Table was modified via a different API */ + ptr = null; + continue; + }else{ + break; + } + } + if(!ptr){ + ptr = oldLen; + ft.grow(1); + } + try{ + /*this will only work if func is a WASM-exported function*/ + ft.set(ptr, func); + if(scoped){ + cache.scopedAlloc[cache.scopedAlloc.length-1].push(ptr); + } + return ptr; + }catch(e){ + if(!(e instanceof TypeError)){ + if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); + throw e; + } + } + // It's not a WASM-exported function, so compile one... + try { + const fptr = target.jsFuncToWasm(func, sig); + ft.set(ptr, fptr); + if(scoped){ + cache.scopedAlloc[cache.scopedAlloc.length-1].push(ptr); + } + }catch(e){ + if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); + throw e; + } + return ptr; + }; + /** Expects a JS function and signature, exactly as for this.jsFuncToWasm(). It uses that function to create a WASM-exported function, installs that function to the next available slot of this.functionTable(), and returns the @@ -526,65 +607,39 @@ addFunction() more than once, which leads to uninstallFunction() breaking clients which do not take care to avoid that case: https://github.com/emscripten-core/emscripten/issues/17323 */ - target.installFunction = function f(func, sig){ - if(2!==arguments.length){ - toss("installFunction() requires exactly 2 arguments"); - } - if('string'===typeof func){ - const x = sig; - sig = func; - func = x; - } - const ft = target.functionTable(); - const oldLen = ft.length; - let ptr; - while(cache.freeFuncIndexes.length){ - ptr = cache.freeFuncIndexes.pop(); - if(ft.get(ptr)){ /* Table was modified via a different API */ - ptr = null; - continue; - }else{ - break; - } - } - if(!ptr){ - ptr = oldLen; - ft.grow(1); - } - try{ - /*this will only work if func is a WASM-exported function*/ - ft.set(ptr, func); - return ptr; - }catch(e){ - if(!(e instanceof TypeError)){ - if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); - throw e; - } - } - // It's not a WASM-exported function, so compile one... - try { - ft.set(ptr, target.jsFuncToWasm(func, sig)); - }catch(e){ - if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); - throw e; - } - return ptr; - }; + target.installFunction = (func, sig)=>__installFunction(func, sig, false); + + /** + EXPERIMENTAL! DO NOT USE IN CLIENT CODE! + + Works exactly like installFunction() but requires that a + scopedAllocPush() is active and uninstalls the given function + when that alloc scope is popped via scopedAllocPop(). + This is used for implementing JS/WASM function bindings which + should only persist for the life of a call into a single + C-side function. + */ + target.scopedInstallFunction = (func, sig)=>__installFunction(func, sig, true); /** Requires a pointer value previously returned from this.installFunction(). Removes that function from the WASM function table, marks its table slot as free for re-use, and returns that function. It is illegal to call this before installFunction() has been called and results are undefined if ptr was not returned by that function. The returned function may be passed back to installFunction() to reinstall it. + + To simplify certain use cases, if passed a falsy non-0 value + (noting that 0 is a valid function table index), this function + has no side effects and returns undefined. */ target.uninstallFunction = function(ptr){ + if(!ptr && 0!==ptr) return undefined; const fi = cache.freeFuncIndexes; const ft = target.functionTable(); fi.push(ptr); const rc = ft.get(ptr); ft.set(ptr, null); @@ -596,24 +651,29 @@ (i8, i16, i32, i64, float (or f32), double (or f64)), this fetches the numeric value from that address and returns it as a number or, for the case of type='i64', a BigInt (noting that that type triggers an exception if this.bigIntEnabled is falsy). Throws if given an invalid type. + + If the first argument is an array, it is treated as an array of + addresses and the result is an array of the values from each of + those address, using the same 2nd argument for determining the + value type to fetch. As a special case, if type ends with a `*`, it is considered to be a pointer type and is treated as the WASM numeric type appropriate for the pointer size (`i32`). - While likely not obvious, this routine and its setMemValue() + While likely not obvious, this routine and its poke() counterpart are how pointer-to-value _output_ parameters in WASM-compiled C code can be interacted with: ``` const ptr = alloc(4); - setMemValue(ptr, 0, 'i32'); // clear the ptr's value + poke(ptr, 0, 'i32'); // clear the ptr's value aCFuncWithOutputPtrToInt32Arg( ptr ); // e.g. void foo(int *x); - const result = getMemValue(ptr, 'i32'); // fetch ptr's value + const result = peek(ptr, 'i32'); // fetch ptr's value dealloc(ptr); ``` scopedAlloc() and friends can be used to make handling of `ptr` safe against leaks in the case of an exception: @@ -621,90 +681,162 @@ ``` let result; const scope = scopedAllocPush(); try{ const ptr = scopedAlloc(4); - setMemValue(ptr, 0, 'i32'); + poke(ptr, 0, 'i32'); aCFuncWithOutputPtrArg( ptr ); - result = getMemValue(ptr, 'i32'); + result = peek(ptr, 'i32'); }finally{ scopedAllocPop(scope); } ``` - As a rule setMemValue() must be called to set (typically zero + As a rule poke() must be called to set (typically zero out) the pointer's value, else it will contain an essentially random value. ACHTUNG: calling this often, e.g. in a loop, can have a noticably painful impact on performance. Rather than doing so, use heapForSize() to fetch the heap object and read directly from it. - See: setMemValue() + See: poke() */ - target.getMemValue = function(ptr, type='i8'){ + target.peek = function f(ptr, type='i8'){ if(type.endsWith('*')) type = ptrIR; const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) ? cache : heapWrappers(); - switch(type){ - case 'i1': - case 'i8': return c.HEAP8[ptr>>0]; - case 'i16': return c.HEAP16[ptr>>1]; - case 'i32': return c.HEAP32[ptr>>2]; - case 'i64': - if(target.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); - break; - case 'float': case 'f32': return c.HEAP32F[ptr>>2]; - case 'double': case 'f64': return Number(c.HEAP64F[ptr>>3]); - default: break; - } - toss('Invalid type for getMemValue():',type); + const list = Array.isArray(ptr) ? [] : undefined; + let rc; + do{ + if(list) ptr = arguments[0].shift(); + switch(type){ + case 'i1': + case 'i8': rc = c.HEAP8[ptr>>0]; break; + case 'i16': rc = c.HEAP16[ptr>>1]; break; + case 'i32': rc = c.HEAP32[ptr>>2]; break; + case 'float': case 'f32': rc = c.HEAP32F[ptr>>2]; break; + case 'double': case 'f64': rc = Number(c.HEAP64F[ptr>>3]); break; + case 'i64': + if(target.bigIntEnabled){ + rc = BigInt(c.HEAP64[ptr>>3]); + break; + } + /* fallthru */ + default: + toss('Invalid type for peek():',type); + } + if(list) list.push(rc); + }while(list && arguments[0].length); + return list || rc; }; /** - The counterpart of getMemValue(), this sets a numeric value at + The counterpart of peek(), this sets a numeric value at the given WASM heap address, using the type to define how many bytes are written. Throws if given an invalid type. See - getMemValue() for details about the type argument. If the 3rd + peek() for details about the type argument. If the 3rd argument ends with `*` then it is treated as a pointer type and this function behaves as if the 3rd argument were `i32`. - This function returns itself. + If the first argument is an array, it is treated like a list + of pointers and the given value is written to each one. + + Returns `this`. (Prior to 2022-12-09 it returns this function.) ACHTUNG: calling this often, e.g. in a loop, can have a noticably painful impact on performance. Rather than doing so, use - heapForSize() to fetch the heap object and assign directly to it. + heapForSize() to fetch the heap object and assign directly to it + or use the heap's set() method. */ - target.setMemValue = function f(ptr, value, type='i8'){ + target.poke = function(ptr, value, type='i8'){ if (type.endsWith('*')) type = ptrIR; const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) ? cache : heapWrappers(); - switch (type) { - case 'i1': - case 'i8': c.HEAP8[ptr>>0] = value; return f; - case 'i16': c.HEAP16[ptr>>1] = value; return f; - case 'i32': c.HEAP32[ptr>>2] = value; return f; - case 'i64': - if(c.HEAP64){ - c.HEAP64[ptr>>3] = BigInt(value); - return f; - } - break; - case 'float': case 'f32': c.HEAP32F[ptr>>2] = value; return f; - case 'double': case 'f64': c.HEAP64F[ptr>>3] = value; return f; - } - toss('Invalid type for setMemValue(): ' + type); + for(const p of (Array.isArray(ptr) ? ptr : [ptr])){ + switch (type) { + case 'i1': + case 'i8': c.HEAP8[p>>0] = value; continue; + case 'i16': c.HEAP16[p>>1] = value; continue; + case 'i32': c.HEAP32[p>>2] = value; continue; + case 'float': case 'f32': c.HEAP32F[p>>2] = value; continue; + case 'double': case 'f64': c.HEAP64F[p>>3] = value; continue; + case 'i64': + if(c.HEAP64){ + c.HEAP64[p>>3] = BigInt(value); + continue; + } + /* fallthru */ + default: + toss('Invalid type for poke(): ' + type); + } + } + return this; }; - - /** Convenience form of getMemValue() intended for fetching - pointer-to-pointer values. */ - target.getPtrValue = (ptr)=>target.getMemValue(ptr, ptrIR); - - /** Convenience form of setMemValue() intended for setting - pointer-to-pointer values. */ - target.setPtrValue = (ptr, value)=>target.setMemValue(ptr, value, ptrIR); + /** + Convenience form of peek() intended for fetching + pointer-to-pointer values. If passed a single non-array argument + it returns the value of that one pointer address. If passed + multiple arguments, or a single array of arguments, it returns an + array of their values. + */ + target.peekPtr = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), ptrIR ); + + /** + A variant of poke() intended for setting pointer-to-pointer + values. Its differences from poke() are that (1) it defaults to a + value of 0 and (2) it always writes to the pointer-sized heap + view. + */ + target.pokePtr = (ptr, value=0)=>target.poke(ptr, value, ptrIR); + + /** + Convenience form of peek() intended for fetching i8 values. If + passed a single non-array argument it returns the value of that + one pointer address. If passed multiple arguments, or a single + array of arguments, it returns an array of their values. + */ + target.peek8 = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'i8' ); + /** + Convience form of poke() intended for setting individual bytes. + Its difference from poke() is that it always writes to the + i8-sized heap view. + */ + target.poke8 = (ptr, value)=>target.poke(ptr, value, 'i8'); + /** i16 variant of peek8(). */ + target.peek16 = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'i16' ); + /** i16 variant of poke8(). */ + target.poke16 = (ptr, value)=>target.poke(ptr, value, 'i16'); + /** i32 variant of peek8(). */ + target.peek32 = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'i32' ); + /** i32 variant of poke8(). */ + target.poke32 = (ptr, value)=>target.poke(ptr, value, 'i32'); + /** i64 variant of peek8(). Will throw if this build is not + configured for BigInt support. */ + target.peek64 = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'i64' ); + /** i64 variant of poke8(). Will throw if this build is not + configured for BigInt support. Note that this returns + a BigInt-type value, not a Number-type value. */ + target.poke64 = (ptr, value)=>target.poke(ptr, value, 'i64'); + /** f32 variant of peek8(). */ + target.peek32f = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'f32' ); + /** f32 variant of poke8(). */ + target.poke32f = (ptr, value)=>target.poke(ptr, value, 'f32'); + /** f64 variant of peek8(). */ + target.peek64f = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'f64' ); + /** f64 variant of poke8(). */ + target.poke64f = (ptr, value)=>target.poke(ptr, value, 'f64'); + + /** Deprecated alias for getMemValue() */ + target.getMemValue = target.peek; + /** Deprecated alias for peekPtr() */ + target.getPtrValue = target.peekPtr; + /** Deprecated alias for poke() */ + target.setMemValue = target.poke; + /** Deprecated alias for pokePtr() */ + target.setPtrValue = target.pokePtr; /** Returns true if the given value appears to be legal for use as a WASM pointer value. Its _range_ of values is not (cannot be) validated except to ensure that it is a 32-bit integer with a @@ -723,15 +855,16 @@ /** Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. Returns the length, in bytes, of the string, as for `strlen(3)`. - As a special case, if !ptr then it it returns `null`. Throws if - ptr is out of range for target.heap8u(). + As a special case, if !ptr or if it's not a pointer then it + returns `null`. Throws if ptr is out of range for + target.heap8u(). */ target.cstrlen = function(ptr){ - if(!ptr) return null; + if(!ptr || !target.isPtr(ptr)) return null; const h = heapWrappers().HEAP8U; let pos = ptr; for( ; h[pos] !== 0; ++pos ){} return pos - ptr; }; @@ -751,13 +884,13 @@ /** Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. This function counts its byte length using cstrlen() then returns a JS-format string representing its contents. As a special case, if - ptr is falsy, `null` is returned. + ptr is falsy or not a pointer, `null` is returned. */ - target.cstringToJs = function(ptr){ + target.cstrToJs = function(ptr){ const n = target.cstrlen(ptr); return n ? __utf8Decode(heapWrappers().HEAP8U, ptr, ptr+n) : (null===n ? n : ""); }; /** @@ -940,14 +1073,23 @@ }; const __allocCStr = function(jstr, returnWithLength, allocator, funcName){ __affirmAlloc(target, funcName); if('string'!==typeof jstr) return null; - const n = target.jstrlen(jstr), - ptr = allocator(n+1); - target.jstrcpy(jstr, target.heap8u(), ptr, n+1, true); - return returnWithLength ? [ptr, n] : ptr; + if(0){/* older impl, possibly more widely compatible? */ + const n = target.jstrlen(jstr), + ptr = allocator(n+1); + target.jstrcpy(jstr, target.heap8u(), ptr, n+1, true); + return returnWithLength ? [ptr, n] : ptr; + }else{/* newer, (probably) faster and (certainly) simpler impl */ + const u = cache.utf8Encoder.encode(jstr), + ptr = allocator(u.length+1), + heap = heapWrappers().HEAP8U; + heap.set(u, ptr); + heap[ptr + u.length] = 0; + return returnWithLength ? [ptr, u.length] : ptr; + } }; /** Uses target.alloc() to allocate enough memory for jstrlen(jstr)+1 bytes of memory, copies jstr to that memory using jstrcpy(), @@ -1033,11 +1175,17 @@ ? cache.scopedAlloc.indexOf(state) : cache.scopedAlloc.length-1; if(n<0) toss("Invalid state object for scopedAllocPop()."); if(0===arguments.length) state = cache.scopedAlloc[n]; cache.scopedAlloc.splice(n,1); - for(let p; (p = state.pop()); ) target.dealloc(p); + for(let p; (p = state.pop()); ){ + if(target.functionEntry(p)){ + //console.warn("scopedAllocPop() uninstalling transient function",p); + target.uninstallFunction(p); + } + else target.dealloc(p); + } }; /** Allocates n bytes of memory using this.alloc() and records that fact in the state for the most recent call of scopedAllocPush(). @@ -1079,43 +1227,70 @@ (jstr, returnWithLength=false)=>__allocCStr(jstr, returnWithLength, target.scopedAlloc, 'scopedAllocCString()'); // impl for allocMainArgv() and scopedAllocMainArgv(). const __allocMainArgv = function(isScoped, list){ - if(!list.length) toss("Cannot allocate empty array."); const pList = target[ isScoped ? 'scopedAlloc' : 'alloc' - ](list.length * target.ptrSizeof); + ]((list.length + 1) * target.ptrSizeof); let i = 0; list.forEach((e)=>{ - target.setPtrValue(pList + (target.ptrSizeof * i++), + target.pokePtr(pList + (target.ptrSizeof * i++), target[ isScoped ? 'scopedAllocCString' : 'allocCString' ](""+e)); }); + target.pokePtr(pList + (target.ptrSizeof * i), 0); return pList; }; /** Creates an array, using scopedAlloc(), suitable for passing to a C-level main() routine. The input is a collection with a length - property and a forEach() method. A block of memory list.length - entries long is allocated and each pointer-sized block of that - memory is populated with a scopedAllocCString() conversion of the - (""+value) of each element. Returns a pointer to the start of the - list, suitable for passing as the 2nd argument to a C-style - main() function. - - Throws if list.length is falsy or scopedAllocPush() is not active. + property and a forEach() method. A block of memory + (list.length+1) entries long is allocated and each pointer-sized + block of that memory is populated with a scopedAllocCString() + conversion of the (""+value) of each element, with the exception + that the final entry is a NULL pointer. Returns a pointer to the + start of the list, suitable for passing as the 2nd argument to a + C-style main() function. + + Throws if scopedAllocPush() is not active. + + Design note: the returned array is allocated with an extra NULL + pointer entry to accommodate certain APIs, but client code which + does not need that functionality should treat the returned array + as list.length entries long. */ target.scopedAllocMainArgv = (list)=>__allocMainArgv(true, list); /** Identical to scopedAllocMainArgv() but uses alloc() instead of - scopedAllocMainArgv + scopedAlloc(). */ target.allocMainArgv = (list)=>__allocMainArgv(false, list); + + /** + Expects to be given a C-style string array and its length. It + returns a JS array of strings and/or nulls: any entry in the + pArgv array which is NULL results in a null entry in the result + array. If argc is 0 then an empty array is returned. + + Results are undefined if any entry in the first argc entries of + pArgv are neither 0 (NULL) nor legal UTF-format C strings. + + To be clear, the expected C-style arguments to be passed to this + function are `(int, char **)` (optionally const-qualified). + */ + target.cArgvToJs = (argc, pArgv)=>{ + const list = []; + for(let i = 0; i < argc; ++i){ + const arg = target.peekPtr(pArgv + (target.ptrSizeof * i)); + list.push( arg ? target.cstrToJs(arg) : null ); + } + return list; + }; /** Wraps function call func() in a scopedAllocPush() and scopedAllocPop() block, such that all calls to scopedAlloc() and friends from within that call will have their memory freed @@ -1131,19 +1306,19 @@ /** Internal impl for allocPtr() and scopedAllocPtr(). */ const __allocPtr = function(howMany, safePtrSize, method){ __affirmAlloc(target, method); const pIr = safePtrSize ? 'i64' : ptrIR; let m = target[method](howMany * (safePtrSize ? 8 : ptrSizeof)); - target.setMemValue(m, 0, pIr) + target.poke(m, 0, pIr) if(1===howMany){ return m; } const a = [m]; for(let i = 1; i < howMany; ++i){ m += (safePtrSize ? 8 : ptrSizeof); a[i] = m; - target.setMemValue(m, 0, pIr); + target.poke(m, 0, pIr); } return a; }; /** @@ -1170,11 +1345,11 @@ The reason for the 2nd argument is.. When one of the returned pointers will refer to a 64-bit value, e.g. a double or int64, an that value must be written or fetched, - e.g. using setMemValue() or getMemValue(), it is important that + e.g. using poke() or peek(), it is important that the pointer in question be aligned to an 8-byte boundary or else it will not be fetched or written properly and will corrupt or read neighboring memory. It is only safe to pass false when the client code is certain that it will only get/fetch 4-byte values (or smaller). @@ -1197,11 +1372,11 @@ return target.exports[name] || toss("Cannot find exported symbol:",name); }; const __argcMismatch = (f,n)=>toss(f+"() requires",n,"argument(s)."); - + /** Looks up a WASM-exported function named fname from target.exports. If found, it is called, passed all remaining arguments, and its return value is returned to xCall's caller. If not found, an exception is thrown. This function does no @@ -1226,36 +1401,51 @@ /** State for use with xWrap() */ cache.xWrap = Object.create(null); - const xcv = cache.xWrap.convert = Object.create(null); + cache.xWrap.convert = Object.create(null); /** Map of type names to argument conversion functions. */ - cache.xWrap.convert.arg = Object.create(null); + cache.xWrap.convert.arg = new Map; /** Map of type names to return result conversion functions. */ - cache.xWrap.convert.result = Object.create(null); + cache.xWrap.convert.result = new Map; + const xArg = cache.xWrap.convert.arg, xResult = cache.xWrap.convert.result; if(target.bigIntEnabled){ - xcv.arg.i64 = (i)=>BigInt(i); - } - xcv.arg.i32 = (i)=>(i | 0); - xcv.arg.i16 = (i)=>((i | 0) & 0xFFFF); - xcv.arg.i8 = (i)=>((i | 0) & 0xFF); - xcv.arg.f32 = xcv.arg.float = (i)=>Number(i).valueOf(); - xcv.arg.f64 = xcv.arg.double = xcv.arg.f32; - xcv.arg.int = xcv.arg.i32; - xcv.result['*'] = xcv.result['pointer'] = xcv.arg['**'] = xcv.arg[ptrIR]; - xcv.result['number'] = (v)=>Number(v); - - { /* Copy certain xcv.arg[...] handlers to xcv.result[...] and + xArg.set('i64', (i)=>BigInt(i)); + } + const __xArgPtr = 'i32' === ptrIR + ? ((i)=>(i | 0)) : ((i)=>(BigInt(i) | BigInt(0))); + xArg.set('i32', __xArgPtr ) + .set('i16', (i)=>((i | 0) & 0xFFFF)) + .set('i8', (i)=>((i | 0) & 0xFF)) + .set('f32', (i)=>Number(i).valueOf()) + .set('float', xArg.get('f32')) + .set('f64', xArg.get('f32')) + .set('double', xArg.get('f64')) + .set('int', xArg.get('i32')) + .set('null', (i)=>i) + .set(null, xArg.get('null')) + .set('**', __xArgPtr) + .set('*', __xArgPtr); + xResult.set('*', __xArgPtr) + .set('pointer', __xArgPtr) + .set('number', (v)=>Number(v)) + .set('void', (v)=>undefined) + .set('null', (v)=>v) + .set(null, xResult.get('null')); + + { /* Copy certain xArg[...] handlers to xResult[...] and add pointer-style variants of them. */ const copyToResult = ['i8', 'i16', 'i32', 'int', 'f32', 'float', 'f64', 'double']; if(target.bigIntEnabled) copyToResult.push('i64'); + const adaptPtr = xArg.get(ptrIR); for(const t of copyToResult){ - xcv.arg[t+'*'] = xcv.result[t+'*'] = xcv.arg[ptrIR]; - xcv.result[t] = xcv.arg[t] || toss("Missing arg converter:",t); + xArg.set(t+'*', adaptPtr); + xResult.set(t+'*', adaptPtr); + xResult.set(t, (xArg.get(t) || toss("Missing arg converter:",t))); } } /** In order for args of type string to work in various contexts in @@ -1265,87 +1455,350 @@ argument is a string or not: - If v is a string, scopeAlloc() a new C-string from it and return that temp string's pointer. - - Else return the value from the arg adaptor defined for ptrIR. + - Else return the value from the arg adapter defined for ptrIR. TODO? Permit an Int8Array/Uint8Array and convert it to a string? Would that be too much magic concentrated in one place, ready to - backfire? - */ - xcv.arg.string = xcv.arg.utf8 = xcv.arg['pointer'] = xcv.arg['*'] - = function(v){ - if('string'===typeof v) return target.scopedAllocCString(v); - return v ? xcv.arg[ptrIR](v) : null; - }; - xcv.result.string = xcv.result.utf8 = (i)=>target.cstringToJs(i); - xcv.result['string:free'] = xcv.result['utf8:free'] = (i)=>{ - try { return i ? target.cstringToJs(i) : null } - finally{ target.dealloc(i) } - }; - xcv.result.json = (i)=>JSON.parse(target.cstringToJs(i)); - xcv.result['json:free'] = (i)=>{ - try{ return i ? JSON.parse(target.cstringToJs(i)) : null } - finally{ target.dealloc(i) } - } - xcv.result['void'] = (v)=>undefined; - xcv.result['null'] = (v)=>v; - - if(0){ - /*** - This idea can't currently work because we don't know the - signature for the func and don't have a way for the user to - convey it. To do this we likely need to be able to match - arg/result handlers by a regex, but that would incur an O(N) - cost as we check the regex one at a time. Another use case for - such a thing would be pseudotypes like "int:-1" to say that - the value will always be treated like -1 (which has a useful - case in the sqlite3 bindings). - */ - xcv.arg['func-ptr'] = function(v){ - if(!(v instanceof Function)) return xcv.arg[ptrIR]; - const f = target.jsFuncToWasm(v, WHAT_SIGNATURE); - }; - } + backfire? We handle that at the client level in sqlite3 with a + custom argument converter. + */ + const __xArgString = function(v){ + if('string'===typeof v) return target.scopedAllocCString(v); + return v ? __xArgPtr(v) : null; + }; + xArg.set('string', __xArgString) + .set('utf8', __xArgString) + .set('pointer', __xArgString); + //xArg.set('*', __xArgString); + + xResult.set('string', (i)=>target.cstrToJs(i)) + .set('utf8', xResult.get('string')) + .set('string:dealloc', (i)=>{ + try { return i ? target.cstrToJs(i) : null } + finally{ target.dealloc(i) } + }) + .set('utf8:dealloc', xResult.get('string:dealloc')) + .set('json', (i)=>JSON.parse(target.cstrToJs(i))) + .set('json:dealloc', (i)=>{ + try{ return i ? JSON.parse(target.cstrToJs(i)) : null } + finally{ target.dealloc(i) } + }); + + /** + Internal-use-only base class for FuncPtrAdapter and potentially + additional stateful argument adapter classes. + + Note that its main interface (convertArg()) is strictly + internal, not to be exposed to client code, as it may still + need re-shaping. Only the constructors of concrete subclasses + should be exposed to clients, and those in such a way that + does not hinder internal redesign of the convertArg() + interface. + */ + const AbstractArgAdapter = class { + constructor(opt){ + this.name = opt.name || 'unnamed adapter'; + } + /** + Gets called via xWrap() to "convert" v to whatever type + this specific class supports. + + argIndex is the argv index of _this_ argument in the + being-xWrap()'d call. argv is the current argument list + undergoing xWrap() argument conversion. argv entries to the + left of argIndex will have already undergone transformation and + those to the right will not have (they will have the values the + client-level code passed in, awaiting conversion). The RHS + indexes must never be relied upon for anything because their + types are indeterminate, whereas the LHS values will be + WASM-compatible values by the time this is called. + */ + convertArg(v,argv,argIndex){ + toss("AbstractArgAdapter must be subclassed."); + } + }; + + /** + An attempt at adding function pointer conversion support to + xWrap(). This type is recognized by xWrap() as a proxy for + converting a JS function to a C-side function, either + permanently, for the duration of a single call into the C layer, + or semi-contextual, where it may keep track of a single binding + for a given context and uninstall the binding if it's replaced. + + The constructor requires an options object with these properties: + + - name (optional): string describing the function binding. This + is solely for debugging and error-reporting purposes. If not + provided, an empty string is assumed. + + - signature: a function signature string compatible with + jsFuncToWasm(). + + - bindScope (string): one of ('transient', 'context', + 'singleton'). Bind scopes are: + + - 'transient': it will convert JS functions to WASM only for + the duration of the xWrap()'d function call, using + scopedInstallFunction(). Before that call returns, the + WASM-side binding will be uninstalled. + + - 'singleton': holds one function-pointer binding for this + instance. If it's called with a different function pointer, + it uninstalls the previous one after converting the new + value. This is only useful for use with "global" functions + which do not rely on any state other than this function + pointer. If the being-converted function pointer is intended + to be mapped to some sort of state object (e.g. an + `sqlite3*`) then "context" (see below) is the proper mode. + + - 'context': similar to singleton mode but for a given + "context", where the context is a key provided by the user + and possibly dependent on a small amount of call-time + context. This mode is the default if bindScope is _not_ set + but a property named contextKey (described below) is. + + - callProxy (function): if set, this must be a function which + will act as a proxy for any "converted" JS function. It is + passed the being-converted function value and must return + either that function or a function which acts on its + behalf. The returned function will be the one which gets + installed into the WASM function table. The proxy must perform + any required argument conversion (noting that it will be called + from C code, so will receive C-format arguments) before passing + them on to the being-converted function. Whether or not the + proxy itself must return a value depends on the context. If it + does, it must be a WASM-friendly value, as it will be returning + from a call made from native code. + + - contextKey (function): is only used if bindScope is 'context' + or if bindScope is not set and this function is, in which case + 'context' is assumed. This function gets bound to this object, + so its "this" is this object. It gets passed (argv,argIndex), + where argIndex is the index of _this_ function pointer in its + _wrapping_ function's arguments and argv is the _current_ + still-being-xWrap()-processed args array. All arguments to the + left of argIndex will have been processed by xWrap() by the + time this is called. argv[argIndex] will be the value the user + passed in to the xWrap()'d function for the argument this + FuncPtrAdapter is mapped to. Arguments to the right of + argv[argIndex] will not yet have been converted before this is + called. The function must return a key which uniquely + identifies this function mapping context for _this_ + FuncPtrAdapter instance (other instances are not considered), + taking into account that C functions often take some sort of + state object as one or more of their arguments. As an example, + if the xWrap()'d function takes `(int,T*,functionPtr,X*)` and + this FuncPtrAdapter is the argv[2]nd arg, contextKey(argv,2) + might return 'T@'+argv[1], or even just argv[1]. Note, + however, that the (X*) argument will not yet have been + processed by the time this is called and should not be used as + part of that key because its pre-conversion data type might be + unpredictable. Similarly, care must be taken with C-string-type + arguments: those to the left in argv will, when this is called, + be WASM pointers, whereas those to the right might (and likely + do) have another data type. When using C-strings in keys, never + use their pointers in the key because most C-strings in this + constellation are transient. + + Yes, that ^^^ is quite awkward, but it's what we have. + + The constructor only saves the above state for later, and does + not actually bind any functions. Its convertArg() method is + called via xWrap() to perform any bindings. + + Shortcomings: + + - These "reverse" bindings, i.e. calling into a JS-defined + function from a WASM-defined function (the generated proxy + wrapper), lack all type conversion support. That means, for + example, that... + + - Function pointers which include C-string arguments may still + need a level of hand-written wrappers around them, depending on + how they're used, in order to provide the client with JS + strings. Alternately, clients will need to perform such conversions + on their own, e.g. using cstrtojs(). Or maybe we can find a way + to perform such conversions here, via addition of an xWrap()-style + function signature to the options argument. + */ + xArg.FuncPtrAdapter = class FuncPtrAdapter extends AbstractArgAdapter { + constructor(opt) { + super(opt); + if(xArg.FuncPtrAdapter.warnOnUse){ + console.warn('xArg.FuncPtrAdapter is an internal-only API', + 'and is not intended to be invoked from', + 'client-level code. Invoked with:',opt); + } + this.signature = opt.signature; + if(opt.contextKey instanceof Function){ + this.contextKey = opt.contextKey; + if(!opt.bindScope) opt.bindScope = 'context'; + } + this.bindScope = opt.bindScope + || toss("FuncPtrAdapter options requires a bindScope (explicit or implied)."); + if(FuncPtrAdapter.bindScopes.indexOf(opt.bindScope)<0){ + toss("Invalid options.bindScope ("+opt.bindMod+") for FuncPtrAdapter. "+ + "Expecting one of: ("+FuncPtrAdapter.bindScopes.join(', ')+')'); + } + this.isTransient = 'transient'===this.bindScope; + this.isContext = 'context'===this.bindScope; + this.singleton = ('singleton'===this.bindScope) ? [] : undefined; + //console.warn("FuncPtrAdapter()",opt,this); + this.callProxy = (opt.callProxy instanceof Function) + ? opt.callProxy : undefined; + } + + /** If true, the constructor emits a warning. The intent is that + this be set to true after bootstrapping of the higher-level + client library is complete, to warn downstream clients that + they shouldn't be relying on this implemenation detail which + does not have a stable interface. */ + static warnOnUse = false; + + /** If true, convertArg() will FuncPtrAdapter.debugOut() when it + (un)installs a function binding to/from WASM. + */ + static debugFuncInstall = false; + + /** Function used for debug output. */ + static debugOut = console.debug.bind(console); + + static bindScopes = [ + 'transient', 'context', 'singleton' + ]; + + /* Dummy impl. Overwritten per-instance as needed. */ + contextKey(argv,argIndex){ + return this; + } + + /* Returns this objects mapping for the given context key, in the + form of an an array, creating the mapping if needed. The key + may be anything suitable for use in a Map. */ + contextMap(key){ + const cm = (this.__cmap || (this.__cmap = new Map)); + let rc = cm.get(key); + if(undefined===rc) cm.set(key, (rc = [])); + return rc; + } + + /** + Gets called via xWrap() to "convert" v to a WASM-bound function + pointer. If v is one of (a pointer, null, undefined) then + (v||0) is returned and any earlier function installed by this + mapping _might_, depending on how it's bound, be uninstalled. + If v is not one of those types, it must be a Function, for + which it creates (if needed) a WASM function binding and + returns the WASM pointer to that binding. If this instance is + not in 'transient' mode, it will remember the binding for at + least the next call, to avoid recreating the function binding + unnecessarily. + + If it's passed a pointer(ish) value for v, it does _not_ + perform any function binding, so this object's bindMode is + irrelevant for such cases. + + See the parent class's convertArg() docs for details on what + exactly the 2nd and 3rd arguments are. + */ + convertArg(v,argv,argIndex){ + //FuncPtrAdapter.debugOut("FuncPtrAdapter.convertArg()",this.signature,this.transient,v); + let pair = this.singleton; + if(!pair && this.isContext){ + pair = this.contextMap(this.contextKey(argv,argIndex)); + } + if(pair && pair[0]===v) return pair[1]; + if(v instanceof Function){ + /* Install a WASM binding and return its pointer. */ + if(this.callProxy) v = this.callProxy(v); + const fp = __installFunction(v, this.signature, this.isTransient); + if(FuncPtrAdapter.debugFuncInstall){ + FuncPtrAdapter.debugOut("FuncPtrAdapter installed", this, + this.contextKey(argv,argIndex), '@'+fp, v); + } + if(pair){ + /* Replace existing stashed mapping */ + if(pair[1]){ + if(FuncPtrAdapter.debugFuncInstall){ + FuncPtrAdapter.debugOut("FuncPtrAdapter uninstalling", this, + this.contextKey(argv,argIndex), '@'+pair[1], v); + } + try{target.uninstallFunction(pair[1])} + catch(e){/*ignored*/} + } + pair[0] = v; + pair[1] = fp; + } + return fp; + }else if(target.isPtr(v) || null===v || undefined===v){ + if(pair && pair[1] && pair[1]!==v){ + /* uninstall stashed mapping and replace stashed mapping with v. */ + if(FuncPtrAdapter.debugFuncInstall){ + FuncPtrAdapter.debugOut("FuncPtrAdapter uninstalling", this, + this.contextKey(argv,argIndex), '@'+pair[1], v); + } + try{target.uninstallFunction(pair[1])} + catch(e){/*ignored*/} + pair[0] = pair[1] = (v | 0); + } + return v || 0; + }else{ + throw new TypeError("Invalid FuncPtrAdapter argument type. "+ + "Expecting a function pointer or a "+ + (this.name ? this.name+' ' : '')+ + "function matching signature "+ + this.signature+"."); + } + }/*convertArg()*/ + }/*FuncPtrAdapter*/; const __xArgAdapterCheck = - (t)=>xcv.arg[t] || toss("Argument adapter not found:",t); + (t)=>xArg.get(t) || toss("Argument adapter not found:",t); const __xResultAdapterCheck = - (t)=>xcv.result[t] || toss("Result adapter not found:",t); - - cache.xWrap.convertArg = (t,v)=>__xArgAdapterCheck(t)(v); + (t)=>xResult.get(t) || toss("Result adapter not found:",t); + + cache.xWrap.convertArg = (t,...args)=>__xArgAdapterCheck(t)(...args); + cache.xWrap.convertArgNoCheck = (t,...args)=>xArg.get(t)(...args); + cache.xWrap.convertResult = (t,v)=>(null===t ? v : (t ? __xResultAdapterCheck(t)(v) : undefined)); + cache.xWrap.convertResultNoCheck = + (t,v)=>(null===t ? v : (t ? xResult.get(t)(v) : undefined)); /** - Creates a wrapper for the WASM-exported function fname. Uses - xGet() to fetch the exported function (which throws on - error) and returns either that function or a wrapper for that + Creates a wrapper for another function which converts the arguments + of the wrapper to argument types accepted by the wrapped function, + then converts the wrapped function's result to another form + for the wrapper. + + The first argument must be one of: + + - A JavaScript function. + - The name of a WASM-exported function. In the latter case xGet() + is used to fetch the exported function, which throws if it's not + found. + - A pointer into the indirect function table. e.g. a pointer + returned from target.installFunction(). + + It returns either the passed-in function or a wrapper for that function which converts the JS-side argument types into WASM-side - types and converts the result type. If the function takes no - arguments and resultType is `null` then the function is returned - as-is, else a wrapper is created for it to adapt its arguments - and result value, as described below. - - (If you're familiar with Emscripten's ccall() and cwrap(), this - function is essentially cwrap() on steroids.) - - This function's arguments are: - - - fname: the exported function's name. xGet() is used to fetch - this, so will throw if no exported function is found with that - name. - - - resultType: the name of the result type. A literal `null` means - to return the original function's value as-is (mnemonic: there - is "null" conversion going on). Literal `undefined` or the - string `"void"` mean to ignore the function's result and return - `undefined`. Aside from those two special cases, it may be one - of the values described below or any mapping installed by the - client using xWrap.resultAdapter(). + types and converts the result type. + + The second argument, `resultType`, describes the conversion for + the wrapped functions result. A literal `null` or the string + `'null'` both mean to return the original function's value as-is + (mnemonic: there is "null" conversion going on). Literal + `undefined` or the string `"void"` both mean to ignore the + function's result and return `undefined`. Aside from those two + special cases, the `resultType` value may be one of the values + described below or any mapping installed by the client using + xWrap.resultAdapter(). If passed 3 arguments and the final one is an array, that array must contain a list of type names (see below) for adapting the arguments from JS to WASM. If passed 2 arguments, more than 3, or the 3rd is not an array, all arguments after the 2nd (if any) @@ -1354,10 +1807,16 @@ ``` xWrap('funcname', 'i32', 'string', 'f64'); // is equivalent to: xWrap('funcname', 'i32', ['string', 'f64']); ``` + + This function enforces that the given list of arguments has the + same arity as the being-wrapped function (as defined by its + `length` property) and it will throw if that is not the case. + Similarly, the created wrapper will throw if passed a differing + argument count. Type names are symbolic names which map the arguments to an adapter function to convert, if needed, the value before passing it on to WASM or to convert a return result from WASM. The list of built-in names: @@ -1367,14 +1826,17 @@ the given bit length. - `N*` (args): a type name in the form `N*`, where N is a numeric type name, is treated the same as WASM pointer. - - `*` and `pointer` (args): have multple semantics. They - behave exactly as described below for `string` args. + - `*` and `pointer` (args): are assumed to be WASM pointer values + and are returned coerced to an appropriately-sized pointer + value (i32 or i64). Non-numeric values will coerce to 0 and + out-of-range values will have undefined results (just as with + any pointer misuse). - - `*` and `pointer` (results): are aliases for the current + - `*` and `pointer` (results): aliases for the current WASM pointer numeric type. - `**` (args): is simply a descriptive alias for the WASM pointer type. It's primarily intended to mark output-pointer arguments. @@ -1381,19 +1843,23 @@ - `i64` (args and results): passes the value to BigInt() to convert it to an int64. Only available if bigIntEnabled is true. - `f32` (`float`), `f64` (`double`) (args and results): pass - their argument to Number(). i.e. the adaptor does not currently + their argument to Number(). i.e. the adapter does not currently distinguish between the two types of floating-point numbers. - `number` (results): converts the result to a JS Number using Number(theValue).valueOf(). Note that this is for result conversions only, as it's not possible to generically know which type of number to convert arguments to. Non-numeric conversions include: + + - `null` literal or `"null"` string (args and results): perform + no translation and pass the arg on as-is. This is primarily + useful for results but may have a use or two for arguments. - `string` or `utf8` (args): has two different semantics in order to accommodate various uses of certain C APIs (e.g. output-style strings)... @@ -1405,15 +1871,15 @@ - Else the arg is assumed to be a pointer to a string the client has already allocated and it's passed on as a WASM pointer. - - `string` or `utf8` (results): treats the result value as a - const C-string, encoded as UTF-8, copies it to a JS string, - and returns that JS string. + - `string` or `utf8` (results): treats the result value as a + const C-string, encoded as UTF-8, copies it to a JS string, + and returns that JS string. - - `string:free` or `utf8:free) (results): treats the result value + - `string:dealloc` or `utf8:dealloc) (results): treats the result value as a non-const UTF-8 C-string, ownership of which has just been transfered to the caller. It copies the C-string to a JS string, frees the C-string, and returns the JS string. If such a result value is NULL, the JS result is `null`. Achtung: when using an API which returns results from a specific allocator, @@ -1420,41 +1886,46 @@ e.g. `my_malloc()`, this conversion _is not legal_. Instead, an equivalent conversion which uses the appropriate deallocator is required. For example: ```js - target.xWrap.resultAdaptor('string:my_free',(i)=>{ - try { return i ? target.cstringToJs(i) : null } + target.xWrap.resultAdapter('string:my_free',(i)=>{ + try { return i ? target.cstrToJs(i) : null } finally{ target.exports.my_free(i) } }; ``` - `json` (results): treats the result as a const C-string and returns the result of passing the converted-to-JS string to JSON.parse(). Returns `null` if the C-string is a NULL pointer. - - `json:free` (results): works exactly like `string:free` but + - `json:dealloc` (results): works exactly like `string:dealloc` but returns the same thing as the `json` adapter. Note the - warning in `string:free` regarding maching allocators and + warning in `string:dealloc` regarding maching allocators and deallocators. The type names for results and arguments are validated when xWrap() is called and any unknown names will trigger an exception. Clients may map their own result and argument adapters using - xWrap.resultAdapter() and xWrap.argAdaptor(), noting that not all + xWrap.resultAdapter() and xWrap.argAdapter(), noting that not all type conversions are valid for both arguments _and_ result types as they often have different memory ownership requirements. + + Design note: the ability to pass in a JS function as the first + argument is of relatively limited use, primarily for testing + argument and result converters. JS functions, by and large, will + not want to deal with C-type arguments. TODOs: - Figure out how/whether we can (semi-)transparently handle pointer-type _output_ arguments. Those currently require explicit handling by allocating pointers, assigning them before - the call using setMemValue(), and fetching them with - getMemValue() after the call. We may be able to automate some + the call using poke(), and fetching them with + peek() after the call. We may be able to automate some or all of that. - Figure out whether it makes sense to extend the arg adapter interface such that each arg adapter gets an array containing the results of the previous arguments in the current call. That @@ -1464,53 +1935,82 @@ argument. Currently that distinction requires hand-writing a wrapper for that function. That case is unusual enough that abstracting it into this API (and taking on the associated costs) may well not make good sense. */ - target.xWrap = function(fname, resultType, ...argTypes){ + target.xWrap = function(fArg, resultType, ...argTypes){ if(3===arguments.length && Array.isArray(arguments[2])){ argTypes = arguments[2]; } - const xf = target.xGet(fname); - if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length); + if(target.isPtr(fArg)){ + fArg = target.functionEntry(fArg) + || toss("Function pointer not found in WASM function table."); + } + const fIsFunc = (fArg instanceof Function); + const xf = fIsFunc ? fArg : target.xGet(fArg); + if(fIsFunc) fArg = xf.name || 'unnamed function'; + if(argTypes.length!==xf.length) __argcMismatch(fArg, xf.length); if((null===resultType) && 0===xf.length){ - /* Func taking no args with an as-is return. We don't need a wrapper. */ + /* Func taking no args with an as-is return. We don't need a wrapper. + We forego the argc check here, though. */ return xf; } /*Verify the arg type conversions are valid...*/; if(undefined!==resultType && null!==resultType) __xResultAdapterCheck(resultType); - argTypes.forEach(__xArgAdapterCheck); + for(const t of argTypes){ + if(t instanceof AbstractArgAdapter) xArg.set(t, (...args)=>t.convertArg(...args)); + else __xArgAdapterCheck(t); + } + const cxw = cache.xWrap; if(0===xf.length){ // No args to convert, so we can create a simpler wrapper... return (...args)=>(args.length - ? __argcMismatch(fname, xf.length) - : cache.xWrap.convertResult(resultType, xf.call(null))); + ? __argcMismatch(fArg, xf.length) + : cxw.convertResult(resultType, xf.call(null))); } return function(...args){ - if(args.length!==xf.length) __argcMismatch(fname, xf.length); + if(args.length!==xf.length) __argcMismatch(fArg, xf.length); const scope = target.scopedAllocPush(); try{ - const rc = xf.apply(null,args.map((v,i)=>cache.xWrap.convertArg(argTypes[i], v))); - return cache.xWrap.convertResult(resultType, rc); + /* + Maintenance reminder re. arguments passed to convertArg(): + The public interface of argument adapters is that they take + ONE argument and return a (possibly) converted result for + it. The passing-on of arguments after the first is an + internal implementation detail for the sake of + AbstractArgAdapter, and not to be relied on or documented + for other cases. The fact that this is how + AbstractArgAdapter.convertArgs() gets its 2nd+ arguments, + and how FuncPtrAdapter.contextKey() gets its args, is also + an implementation detail and subject to change. i.e. the + public interface of 1 argument is stable. The fact that any + arguments may be passed in after that one, and what those + arguments are, is _not_ part of the public interface and is + _not_ stable. + */ + for(const i in args) args[i] = cxw.convertArgNoCheck( + argTypes[i], args[i], args, i + ); + return cxw.convertResultNoCheck(resultType, xf.apply(null,args)); }finally{ target.scopedAllocPop(scope); } }; }/*xWrap()*/; - /** Internal impl for xWrap.resultAdapter() and argAdaptor(). */ + /** Internal impl for xWrap.resultAdapter() and argAdapter(). */ const __xAdapter = function(func, argc, typeName, adapter, modeName, xcvPart){ if('string'===typeof typeName){ - if(1===argc) return xcvPart[typeName]; + if(1===argc) return xcvPart.get(typeName); else if(2===argc){ if(!adapter){ - delete xcvPart[typeName]; + delete xcvPart.get(typeName); return func; }else if(!(adapter instanceof Function)){ toss(modeName,"requires a function argument."); } - xcvPart[typeName] = adapter; + xcvPart.set(typeName, adapter); return func; } } toss("Invalid arguments to",modeName); }; @@ -1543,11 +2043,11 @@ Except when called as a getter, this function returns itself. */ target.xWrap.resultAdapter = function f(typeName, adapter){ return __xAdapter(f, arguments.length, typeName, adapter, - 'resultAdaptor()', xcv.result); + 'resultAdapter()', xResult); }; /** Functions identically to xWrap.resultAdapter() but applies to call argument conversions instead of result value conversions. @@ -1573,36 +2073,67 @@ perform argument validation, as opposed (or in addition) to conversion. */ target.xWrap.argAdapter = function f(typeName, adapter){ return __xAdapter(f, arguments.length, typeName, adapter, - 'argAdaptor()', xcv.arg); + 'argAdapter()', xArg); }; + + target.xWrap.FuncPtrAdapter = xArg.FuncPtrAdapter; /** Functions like xCall() but performs argument and result type - conversions as for xWrap(). The first argument is the name of the - exported function to call. The 2nd its the name of its result - type, as documented for xWrap(). The 3rd is an array of argument - type name, as documented for xWrap() (use a falsy value or an - empty array for nullary functions). The 4th+ arguments are - arguments for the call, with the special case that if the 4th - argument is an array, it is used as the arguments for the - call. Returns the converted result of the call. + conversions as for xWrap(). The first, second, and third + arguments are as documented for xWrap(), except that the 3rd + argument may be either a falsy value or empty array to represent + nullary functions. The 4th+ arguments are arguments for the call, + with the special case that if the 4th argument is an array, it is + used as the arguments for the call. Returns the converted result + of the call. This is just a thin wrapper around xWrap(). If the given function is to be called more than once, it's more efficient to use xWrap() to create a wrapper, then to call that wrapper as many times as needed. For one-shot calls, however, this variant is arguably more efficient because it will hypothetically free the wrapper function quickly. */ - target.xCallWrapped = function(fname, resultType, argTypes, ...args){ + target.xCallWrapped = function(fArg, resultType, argTypes, ...args){ if(Array.isArray(arguments[3])) args = arguments[3]; - return target.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); + return target.xWrap(fArg, resultType, argTypes||[]).apply(null, args||[]); }; + /** + This function is ONLY exposed in the public API to facilitate + testing. It should not be used in application-level code, only + in test code. + + Expects to be given (typeName, value) and returns a conversion + of that value as has been registered using argAdapter(). + It throws if no adapter is found. + + ACHTUNG: the adapter may require that a scopedAllocPush() is + active and it may allocate memory within that scope. It may also + require additional arguments, depending on the type of + conversion. + */ + target.xWrap.testConvertArg = cache.xWrap.convertArg; + + /** + This function is ONLY exposed in the public API to facilitate + testing. It should not be used in application-level code, only + in test code. + + Expects to be given (typeName, value) and returns a conversion + of that value as has been registered using resultAdapter(). + It throws if no adapter is found. + + ACHTUNG: the adapter may allocate memory which the caller may need + to know how to free. + */ + target.xWrap.testConvertResult = cache.xWrap.convertResult; + return target; }; /** yawl (Yet Another Wasm Loader) provides very basic wasm loader. Index: ext/wasm/demo-worker1-promiser.js ================================================================== --- ext/wasm/demo-worker1-promiser.js +++ ext/wasm/demo-worker1-promiser.js @@ -74,13 +74,11 @@ let sqConfig; await wtest('config-get', (ev)=>{ const r = ev.result; log('sqlite3.config subset:', r); - T.assert('boolean' === typeof r.bigIntEnabled) - .assert('string'===typeof r.wasmfsOpfsDir) - .assert('boolean' === typeof r.wasmfsOpfsEnabled); + T.assert('boolean' === typeof r.bigIntEnabled); sqConfig = r; }); logHtml('', "Sending 'open' message and waiting for its response before continuing..."); @@ -211,11 +209,11 @@ }; await wtest('exec',{ sql: "select 'foo' foo, a bar, 'baz' baz from t limit 2", callback: resultRowTest3, columnNames: [], - rowMode: ':bar' + rowMode: '$bar' }, function(ev){ log("exec() result row:",ev); T.assert(2===resultRowTest3.counter); }); Index: ext/wasm/dist.make ================================================================== --- ext/wasm/dist.make +++ ext/wasm/dist.make @@ -4,64 +4,68 @@ # Intended to include'd by ./GNUmakefile. # # 'make dist' rules for creating a distribution archive of the WASM/JS # pieces, noting that we only build a dist of the built files, not the # numerous pieces required to build them. +# +# Use 'make snapshot' to create "snapshot" releases. They use a +# distinctly different zip file and top directory name to distinguish +# them from release builds. ####################################################################### MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) ######################################################################## -# Chicken/egg situation: we need $(bin.version-info) to get the version -# info for the archive name, but that binary may not yet be built, and -# won't be built until we expand the dependencies. We have to use a -# temporary name for the archive. -dist-name = sqlite-wasm-TEMP -#ifeq (0,1) -# $(info WARNING *******************************************************************) -# $(info ** Be sure to create the desired build configuration before creating the) -# $(info ** distribution archive. Use one of the following targets to do so:) -# $(info **) -# $(info ** o2: builds with -O2, resulting in the fastest builds) -# $(info ** oz: builds with -Oz, resulting in the smallest builds) -# $(info /WARNING *******************************************************************) -#endif +# Chicken/egg situation: we need $(bin.version-info) to get the +# version info for the archive name, but that binary may not yet be +# built, and won't be built until we expand the dependencies. Thus we +# have to use a temporary name for the archive until we can get +# that binary built. +ifeq (,$(filter snapshot,$(MAKECMDGOALS))) +dist-name-prefix := sqlite-wasm +else +dist-name-prefix := sqlite-wasm-snapshot-$(shell /usr/bin/date +%Y%m%d) +endif +dist-name := $(dist-name-prefix)-TEMP ######################################################################## -# dist.build must be the name of a target which triggers the -# build of the files to be packed into the dist archive. The -# intention is that it be one of (o0, o1, o2, o3, os, oz), each of -# which uses like-named -Ox optimization level flags. The o2 target -# provides the best overall runtime speeds. The oz target provides -# slightly slower speeds (roughly 10%) with significantly smaller WASM -# file sizes. Note that -O2 (the o2 target) results in faster binaries -# than both -O3 and -Os (the o3 and os targets) in all tests run to -# date. -dist.build ?= oz +# dist.build must be the name of a target which triggers the build of +# the files to be packed into the dist archive. The intention is that +# it be one of (o0, o1, o2, o3, os, oz), each of which uses like-named +# -Ox optimization level flags. The o2 target provides the best +# overall runtime speeds. The oz target provides slightly slower +# speeds (roughly 10%) with significantly smaller WASM file +# sizes. Note that -O2 (the o2 target) results in faster binaries than +# both -O3 and -Os (the o3 and os targets) in all tests run to +# date. Our general policy is that we want the smallest binaries for +# dist zip files, so use the oz build unless there is a compelling +# reason not to. +dist.build ?= qoz dist-dir.top := $(dist-name) dist-dir.jswasm := $(dist-dir.top)/$(notdir $(dir.dout)) dist-dir.common := $(dist-dir.top)/common dist.top.extras := \ demo-123.html demo-123-worker.html demo-123.js \ - tester1.html tester1-worker.html tester1.js \ + tester1.html tester1-worker.html tester1-esm.html \ + tester1.js tester1.mjs \ demo-jsstorage.html demo-jsstorage.js \ demo-worker1.html demo-worker1.js \ demo-worker1-promiser.html demo-worker1-promiser.js dist.jswasm.extras := $(sqlite3-api.ext.jses) $(sqlite3.wasm) dist.common.extras := \ $(wildcard $(dir.common)/*.css) \ $(dir.common)/SqliteTestUtil.js -.PHONY: dist +.PHONY: dist snapshot ######################################################################## # dist: create the end-user deliverable archive. # # Maintenance reminder: because dist depends on $(dist.build), and # $(dist.build) will depend on clean, having any deps on # $(dist-archive) which themselves may be cleaned up by the clean # target will lead to grief in parallel builds (-j #). Thus -# $(dist-target)'s deps must be trimmed to non-generated files or +# dist's deps must be trimmed to non-generated files or # files which are _not_ cleaned up by the clean target. # # Note that we require $(bin.version-info) in order to figure out the # dist file's name, so cannot (without a recursive make) have the # target name equal to the archive name. @@ -76,26 +80,38 @@ @cp -p README-dist.txt $(dist-dir.top)/README.txt @cp -p index-dist.html $(dist-dir.top)/index.html @cp -p $(dist.jswasm.extras) $(dist-dir.jswasm) @$(bin.stripccomments) -k -k < $(sqlite3.js) \ > $(dist-dir.jswasm)/$(notdir $(sqlite3.js)) + @$(bin.stripccomments) -k -k < $(sqlite3.mjs) \ + > $(dist-dir.jswasm)/$(notdir $(sqlite3.mjs)) @cp -p $(dist.common.extras) $(dist-dir.common) @set -e; \ vnum=$$($(bin.version-info) --download-version); \ - vdir=sqlite-wasm-$$vnum; \ + vdir=$(dist-name-prefix)-$$vnum; \ arczip=$$vdir.zip; \ echo "Making $$arczip ..."; \ rm -fr $$arczip $$vdir; \ mv $(dist-dir.top) $$vdir; \ zip -qr $$arczip $$vdir; \ rm -fr $$vdir; \ ls -la $$arczip; \ set +e; \ unzip -lv $$arczip || echo "Missing unzip app? Not fatal." - +ifeq (,$(wasm.docs.found)) +snapshot: dist + @echo "To upload the snapshot build to the wasm docs server:"; \ + echo "1) move $(dist-name-prefix)*.zip to the top of a wasm docs checkout."; \ + echo "2) run 'make uv-sync'" +else +snapshot: dist + @echo "Moving snapshot to [$(wasm.docs.found)]..."; \ + mv $(dist-name-prefix)*.zip $(wasm.docs.found)/. + @echo "Run 'make uv-sync' from $(wasm.docs.found) to upload it." +endif # We need a separate `clean` rule to account for weirdness in # a sub-make, where we get a copy of the $(dist-name) dir # copied into the new $(dist-name) dir. .PHONY: dist-clean clean: dist-clean dist-clean: rm -fr $(dist-name) $(wildcard sqlite-wasm-*.zip) Index: ext/wasm/fiddle.make ================================================================== --- ext/wasm/fiddle.make +++ ext/wasm/fiddle.make @@ -37,10 +37,11 @@ -sEXPORT_NAME=$(sqlite3.js.init-func) \ -Wno-limited-postlink-optimizations \ $(sqlite3.js.flags.--post-js) \ $(emcc.exportedRuntimeMethods) \ -sEXPORTED_FUNCTIONS=@$(abspath $(EXPORTED_FUNCTIONS.fiddle)) \ + -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory \ $(SQLITE_OPT) $(SHELL_OPT) \ -DSQLITE_SHELL_FIDDLE # -D_POSIX_C_SOURCE is needed for strdup() with emcc fiddle.EXPORTED_FUNCTIONS.in := \ @@ -56,16 +57,16 @@ fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) $(fiddle.SOAP.js): $(SOAP.js) cp $< $@ -$(eval $(call call-make-pre-js,fiddle-module)) +$(eval $(call call-make-pre-js,fiddle-module,vanilla)) $(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \ $(EXPORTED_FUNCTIONS.fiddle) \ - $(fiddle.cses) $(pre-post-fiddle-module.deps) $(fiddle.SOAP.js) + $(fiddle.cses) $(pre-post-fiddle-module.deps.vanilla) $(fiddle.SOAP.js) $(emcc.bin) -o $@ $(fiddle.emcc-flags) \ - $(pre-post-common.flags) $(pre-post-fiddle-module.flags) \ + $(pre-post-fiddle-module.flags.vanilla) \ $(fiddle.cses) $(maybe-wasm-strip) $(fiddle-module.wasm) gzip < $@ > $@.gz gzip < $(fiddle-module.wasm) > $(fiddle-module.wasm).gz Index: ext/wasm/index-dist.html ================================================================== --- ext/wasm/index-dist.html +++ ext/wasm/index-dist.html @@ -17,10 +17,14 @@ font-family: monospace; } header { font-size: 130%; font-weight: bold; + background: #044a64; + color: white; + padding: 0.5em; + border-radius: 0.25em; } .hidden, .initially-hidden { position: absolute !important; opacity: 0 !important; pointer-events: none !important; @@ -54,10 +58,20 @@
  • tester1: Core unit and regression tests for the various APIs and surrounding utility code.
  • tester1-worker: same thing but running in a Worker.
    • +
  • tester1-esm: same as + tester1 but loads sqlite3 in the main thread via + an ES6 module. +
    • +
  • tester1-worker?esm: + same as tester1-esm but loads a Worker Module which + then loads the sqlite3 API via an ES6 module. Note that + not all browsers permit loading modules in Worker + threads. +
  • Higher-level apps and demos...
    • demo-123 provides a Index: ext/wasm/index.html ================================================================== --- ext/wasm/index.html +++ ext/wasm/index.html @@ -6,10 +6,18 @@ sqlite3 WASM Testing Page Index +
      sqlite3 WASM test pages
      Below is the list of test pages for the sqlite3 WASM builds. All of them require that this directory have been "make"d first. The intent is that this page be run @@ -28,15 +36,11 @@
    • Any OPFS-related pages require very recent version of Chrome or Chromium (v102 at least, possibly newer). OPFS support in the other major browsers is pending. Development and testing is currently done against a dev-channel release - of Chrome (v107 as of 2022-09-26). -
      • -
    • Whether or not WASMFS/OPFS support is enabled on any given - page may depend on build-time options which are off by - default. + of Chrome (v110 as of 2022-12-02).
    The tests and demos...
      @@ -44,11 +48,21 @@
      • tester1: Core unit and regression tests for the various APIs and surrounding utility code.
      • tester1-worker: same thing - but running in a Worker.
        • + but running in a Worker. +
      • tester1-esm: same as + tester1 but loads sqlite3 in the main thread via + an ES6 module. +
        • +
      • tester1-worker?esm: + same as tester1-esm but loads a Worker Module which + then loads the sqlite3 API via an ES6 module. Note that + not all browsers permit loading modules in Worker + threads. +
    • High-level apps and demos...
      • fiddle is an HTML front-end @@ -70,38 +84,47 @@
    • speedtest1 ports (sqlite3's primary benchmarking tool)...
    • The obligatory "misc." category...
      • +
    + sqlite3 tester #1: Worker thread + -

    sqlite3 WASM/JS tester #1 (Worker thread)

    +

    sqlite3 tester #1: Worker thread

    See tester1.html for the UI-thread variant.
    @@ -38,26 +34,44 @@ const cbReverseIt = ()=>{ logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); }; cbReverse.addEventListener('change',cbReverseIt,true); cbReverseIt(); - const w = new Worker("tester1.js?sqlite3.dir=jswasm"); + const urlParams = new URL(self.location.href).searchParams; + const workerArgs = []; + if(urlParams.has('esm')){ + logHtml('warning',"Attempting to run an ES6 Worker Module, "+ + "which is not supported by all browsers! "+ + "e.g. Firefox (as of 2022-12) cannot do this."); + workerArgs.push("tester1.mjs",{type:"module"}); + document.querySelectorAll('title,#color-target').forEach((e)=>{ + e.innerText = "sqlite3 tester #1: ES6 Worker Module"; + }); + }else{ + workerArgs.push("tester1.js?sqlite3.dir=jswasm"); + } + const w = new Worker(...workerArgs); w.onmessage = function({data}){ switch(data.type){ case 'log': logHtml(data.payload.cssClass, ...data.payload.args); break; case 'error': logHtml('error', ...data.payload.args); break; - case 'test-result': - document.querySelector('#color-target').classList.add( - data.payload.pass ? 'tests-pass' : 'tests-fail' - ); - break; + case 'test-result':{ + document.querySelector('#color-target').classList.add( + data.payload.pass ? 'tests-pass' : 'tests-fail' + ); + const e = document.querySelector('title'); + e.innerText = ( + data.payload.pass ? 'PASS' : 'FAIL' + ) + ': ' + e.innerText; + break; + } default: logHtml('error',"Unhandled message:",data.type); }; }; })(); ADDED ext/wasm/tester1.c-pp.html Index: ext/wasm/tester1.c-pp.html ================================================================== --- /dev/null +++ ext/wasm/tester1.c-pp.html @@ -0,0 +1,37 @@ + + + + + + + + + sqlite3 tester #1: +//#if target=es6-module +ES6 Module in UI thread +//#else +UI thread +//#endif + + + +

    +
    See tester1-worker.html + for the Worker-thread variant.
    +
    + + +
    +
    + +//#if target=es6-module + +//#else + + +//#endif + + ADDED ext/wasm/tester1.c-pp.js Index: ext/wasm/tester1.c-pp.js ================================================================== --- /dev/null +++ ext/wasm/tester1.c-pp.js @@ -0,0 +1,2859 @@ +/* + 2022-10-12 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + Main functional and regression tests for the sqlite3 WASM API. + + This mini-framework works like so: + + This script adds a series of test groups, each of which contains an + arbitrary number of tests, into a queue. After loading of the + sqlite3 WASM/JS module is complete, that queue is processed. If any + given test fails, the whole thing fails. This script is built such + that it can run from the main UI thread or worker thread. Test + groups and individual tests can be assigned a predicate function + which determines whether to run them or not, and this is + specifically intended to be used to toggle certain tests on or off + for the main/worker threads or the availability (or not) of + optional features such as int64 support. + + Each test group defines a single state object which gets applied as + the test functions' `this` for all tests in that group. Test + functions can use that to, e.g., set up a db in an early test and + close it in a later test. Each test gets passed the sqlite3 + namespace object as its only argument. +*/ +/* + This file is intended to be processed by c-pp to inject (or not) + code specific to ES6 modules which is illegal in non-module code. + + Non-ES6 module build and ES6 module for the main-thread: + + ./c-pp -f tester1.c-pp.js -o tester1.js + + ES6 worker module build: + + ./c-pp -f tester1.c-pp.js -o tester1-esm.js -Dtarget=es6-module +*/ +//#if target=es6-module +import {default as sqlite3InitModule} from './jswasm/sqlite3.mjs'; +self.sqlite3InitModule = sqlite3InitModule; +//#else +'use strict'; +//#endif +(function(self){ + /** + Set up our output channel differently depending + on whether we are running in a worker thread or + the main (UI) thread. + */ + let logClass; + /* Predicate for tests/groups. */ + const isUIThread = ()=>(self.window===self && self.document); + /* Predicate for tests/groups. */ + const isWorker = ()=>!isUIThread(); + /* Predicate for tests/groups. */ + const testIsTodo = ()=>false; + const haveWasmCTests = ()=>{ + return !!wasm.exports.sqlite3_wasm_test_intptr; + }; + { + const mapToString = (v)=>{ + switch(typeof v){ + case 'number': case 'string': case 'boolean': + case 'undefined': case 'bigint': + return ''+v; + default: break; + } + if(null===v) return 'null'; + if(v instanceof Error){ + v = { + message: v.message, + stack: v.stack, + errorClass: v.name + }; + } + return JSON.stringify(v,undefined,2); + }; + const normalizeArgs = (args)=>args.map(mapToString); + if( isUIThread() ){ + console.log("Running in the UI thread."); + const logTarget = document.querySelector('#test-output'); + logClass = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass){ + for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ + ln.classList.add(c); + } + } + ln.append(document.createTextNode(normalizeArgs(args).join(' '))); + logTarget.append(ln); + }; + const cbReverse = document.querySelector('#cb-log-reverse'); + const cbReverseKey = 'tester1:cb-log-reverse'; + const cbReverseIt = ()=>{ + logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + }; + cbReverse.addEventListener('change', cbReverseIt, true); + /*if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + }*/ + cbReverseIt(); + }else{ /* Worker thread */ + console.log("Running in a Worker thread."); + logClass = function(cssClass,...args){ + postMessage({ + type:'log', + payload:{cssClass, args: normalizeArgs(args)} + }); + }; + } + } + const reportFinalTestStatus = function(pass){ + if(isUIThread()){ + let e = document.querySelector('#color-target'); + e.classList.add(pass ? 'tests-pass' : 'tests-fail'); + e = document.querySelector('title'); + e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; + }else{ + postMessage({type:'test-result', payload:{pass}}); + } + }; + const log = (...args)=>{ + //console.log(...args); + logClass('',...args); + } + const warn = (...args)=>{ + console.warn(...args); + logClass('warning',...args); + } + const error = (...args)=>{ + console.error(...args); + logClass('error',...args); + }; + + const toss = (...args)=>{ + error(...args); + throw new Error(args.join(' ')); + }; + const tossQuietly = (...args)=>{ + throw new Error(args.join(' ')); + }; + + const roundMs = (ms)=>Math.round(ms*100)/100; + + /** + Helpers for writing sqlite3-specific tests. + */ + const TestUtil = { + /** Running total of the number of tests run via + this API. */ + counter: 0, + /* Separator line for log messages. */ + separator: '------------------------------------------------------------', + /** + If expr is a function, it is called and its result + is returned, coerced to a bool, else expr, coerced to + a bool, is returned. + */ + toBool: function(expr){ + return (expr instanceof Function) ? !!expr() : !!expr; + }, + /** Throws if expr is false. If expr is a function, it is called + and its result is evaluated. If passed multiple arguments, + those after the first are a message string which get applied + as an exception message if the assertion fails. The message + arguments are concatenated together with a space between each. + */ + assert: function f(expr, ...msg){ + ++this.counter; + if(!this.toBool(expr)){ + throw new Error(msg.length ? msg.join(' ') : "Assertion failed."); + } + return this; + }, + /** Calls f() and squelches any exception it throws. If it + does not throw, this function throws. */ + mustThrow: function(f, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + return this; + }, + /** + Works like mustThrow() but expects filter to be a regex, + function, or string to match/filter the resulting exception + against. If f() does not throw, this test fails and an Error is + thrown. If filter is a regex, the test passes if + filter.test(error.message) passes. If it's a function, the test + passes if filter(error) returns truthy. If it's a string, the + test passes if the filter matches the exception message + precisely. In all other cases the test fails, throwing an + Error. + + If it throws, msg is used as the error report unless it's falsy, + in which case a default is used. + */ + mustThrowMatching: function(f, filter, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + let pass = false; + if(filter instanceof RegExp) pass = filter.test(err.message); + else if(filter instanceof Function) pass = filter(err); + else if('string' === typeof filter) pass = (err.message === filter); + if(!pass){ + throw new Error(msg || ("Filter rejected this exception: "+err.message)); + } + return this; + }, + /** Throws if expr is truthy or expr is a function and expr() + returns truthy. */ + throwIf: function(expr, msg){ + ++this.counter; + if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); + return this; + }, + /** Throws if expr is falsy or expr is a function and expr() + returns falsy. */ + throwUnless: function(expr, msg){ + ++this.counter; + if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); + return this; + }, + eqApprox: (v1,v2,factor=0.05)=>(v1>=(v2-factor) && v1<=(v2+factor)), + TestGroup: (function(){ + let groupCounter = 0; + const TestGroup = function(name, predicate){ + this.number = ++groupCounter; + this.name = name; + this.predicate = predicate; + this.tests = []; + }; + TestGroup.prototype = { + addTest: function(testObj){ + this.tests.push(testObj); + return this; + }, + run: async function(sqlite3){ + log(TestUtil.separator); + logClass('group-start',"Group #"+this.number+':',this.name); + const indent = ' '; + if(this.predicate){ + const p = this.predicate(sqlite3); + if(!p || 'string'===typeof p){ + logClass('warning',indent, + "SKIPPING group:", p ? p : "predicate says to" ); + return; + } + } + const assertCount = TestUtil.counter; + const groupState = Object.create(null); + const skipped = []; + let runtime = 0, i = 0; + for(const t of this.tests){ + ++i; + const n = this.number+"."+i; + log(indent, n+":", t.name); + if(t.predicate){ + const p = t.predicate(sqlite3); + if(!p || 'string'===typeof p){ + logClass('warning',indent, + "SKIPPING:", p ? p : "predicate says to" ); + skipped.push( n+': '+t.name ); + continue; + } + } + const tc = TestUtil.counter, now = performance.now(); + await t.test.call(groupState, sqlite3); + const then = performance.now(); + runtime += then - now; + logClass('faded',indent, indent, + TestUtil.counter - tc, 'assertion(s) in', + roundMs(then-now),'ms'); + } + logClass('green', + "Group #"+this.number+":",(TestUtil.counter - assertCount), + "assertion(s) in",roundMs(runtime),"ms"); + if(0 && skipped.length){ + logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped); + } + } + }; + return TestGroup; + })()/*TestGroup*/, + testGroups: [], + currentTestGroup: undefined, + addGroup: function(name, predicate){ + this.testGroups.push( this.currentTestGroup = + new this.TestGroup(name, predicate) ); + return this; + }, + addTest: function(name, callback){ + let predicate; + if(1===arguments.length){ + this.currentTestGroup.addTest(arguments[0]); + }else{ + this.currentTestGroup.addTest({ + name, predicate, test: callback + }); + } + return this; + }, + runTests: async function(sqlite3){ + return new Promise(async function(pok,pnok){ + try { + let runtime = 0; + for(let g of this.testGroups){ + const now = performance.now(); + await g.run(sqlite3); + runtime += performance.now() - now; + } + log(TestUtil.separator); + logClass(['strong','green'], + "Done running tests.",TestUtil.counter,"assertions in", + roundMs(runtime),'ms'); + pok(); + reportFinalTestStatus(true); + }catch(e){ + error(e); + pnok(e); + reportFinalTestStatus(false); + } + }.bind(this)); + } + }/*TestUtil*/; + const T = TestUtil; + T.g = T.addGroup; + T.t = T.addTest; + let capi, wasm/*assigned after module init*/; + //////////////////////////////////////////////////////////////////////// + // End of infrastructure setup. Now define the tests... + //////////////////////////////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////// + T.g('Basic sanity checks') + .t({ + name:'sqlite3_config()', + test:function(sqlite3){ + for(const k of [ + 'SQLITE_CONFIG_GETMALLOC', 'SQLITE_CONFIG_URI' + ]){ + T.assert(capi[k] > 0); + } + T.assert(capi.SQLITE_MISUSE===capi.sqlite3_config( + capi.SQLITE_CONFIG_URI, 1 + ), "MISUSE because the library has already been initialized."); + T.assert(capi.SQLITE_MISUSE === capi.sqlite3_config( + // not enough args + capi.SQLITE_CONFIG_GETMALLOC + )); + T.assert(capi.SQLITE_NOTFOUND === capi.sqlite3_config( + // unhandled-in-JS config option + capi.SQLITE_CONFIG_GETMALLOC, 1 + )); + if(0){ + log("We cannot _fully_ test sqlite3_config() after the library", + "has been initialized (which it necessarily has been to", + "set up various bindings) and we cannot shut it down ", + "without losing the VFS registrations."); + T.assert(0 === capi.sqlite3_config( + capi.SQLITE_CONFIG_URI, 1 + )); + } + } + })/*sqlite3_config()*/ + + //////////////////////////////////////////////////////////////////// + .t({ + name: "JS wasm-side allocator", + test: function(sqlite3){ + if(sqlite3.config.useStdAlloc){ + warn("Using system allocator. This violates the docs and", + "may cause grief with certain APIs", + "(e.g. sqlite3_deserialize())."); + T.assert(wasm.alloc.impl === wasm.exports.malloc) + .assert(wasm.dealloc === wasm.exports.free) + .assert(wasm.realloc.impl === wasm.exports.realloc); + }else{ + T.assert(wasm.alloc.impl === wasm.exports.sqlite3_malloc) + .assert(wasm.dealloc === wasm.exports.sqlite3_free) + .assert(wasm.realloc.impl === wasm.exports.sqlite3_realloc); + } + } + }) + .t('Namespace object checks', function(sqlite3){ + const wasmCtypes = wasm.ctype; + T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). + assert(wasmCtypes.structs[0].members.szOsFile.sizeof>=4). + assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ + ].members.xFileSize.offset>0); + [ /* Spot-check a handful of constants to make sure they got installed... */ + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', + 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' + ].forEach((k)=>T.assert('number' === typeof capi[k])); + [/* Spot-check a few of the WASM API methods. */ + 'alloc', 'dealloc', 'installFunction' + ].forEach((k)=>T.assert(wasm[k] instanceof Function)); + + T.assert(capi.sqlite3_errstr(capi.SQLITE_IOERR_ACCESS).indexOf("I/O")>=0). + assert(capi.sqlite3_errstr(capi.SQLITE_CORRUPT).indexOf('malformed')>0). + assert(capi.sqlite3_errstr(capi.SQLITE_OK) === 'not an error'); + + try { + throw new sqlite3.WasmAllocError; + }catch(e){ + T.assert(e instanceof Error) + .assert(e instanceof sqlite3.WasmAllocError) + .assert("Allocation failed." === e.message); + } + try { + throw new sqlite3.WasmAllocError("test",{ + cause: 3 + }); + }catch(e){ + T.assert(3 === e.cause) + .assert("test" === e.message); + } + try {throw new sqlite3.WasmAllocError("test","ing",".")} + catch(e){T.assert("test ing ." === e.message)} + + try{ throw new sqlite3.SQLite3Error(capi.SQLITE_SCHEMA) } + catch(e){ + T.assert('SQLITE_SCHEMA' === e.message) + .assert(capi.SQLITE_SCHEMA === e.resultCode); + } + try{ sqlite3.SQLite3Error.toss(capi.SQLITE_CORRUPT,{cause: true}) } + catch(e){ + T.assert('SQLITE_CORRUPT' === e.message) + .assert(capi.SQLITE_CORRUPT === e.resultCode) + .assert(true===e.cause); + } + try{ sqlite3.SQLite3Error.toss("resultCode check") } + catch(e){ + T.assert(capi.SQLITE_ERROR === e.resultCode) + .assert('resultCode check' === e.message); + } + }) + //////////////////////////////////////////////////////////////////// + .t('strglob/strlike', function(sqlite3){ + T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). + assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). + assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). + assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + }) + + //////////////////////////////////////////////////////////////////// + ;/*end of basic sanity checks*/ + + //////////////////////////////////////////////////////////////////// + T.g('C/WASM Utilities') + .t('sqlite3.wasm namespace', function(sqlite3){ + // TODO: break this into smaller individual test functions. + const w = wasm; + const chr = (x)=>x.charCodeAt(0); + //log("heap getters..."); + { + const li = [8, 16, 32]; + if(w.bigIntEnabled) li.push(64); + for(const n of li){ + const bpe = n/8; + const s = w.heapForSize(n,false); + T.assert(bpe===s.BYTES_PER_ELEMENT). + assert(w.heapForSize(s.constructor) === s); + const u = w.heapForSize(n,true); + T.assert(bpe===u.BYTES_PER_ELEMENT). + assert(s!==u). + assert(w.heapForSize(u.constructor) === u); + } + } + + // alloc(), realloc(), allocFromTypedArray() + { + let m = w.alloc(14); + let m2 = w.realloc(m, 16); + T.assert(m === m2/* because of alignment */); + T.assert(0 === w.realloc(m, 0)); + m = m2 = 0; + + // Check allocation limits and allocator's responses... + T.assert('number' === typeof sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE); + if(!sqlite3.config.useStdAlloc){ + const tooMuch = sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE + 1, + isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; + T.mustThrowMatching(()=>w.alloc(tooMuch), isAllocErr) + .assert(0 === w.alloc.impl(tooMuch)) + .mustThrowMatching(()=>w.realloc(0, tooMuch), isAllocErr) + .assert(0 === w.realloc.impl(0, tooMuch)); + } + + // Check allocFromTypedArray()... + const byteList = [11,22,33] + const u = new Uint8Array(byteList); + m = w.allocFromTypedArray(u); + for(let i = 0; i < u.length; ++i){ + T.assert(u[i] === byteList[i]) + .assert(u[i] === w.peek8(m + i)); + } + w.dealloc(m); + m = w.allocFromTypedArray(u.buffer); + for(let i = 0; i < u.length; ++i){ + T.assert(u[i] === byteList[i]) + .assert(u[i] === w.peek8(m + i)); + } + + w.dealloc(m); + T.mustThrowMatching( + ()=>w.allocFromTypedArray(1), + 'Value is not of a supported TypedArray type.' + ); + } + + { // Test peekXYZ()/pokeXYZ()... + const m = w.alloc(8); + T.assert( 17 === w.poke8(m,17).peek8(m) ) + .assert( 31987 === w.poke16(m,31987).peek16(m) ) + .assert( 345678 === w.poke32(m,345678).peek32(m) ) + .assert( + T.eqApprox( 345678.9, w.poke32f(m,345678.9).peek32f(m) ) + ).assert( + T.eqApprox( 4567890123.4, w.poke64f(m, 4567890123.4).peek64f(m) ) + ); + if(w.bigIntEnabled){ + T.assert( + BigInt(Number.MAX_SAFE_INTEGER) === + w.poke64(m, Number.MAX_SAFE_INTEGER).peek64(m) + ); + } + w.dealloc(m); + } + + // isPtr32() + { + const ip = w.isPtr32; + T.assert(ip(0)) + .assert(!ip(-1)) + .assert(!ip(1.1)) + .assert(!ip(0xffffffff)) + .assert(ip(0x7fffffff)) + .assert(!ip()) + .assert(!ip(null)/*might change: under consideration*/) + ; + } + + //log("jstrlen()..."); + { + T.assert(3 === w.jstrlen("abc")).assert(4 === w.jstrlen("äbc")); + } + + //log("jstrcpy()..."); + { + const fillChar = 10; + let ua = new Uint8Array(8), rc, + refill = ()=>ua.fill(fillChar); + refill(); + rc = w.jstrcpy("hello", ua); + T.assert(6===rc).assert(0===ua[5]).assert(chr('o')===ua[4]); + refill(); + ua[5] = chr('!'); + rc = w.jstrcpy("HELLO", ua, 0, -1, false); + T.assert(5===rc).assert(chr('!')===ua[5]).assert(chr('O')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4); + //log("rc,ua",rc,ua); + T.assert(4===rc).assert(0===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4, -1, false); + T.assert(4===rc).assert(chr(' ')===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy("x", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 1, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, false); + T.assert(2===rc).assert(fillChar!==ua[1]).assert(fillChar===ua[2]); + }/*jstrcpy()*/ + + //log("cstrncpy()..."); + { + const scope = w.scopedAllocPush(); + try { + let cStr = w.scopedAllocCString("hello"); + const n = w.cstrlen(cStr); + let cpy = w.scopedAlloc(n+10); + let rc = w.cstrncpy(cpy, cStr, n+10); + T.assert(n+1 === rc). + assert("hello" === w.cstrToJs(cpy)). + assert(chr('o') === w.peek8(cpy+n-1)). + assert(0 === w.peek8(cpy+n)); + let cStr2 = w.scopedAllocCString("HI!!!"); + rc = w.cstrncpy(cpy, cStr2, 3); + T.assert(3===rc). + assert("HI!lo" === w.cstrToJs(cpy)). + assert(chr('!') === w.peek8(cpy+2)). + assert(chr('l') === w.peek8(cpy+3)); + }finally{ + w.scopedAllocPop(scope); + } + } + + //log("jstrToUintArray()..."); + { + let a = w.jstrToUintArray("hello", false); + T.assert(5===a.byteLength).assert(chr('o')===a[4]); + a = w.jstrToUintArray("hello", true); + T.assert(6===a.byteLength).assert(chr('o')===a[4]).assert(0===a[5]); + a = w.jstrToUintArray("äbä", false); + T.assert(5===a.byteLength).assert(chr('b')===a[2]); + a = w.jstrToUintArray("äbä", true); + T.assert(6===a.byteLength).assert(chr('b')===a[2]).assert(0===a[5]); + } + + //log("allocCString()..."); + { + const jstr = "hällo, world!"; + const [cstr, n] = w.allocCString(jstr, true); + T.assert(14 === n) + .assert(0===w.peek8(cstr+n)) + .assert(chr('!')===w.peek8(cstr+n-1)); + w.dealloc(cstr); + } + + //log("scopedAlloc() and friends..."); + { + const alloc = w.alloc, dealloc = w.dealloc; + w.alloc = w.dealloc = null; + T.assert(!w.scopedAlloc.level) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.alloc = alloc; + T.mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.dealloc = dealloc; + T.mustThrowMatching(()=>w.scopedAllocPop(), /^Invalid state/) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAlloc.level=0, /read-only/); + const asc = w.scopedAllocPush(); + let asc2; + try { + const p1 = w.scopedAlloc(16), + p2 = w.scopedAlloc(16); + T.assert(1===w.scopedAlloc.level) + .assert(Number.isFinite(p1)) + .assert(Number.isFinite(p2)) + .assert(asc[0] === p1) + .assert(asc[1]===p2); + asc2 = w.scopedAllocPush(); + const p3 = w.scopedAlloc(16); + T.assert(2===w.scopedAlloc.level) + .assert(Number.isFinite(p3)) + .assert(2===asc.length) + .assert(p3===asc2[0]); + + const [z1, z2, z3] = w.scopedAllocPtr(3); + T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) + .assert(0===w.peek32(z1), 'allocPtr() must zero the targets') + .assert(0===w.peek32(z3)); + }finally{ + // Pop them in "incorrect" order to make sure they behave: + w.scopedAllocPop(asc); + T.assert(0===asc.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc), + /^Invalid state object/); + if(asc2){ + T.assert(2===asc2.length,'Should be p3 and z1'); + w.scopedAllocPop(asc2); + T.assert(0===asc2.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc2), + /^Invalid state object/); + } + } + T.assert(0===w.scopedAlloc.level); + w.scopedAllocCall(function(){ + T.assert(1===w.scopedAlloc.level); + const [cstr, n] = w.scopedAllocCString("hello, world", true); + T.assert(12 === n) + .assert(0===w.peek8(cstr+n)) + .assert(chr('d')===w.peek8(cstr+n-1)); + }); + }/*scopedAlloc()*/ + + //log("xCall()..."); + { + const pJson = w.xCall('sqlite3_wasm_enum_json'); + T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); + } + + //log("xWrap()..."); + { + T.mustThrowMatching(()=>w.xWrap('sqlite3_libversion',null,'i32'), + /requires 0 arg/). + assert(w.xWrap.resultAdapter('i32') instanceof Function). + assert(w.xWrap.argAdapter('i32') instanceof Function); + let fw = w.xWrap('sqlite3_libversion','utf8'); + T.mustThrowMatching(()=>fw(1), /requires 0 arg/); + let rc = fw(); + T.assert('string'===typeof rc).assert(rc.length>5); + rc = w.xCallWrapped('sqlite3_wasm_enum_json','*'); + T.assert(rc>0 && Number.isFinite(rc)); + rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); + T.assert('string'===typeof rc).assert(rc.length>300); + + + { // 'string:static' argAdapter() sanity checks... + let argAd = w.xWrap.argAdapter('string:static'); + let p0 = argAd('foo'), p1 = argAd('bar'); + T.assert(w.isPtr(p0) && w.isPtr(p1)) + .assert(p0 !== p1) + .assert(p0 === argAd('foo')) + .assert(p1 === argAd('bar')); + } + + // 'string:flexible' argAdapter() sanity checks... + w.scopedAllocCall(()=>{ + const argAd = w.xWrap.argAdapter('string:flexible'); + const cj = (v)=>w.cstrToJs(argAd(v)); + T.assert('Hi' === cj('Hi')) + .assert('hi' === cj(['h','i'])) + .assert('HI' === cj(new Uint8Array([72, 73]))); + }); + + // jsFuncToWasm() + { + const fsum3 = (x,y,z)=>x+y+z; + fw = w.jsFuncToWasm('i(iii)', fsum3); + T.assert(fw instanceof Function) + .assert( fsum3 !== fw ) + .assert( 3 === fw.length ) + .assert( 6 === fw(1,2,3) ); + T.mustThrowMatching( ()=>w.jsFuncToWasm('x()', function(){}), + 'Invalid signature letter: x'); + } + + // xWrap(Function,...) + { + let fp; + try { + const fmy = function fmy(i,s,d){ + if(fmy.debug) log("fmy(",...arguments,")"); + T.assert( 3 === i ) + .assert( w.isPtr(s) ) + .assert( w.cstrToJs(s) === 'a string' ) + .assert( T.eqApprox(1.2, d) ); + return w.allocCString("hi"); + }; + fmy.debug = false; + const xwArgs = ['string:dealloc', ['i32', 'string', 'f64']]; + fw = w.xWrap(fmy, ...xwArgs); + const fmyArgs = [3, 'a string', 1.2]; + let rc = fw(...fmyArgs); + T.assert( 'hi' === rc ); + if(0){ + /* Retain this as a "reminder to self"... + + This extra level of indirection does not work: the + string argument is ending up as a null in fmy() but + the numeric arguments are making their ways through + + What's happening is: installFunction() is creating a + WASM-compatible function instance. When we pass a JS string + into there it's getting coerced into `null` before being passed + on to the lower-level wrapper. + */ + fmy.debug = true; + fp = wasm.installFunction('i(isd)', fw); + fw = w.functionEntry(fp); + rc = fw(...fmyArgs); + log("rc =",rc); + T.assert( 'hi' === rc ); + // Similarly, this does not work: + //let fpw = w.xWrap(fp, null, [null,null,null]); + //rc = fpw(...fmyArgs); + //log("rc =",rc); + //T.assert( 'hi' === rc ); + } + }finally{ + wasm.uninstallFunction(fp); + } + } + + if(haveWasmCTests()){ + if(!sqlite3.config.useStdAlloc){ + fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:dealloc',['i32']); + rc = fw(0); + T.assert('hello'===rc); + rc = fw(1); + T.assert(null===rc); + } + + if(w.bigIntEnabled){ + w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); + w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); + fw = w.xWrap('sqlite3_wasm_test_int64_times2','thrice','twice'); + rc = fw(1); + T.assert(12n===rc); + + w.scopedAllocCall(function(){ + const pI1 = w.scopedAlloc(8), pI2 = pI1+4; + w.pokePtr([pI1, pI2], 0); + const f = w.xWrap('sqlite3_wasm_test_int64_minmax',undefined,['i64*','i64*']); + const [r1, r2] = w.peek64([pI1, pI2]); + T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); + }); + } + } + }/*xWrap()*/ + }/*WhWasmUtil*/) + + //////////////////////////////////////////////////////////////////// + .t('sqlite3.StructBinder (jaccwabyt🐇)', function(sqlite3){ + const S = sqlite3, W = S.wasm; + const MyStructDef = { + sizeof: 16, + members: { + p4: {offset: 0, sizeof: 4, signature: "i"}, + pP: {offset: 4, sizeof: 4, signature: "P"}, + ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, + cstr: {offset: 12, sizeof: 4, signature: "s"} + } + }; + if(W.bigIntEnabled){ + const m = MyStructDef; + m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; + m.sizeof += m.members.p8.sizeof; + } + const StructType = S.StructBinder.StructType; + const K = S.StructBinder('my_struct',MyStructDef); + T.mustThrowMatching(()=>K(), /via 'new'/). + mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); + const k1 = new K(), k2 = new K(); + try { + T.assert(k1.constructor === K). + assert(K.isA(k1)). + assert(k1 instanceof K). + assert(K.prototype.lookupMember('p4').key === '$p4'). + assert(K.prototype.lookupMember('$p4').name === 'p4'). + mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). + assert(undefined === K.prototype.lookupMember('nope',false)). + assert(k1 instanceof StructType). + assert(StructType.isA(k1)). + mustThrowMatching(()=>k1.$ro = 1, /read-only/); + Object.keys(MyStructDef.members).forEach(function(key){ + key = K.memberKey(key); + T.assert(0 == k1[key], + "Expecting allocation to zero the memory "+ + "for "+key+" but got: "+k1[key]+ + " from "+k1.memoryDump()); + }); + T.assert('number' === typeof k1.pointer). + mustThrowMatching(()=>k1.pointer = 1, /pointer/); + k1.$p4 = 1; k1.$pP = 2; + T.assert(1 === k1.$p4).assert(2 === k1.$pP); + if(MyStructDef.members.$p8){ + k1.$p8 = 1/*must not throw despite not being a BigInt*/; + k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + } + T.assert(!k1.ondispose); + k1.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(k1.ondispose)). + assert(k1.ondispose[0] === k1.$cstr). + assert('number' === typeof k1.$cstr). + assert('A C-string.' === k1.memberToJsString('cstr')); + k1.$pP = k2; + T.assert(k1.$pP === k2.pointer); + k1.$pP = null/*null is special-cased to 0.*/; + T.assert(0===k1.$pP); + let ptr = k1.pointer; + k1.dispose(); + T.assert(undefined === k1.pointer). + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + }finally{ + k1.dispose(); + k2.dispose(); + } + + if(!W.bigIntEnabled){ + log("Skipping WasmTestStruct tests: BigInt not enabled."); + return; + } + + const WTStructDesc = + W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; + const autoResolvePtr = true /* EXPERIMENTAL */; + if(autoResolvePtr){ + WTStructDesc.members.ppV.signature = 'P'; + } + const WTStruct = S.StructBinder(WTStructDesc); + //log(WTStruct.structName, WTStruct.structInfo); + const wts = new WTStruct(); + //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); + try{ + T.assert(wts.constructor === WTStruct). + assert(WTStruct.memberKeys().indexOf('$ppV')>=0). + assert(wts.memberKeys().indexOf('$v8')>=0). + assert(!K.isA(wts)). + assert(WTStruct.isA(wts)). + assert(wts instanceof WTStruct). + assert(wts instanceof StructType). + assert(StructType.isA(wts)). + assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). + assert(0===wts.$ppV).assert(0===wts.$xFunc); + const testFunc = + W.xGet('sqlite3_wasm_test_struct'/*name gets mangled in -O3 builds!*/); + let counter = 0; + //log("wts.pointer =",wts.pointer); + const wtsFunc = function(arg){ + /*log("This from a JS function called from C, "+ + "which itself was called from JS. arg =",arg);*/ + ++counter; + if(3===counter){ + tossQuietly("Testing exception propagation."); + } + } + wts.$v4 = 10; wts.$v8 = 20; + wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) + T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) + .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) + .assert(0 === wts.$cstr) + .assert(wts.memberIsString('$cstr')) + .assert(!wts.memberIsString('$v4')) + .assert(null === wts.memberToJsString('$cstr')) + .assert(W.functionEntry(wts.$xFunc) instanceof Function); + /* It might seem silly to assert that the values match + what we just set, but recall that all of those property + reads and writes are, via property interceptors, + actually marshaling their data to/from a raw memory + buffer, so merely reading them back is actually part of + testing the struct-wrapping API. */ + + testFunc(wts.pointer); + //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); + T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) + .assert(wts.$ppV === wts.pointer) + .assert('string' === typeof wts.memberToJsString('cstr')) + .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) + .mustThrowMatching(()=>wts.memberToJsString('xFunc'), + /Invalid member type signature for C-string/) + ; + testFunc(wts.pointer); + T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) + .assert(wts.$ppV === wts.pointer); + /** The 3rd call to wtsFunc throw from JS, which is called + from C, which is called from JS. Let's ensure that + that exception propagates back here... */ + T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); + W.uninstallFunction(wts.$xFunc); + wts.$xFunc = 0; + wts.$ppV = 0; + T.assert(!wts.$ppV); + //WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts.pointer === wts.$ppV) + wts.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(wts.ondispose)). + assert(wts.ondispose[0] === wts.$cstr). + assert('A C-string.' === wts.memberToJsString('cstr')); + const ptr = wts.pointer; + wts.dispose(); + T.assert(ptr).assert(undefined === wts.pointer); + }finally{ + wts.dispose(); + } + + if(1){ // ondispose of other struct instances + const s1 = new WTStruct, s2 = new WTStruct, s3 = new WTStruct; + T.assert(s1.lookupMember instanceof Function) + .assert(s1.addOnDispose instanceof Function); + s1.addOnDispose(s2,"testing variadic args"); + T.assert(2===s1.ondispose.length); + s2.addOnDispose(s3); + s1.dispose(); + T.assert(!s2.pointer,"Expecting s2 to be ondispose'd by s1."); + T.assert(!s3.pointer,"Expecting s3 to be ondispose'd by s2."); + } + }/*StructBinder*/) + + //////////////////////////////////////////////////////////////////// + .t('sqlite3.wasm.pstack', function(sqlite3){ + const P = wasm.pstack; + const isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; + const stack = P.pointer; + T.assert(0===stack % 8 /* must be 8-byte aligned */); + try{ + const remaining = P.remaining; + T.assert(P.quota >= 4096) + .assert(remaining === P.quota) + .mustThrowMatching(()=>P.alloc(0), isAllocErr) + .mustThrowMatching(()=>P.alloc(-1), isAllocErr) + .mustThrowMatching( + ()=>P.alloc('i33'), + (e)=>e instanceof sqlite3.WasmAllocError + ); + ; + let p1 = P.alloc(12); + T.assert(p1 === stack - 16/*8-byte aligned*/) + .assert(P.pointer === p1); + let p2 = P.alloc(7); + T.assert(p2 === p1-8/*8-byte aligned, stack grows downwards*/) + .mustThrowMatching(()=>P.alloc(remaining), isAllocErr) + .assert(24 === stack - p2) + .assert(P.pointer === p2); + let n = remaining - (stack - p2); + let p3 = P.alloc(n); + T.assert(p3 === stack-remaining) + .mustThrowMatching(()=>P.alloc(1), isAllocErr); + }finally{ + P.restore(stack); + } + + T.assert(P.pointer === stack); + try { + const [p1, p2, p3] = P.allocChunks(3,'i32'); + T.assert(P.pointer === stack-16/*always rounded to multiple of 8*/) + .assert(p2 === p1 + 4) + .assert(p3 === p2 + 4); + T.mustThrowMatching(()=>P.allocChunks(1024, 1024 * 16), + (e)=>e instanceof sqlite3.WasmAllocError) + }finally{ + P.restore(stack); + } + + T.assert(P.pointer === stack); + try { + let [p1, p2, p3] = P.allocPtr(3,false); + let sPos = stack-16/*always rounded to multiple of 8*/; + T.assert(P.pointer === sPos) + .assert(p2 === p1 + 4) + .assert(p3 === p2 + 4); + [p1, p2, p3] = P.allocPtr(3); + T.assert(P.pointer === sPos-24/*3 x 8 bytes*/) + .assert(p2 === p1 + 8) + .assert(p3 === p2 + 8); + p1 = P.allocPtr(); + T.assert('number'===typeof p1); + }finally{ + P.restore(stack); + } + }/*pstack tests*/) + //////////////////////////////////////////////////////////////////// + ;/*end of C/WASM utils checks*/ + + T.g('sqlite3_randomness()') + .t('To memory buffer', function(sqlite3){ + const stack = wasm.pstack.pointer; + try{ + const n = 520; + const p = wasm.pstack.alloc(n); + T.assert(0===wasm.peek8(p)) + .assert(0===wasm.peek8(p+n-1)); + T.assert(undefined === capi.sqlite3_randomness(n - 10, p)); + let j, check = 0; + const heap = wasm.heap8u(); + for(j = 0; j < 10 && 0===check; ++j){ + check += heap[p + j]; + } + T.assert(check > 0); + check = 0; + // Ensure that the trailing bytes were not modified... + for(j = n - 10; j < n && 0===check; ++j){ + check += heap[p + j]; + } + T.assert(0===check); + }finally{ + wasm.pstack.restore(stack); + } + }) + .t('To byte array', function(sqlite3){ + const ta = new Uint8Array(117); + let i, n = 0; + for(i=0; i0); + const t0 = new Uint8Array(0); + T.assert(t0 === capi.sqlite3_randomness(t0), + "0-length array is a special case"); + }) + ;/*end sqlite3_randomness() checks*/ + + //////////////////////////////////////////////////////////////////////// + T.g('sqlite3.oo1') + .t('Create db', function(sqlite3){ + const dbFile = '/tester1.db'; + wasm.sqlite3_wasm_vfs_unlink(0, dbFile); + const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); + db.onclose = { + disposeAfter: [], + disposeBefore: [ + (db)=>{ + //console.debug("db.onclose.before dropping modules"); + //sqlite3.capi.sqlite3_drop_modules(db.pointer, 0); + } + ], + before: function(db){ + while(this.disposeBefore.length){ + const v = this.disposeBefore.shift(); + console.debug("db.onclose.before cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + }else if(v instanceof Function){ + try{ v(db) } catch(e){ + console.warn("beforeDispose() callback threw:",e); + } + } + } + }, + after: function(){ + while(this.disposeAfter.length){ + const v = this.disposeAfter.shift(); + console.debug("db.onclose.after cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + }else if(v instanceof Function){ + try{v()} catch(e){/*ignored*/} + } + } + } + }; + + T.assert(wasm.isPtr(db.pointer)) + .mustThrowMatching(()=>db.pointer=1, /read-only/) + .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) + .assert('main'===db.dbName(0)) + .assert('string' === typeof db.dbVfsName()) + .assert(db.pointer === wasm.xWrap.testConvertArg('sqlite3*',db)); + // Custom db error message handling via sqlite3_prepare_v2/v3() + let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); + T.assert(capi.SQLITE_MISUSE === rc) + .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) + .assert(dbFile === db.dbFilename()) + .assert(!db.dbFilename('nope')); + //Sanity check DB.checkRc()... + let ex; + try{db.checkRc(rc)} + catch(e){ex = e} + T.assert(ex instanceof sqlite3.SQLite3Error) + .assert(0===ex.message.indexOf("sqlite3 result code")) + .assert(ex.message.indexOf("Invalid SQL")>0); + T.assert(db === db.checkRc(0)) + .assert(db === sqlite3.oo1.DB.checkRc(db,0)) + .assert(null === sqlite3.oo1.DB.checkRc(null,0)); + + this.progressHandlerCount = 0; + capi.sqlite3_progress_handler(db, 5, (p)=>{ + ++this.progressHandlerCount; + return 0; + }, 0); + }) + //////////////////////////////////////////////////////////////////// + .t('sqlite3_db_config() and sqlite3_db_status()', function(sqlite3){ + let rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, 0, 0); + T.assert(0===rc); + rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_MAX+1, 0); + T.assert(capi.SQLITE_MISUSE === rc); + rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_MAINDBNAME, "main"); + T.assert(0 === rc); + const stack = wasm.pstack.pointer; + try { + const [pCur, pHi] = wasm.pstack.allocChunks(2,'i64'); + rc = capi.sqlite3_db_status(this.db, capi.SQLITE_DBSTATUS_LOOKASIDE_USED, + pCur, pHi, 0); + T.assert(0===rc); + if(wasm.peek32(pCur)){ + warn("Cannot test db_config(SQLITE_DBCONFIG_LOOKASIDE)", + "while lookaside memory is in use."); + }else{ + rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_LOOKASIDE, + 0, 4096, 12); + T.assert(0 === rc); + } + wasm.poke32([pCur, pHi], 0); + let [vCur, vHi] = wasm.peek32(pCur, pHi); + T.assert(0===vCur).assert(0===vHi); + rc = capi.sqlite3_status(capi.SQLITE_STATUS_MEMORY_USED, + pCur, pHi, 0); + [vCur, vHi] = wasm.peek32(pCur, pHi); + //console.warn("i32 vCur,vHi",vCur,vHi); + T.assert(0 === rc).assert(vCur > 0).assert(vHi >= vCur); + if(wasm.bigIntEnabled){ + // Again in 64-bit. Recall that pCur and pHi are allocated + // large enough to account for this re-use. + wasm.poke64([pCur, pHi], 0); + rc = capi.sqlite3_status64(capi.SQLITE_STATUS_MEMORY_USED, + pCur, pHi, 0); + [vCur, vHi] = wasm.peek64([pCur, pHi]); + //console.warn("i64 vCur,vHi",vCur,vHi); + T.assert(0 === rc).assert(vCur > 0).assert(vHi >= vCur); + } + }finally{ + wasm.pstack.restore(stack); + } + }) + + //////////////////////////////////////////////////////////////////// + .t('DB.Stmt', function(sqlite3){ + let st = this.db.prepare( + new TextEncoder('utf-8').encode("select 3 as a") + ); + //debug("statement =",st); + this.progressHandlerCount = 0; + try { + T.assert(wasm.isPtr(st.pointer)) + .mustThrowMatching(()=>st.pointer=1, /read-only/) + .assert(1===this.db.openStatementCount()) + .assert( + capi.sqlite3_stmt_status( + st, capi.SQLITE_STMTSTATUS_RUN, 0 + ) === 0) + .assert(!st._mayGet) + .assert('a' === st.getColumnName(0)) + .assert(1===st.columnCount) + .assert(0===st.parameterCount) + .mustThrow(()=>st.bind(1,null)) + .assert(true===st.step()) + .assert(3 === st.get(0)) + .mustThrow(()=>st.get(1)) + .mustThrow(()=>st.get(0,~capi.SQLITE_INTEGER)) + .assert(3 === st.get(0,capi.SQLITE_INTEGER)) + .assert(3 === st.getInt(0)) + .assert('3' === st.get(0,capi.SQLITE_TEXT)) + .assert('3' === st.getString(0)) + .assert(3.0 === st.get(0,capi.SQLITE_FLOAT)) + .assert(3.0 === st.getFloat(0)) + .assert(3 === st.get({}).a) + .assert(3 === st.get([])[0]) + .assert(3 === st.getJSON(0)) + .assert(st.get(0,capi.SQLITE_BLOB) instanceof Uint8Array) + .assert(1===st.get(0,capi.SQLITE_BLOB).length) + .assert(st.getBlob(0) instanceof Uint8Array) + .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) + .assert(st._mayGet) + .assert(false===st.step()) + .assert(!st._mayGet) + .assert( + capi.sqlite3_stmt_status( + st, capi.SQLITE_STMTSTATUS_RUN, 0 + ) > 0); + + T.assert(this.progressHandlerCount > 0, + "Expecting progress callback."). + assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). + assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). + assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). + assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + }finally{ + st.finalize(); + } + T.assert(!st.pointer) + .assert(0===this.db.openStatementCount()); + + T.mustThrowMatching(()=>new sqlite3.oo1.Stmt("hi"), function(err){ + return (err instanceof sqlite3.SQLite3Error) + && capi.SQLITE_MISUSE === err.resultCode + && 0 < err.message.indexOf("Do not call the Stmt constructor directly.") + }); + }) + + //////////////////////////////////////////////////////////////////////// + .t('sqlite3_js_...()', function(){ + const db = this.db; + if(1){ + const vfsList = capi.sqlite3_js_vfs_list(); + T.assert(vfsList.length>1); + //log("vfsList =",vfsList); + wasm.scopedAllocCall(()=>{ + const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); + for(const v of vfsList){ + T.assert('string' === typeof v); + const pVfs = capi.sqlite3_vfs_find(v); + T.assert(wasm.isPtr(pVfs)) + .assert(pVfs===vfsArg(v)); + const vfs = new capi.sqlite3_vfs(pVfs); + try { T.assert(vfsArg(vfs)===pVfs) } + finally{ vfs.dispose() } + } + }); + } + /** + Trivia: the magic db name ":memory:" does not actually use the + "memdb" VFS unless "memdb" is _explicitly_ provided as the VFS + name. Instead, it uses the default VFS with an in-memory btree. + Thus this.db's VFS may not be memdb even though it's an in-memory + db. + */ + const pVfsMem = capi.sqlite3_vfs_find('memdb'), + pVfsDflt = capi.sqlite3_vfs_find(0), + pVfsDb = capi.sqlite3_js_db_vfs(db.pointer); + T.assert(pVfsMem > 0) + .assert(pVfsDflt > 0) + .assert(pVfsDb > 0) + .assert(pVfsMem !== pVfsDflt + /* memdb lives on top of the default vfs */) + .assert(pVfsDb === pVfsDflt || pVfsdb === pVfsMem) + ; + /*const vMem = new capi.sqlite3_vfs(pVfsMem), + vDflt = new capi.sqlite3_vfs(pVfsDflt), + vDb = new capi.sqlite3_vfs(pVfsDb);*/ + const duv = capi.sqlite3_js_db_uses_vfs; + T.assert(pVfsDflt === duv(db.pointer, 0) + || pVfsMem === duv(db.pointer,0)) + .assert(!duv(db.pointer, "foo")) + ; + }/*sqlite3_js_...()*/) + + //////////////////////////////////////////////////////////////////// + .t('Table t', function(sqlite3){ + const db = this.db; + let list = []; + this.progressHandlerCount = 0; + let rc = db.exec({ + sql:['CREATE TABLE t(a,b);', + // ^^^ using TEMP TABLE breaks the db export test + "INSERT INTO t(a,b) VALUES(1,2),(3,4),", + "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for + off-by-one bug in string-to-WASM conversion*/], + saveSql: list, + bind: [5,6] + }); + //debug("Exec'd SQL:", list); + T.assert(rc === db) + .assert(2 === list.length) + .assert('string'===typeof list[1]) + .assert(4===db.changes()) + .assert(this.progressHandlerCount > 0, + "Expecting progress callback.") + if(wasm.bigIntEnabled){ + T.assert(4n===db.changes(false,true)); + } + + let vals = db.selectValues('select a from t order by a limit 2'); + T.assert( 2 === vals.length ) + .assert( 1===vals[0] && 3===vals[1] ); + vals = db.selectValues('select a from t order by a limit $L', + {$L:2}, capi.SQLITE_TEXT); + T.assert( 2 === vals.length ) + .assert( '1'===vals[0] && '3'===vals[1] ); + vals = undefined; + + let blob = db.selectValue("select b from t where a='blob'"); + T.assert(blob instanceof Uint8Array). + assert(0x68===blob[0] && 0x69===blob[1]); + blob = null; + let counter = 0, colNames = []; + list.length = 0; + db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ + rowMode: 'object', + resultRows: list, + columnNames: colNames, + _myState: 3 /* Accessible from the callback */, + callback: function(row,stmt){ + ++counter; + T.assert( + 3 === this._myState + /* Recall that "this" is the options object. */ + ).assert( + this.columnNames[0]==='a' && this.columnNames[1]==='b' + /* options.columnNames is filled out before the first + Stmt.step(). */ + ).assert( + (row.a%2 && row.a<6) || 'blob'===row.a + ); + } + }); + T.assert(2 === colNames.length) + .assert('a' === colNames[0]) + .assert(4 === counter) + .assert(4 === list.length); + list.length = 0; + db.exec("SELECT a a, b b FROM t",{ + rowMode: 'array', + callback: function(row,stmt){ + ++counter; + T.assert(Array.isArray(row)) + .assert((0===row[1]%2 && row[1]<7) + || (row[1] instanceof Uint8Array)); + } + }); + T.assert(8 === counter); + T.assert(Number.MIN_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MIN_SAFE_INTEGER)). + assert(Number.MAX_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MAX_SAFE_INTEGER)); + + counter = 0; + db.exec({ + sql: "SELECT a FROM t", + callback: ()=>(1===++counter), + }); + T.assert(2===counter, + "Expecting exec step() loop to stop if callback returns false."); + if(wasm.bigIntEnabled && haveWasmCTests()){ + const mI = wasm.xCall('sqlite3_wasm_test_int64_max'); + const b = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(b === db.selectValue("SELECT "+b)). + assert(b === db.selectValue("SELECT ?", b)). + assert(mI == db.selectValue("SELECT $x", {$x:mI})); + }else{ + /* Curiously, the JS spec seems to be off by one with the definitions + of MIN/MAX_SAFE_INTEGER: + + https://github.com/emscripten-core/emscripten/issues/17391 */ + T.mustThrow(()=>db.selectValue("SELECT "+(Number.MAX_SAFE_INTEGER+1))). + mustThrow(()=>db.selectValue("SELECT "+(Number.MIN_SAFE_INTEGER-1))); + } + + let st = db.prepare("update t set b=:b where a='blob'"); + try { + const ndx = st.getParamIndex(':b'); + T.assert(1===ndx); + st.bindAsBlob(ndx, "ima blob").reset(true); + } finally { + st.finalize(); + } + + try { + db.prepare("/*empty SQL*/"); + toss("Must not be reached."); + }catch(e){ + T.assert(e instanceof sqlite3.SQLite3Error) + .assert(0==e.message.indexOf('Cannot prepare empty')); + } + })/*setup table T*/ + + //////////////////////////////////////////////////////////////////// + .t({ + name: "sqlite3_set_authorizer()", + test:function(sqlite3){ + T.assert(capi.SQLITE_IGNORE>0) + .assert(capi.SQLITE_DENY>0); + const db = this.db; + const ssa = capi.sqlite3_set_authorizer; + const n = db.selectValue('select count(*) from t'); + T.assert(n>0); + let authCount = 0; + let rc = ssa(db, function(pV, iCode, s0, s1, s2, s3){ + ++authCount; + return capi.SQLITE_IGNORE; + }, 0); + T.assert(0===rc) + .assert( + undefined === db.selectValue('select count(*) from t') + /* Note that the count() never runs, so we get undefined + instead of 0. */ + ) + .assert(authCount>0); + authCount = 0; + db.exec("update t set a=-9999"); + T.assert(authCount>0); + /* Reminder: we don't use DELETE because, from the C API docs: + + "If the action code is [SQLITE_DELETE] and the callback + returns [SQLITE_IGNORE] then the [DELETE] operation proceeds + but the [truncate optimization] is disabled and all rows are + deleted individually." + */ + rc = ssa(db, null, 0); + authCount = 0; + T.assert(-9999 != db.selectValue('select a from t')) + .assert(0===authCount); + rc = ssa(db, function(pV, iCode, s0, s1, s2, s3){ + ++authCount; + return capi.SQLITE_DENY; + }, 0); + T.assert(0===rc); + let err; + try{ db.exec("select 1 from t") } + catch(e){ err = e } + T.assert(err instanceof sqlite3.SQLite3Error) + .assert(err.message.indexOf('not authorized'>0)) + .assert(1===authCount); + authCount = 0; + rc = ssa(db, function(...args){ + ++authCount; + return capi.SQLITE_OK; + }, 0); + T.assert(0===rc); + T.assert(n === db.selectValue('select count(*) from t')) + .assert(authCount>0); + authCount = 0; + rc = ssa(db, function(pV, iCode, s0, s1, s2, s3){ + ++authCount; + throw new Error("Testing catching of authorizer."); + }, 0); + T.assert(0===rc); + authCount = 0; + err = undefined; + try{ db.exec("select 1 from t") } + catch(e){err = e} + T.assert(err instanceof Error) + .assert(err.message.indexOf('not authorized')>0) + /* Note that the thrown message is trumped/overwritten + by the authorizer process. */ + .assert(1===authCount); + rc = ssa(db, 0, 0); + authCount = 0; + T.assert(0===rc); + T.assert(n === db.selectValue('select count(*) from t')) + .assert(0===authCount); + } + })/*sqlite3_set_authorizer()*/ + + //////////////////////////////////////////////////////////////////////// + .t("sqlite3_table_column_metadata()", function(sqlite3){ + const stack = wasm.pstack.pointer; + try{ + const [pzDT, pzColl, pNotNull, pPK, pAuto] = + wasm.pstack.allocPtr(5); + const rc = capi.sqlite3_table_column_metadata( + this.db, "main", "t", "rowid", + pzDT, pzColl, pNotNull, pPK, pAuto + ); + T.assert(0===rc) + .assert("INTEGER"===wasm.cstrToJs(wasm.peekPtr(pzDT))) + .assert("BINARY"===wasm.cstrToJs(wasm.peekPtr(pzColl))) + .assert(0===wasm.peek32(pNotNull)) + .assert(1===wasm.peek32(pPK)) + .assert(0===wasm.peek32(pAuto)) + }finally{ + wasm.pstack.restore(stack); + } + }) + + //////////////////////////////////////////////////////////////////////// + .t('selectArray/Object()', function(sqlite3){ + const db = this.db; + let rc = db.selectArray('select a, b from t where a=?', 5); + T.assert(Array.isArray(rc)) + .assert(2===rc.length) + .assert(5===rc[0] && 6===rc[1]); + rc = db.selectArray('select a, b from t where b=-1'); + T.assert(undefined === rc); + rc = db.selectObject('select a A, b b from t where b=?', 6); + T.assert(rc && 'object'===typeof rc) + .assert(5===rc.A) + .assert(6===rc.b); + rc = db.selectArray('select a, b from t where b=-1'); + T.assert(undefined === rc); + }) + //////////////////////////////////////////////////////////////////////// + .t('selectArrays/Objects()', function(sqlite3){ + const db = this.db; + const sql = 'select a, b from t where a=? or b=? order by a'; + let rc = db.selectArrays(sql, [1, 4]); + T.assert(Array.isArray(rc)) + .assert(2===rc.length) + .assert(2===rc[0].length) + .assert(1===rc[0][0]) + .assert(2===rc[0][1]) + .assert(3===rc[1][0]) + .assert(4===rc[1][1]) + rc = db.selectArrays(sql, [99,99]); + T.assert(Array.isArray(rc)).assert(0===rc.length); + rc = db.selectObjects(sql, [1,4]); + T.assert(Array.isArray(rc)) + .assert(2===rc.length) + .assert('object' === typeof rc[1]) + .assert(1===rc[0].a) + .assert(2===rc[0].b) + .assert(3===rc[1].a) + .assert(4===rc[1].b); + }) + + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'sqlite3_js_db_export()', + predicate: ()=>true, + test: function(sqlite3){ + const db = this.db; + const xp = capi.sqlite3_js_db_export(db.pointer); + T.assert(xp instanceof Uint8Array) + .assert(xp.byteLength>0) + .assert(0 === xp.byteLength % 512); + this.dbExport = xp; + } + }/*sqlite3_js_db_export()*/) + .t({ + name: 'sqlite3_js_vfs_create_file() with db in default VFS', + predicate: ()=>true, + test: function(sqlite3){ + const db = this.db; + const pVfs = capi.sqlite3_js_db_vfs(db); + const filename = "sqlite3_js_vfs_create_file().db"; + capi.sqlite3_js_vfs_create_file(pVfs, filename, this.dbExport); + delete this.dbExport; + const db2 = new sqlite3.oo1.DB(filename,'r'); + try { + const sql = "select count(*) from t"; + const n = db.selectValue(sql); + T.assert(n>0 && db2.selectValue(sql) === n); + }finally{ + db2.close(); + wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); + } + } + }/*sqlite3_js_vfs_create_file()*/) + + //////////////////////////////////////////////////////////////////// + .t({ + name:'Scalar UDFs', + test: function(sqlite3){ + const db = this.db; + db.createFunction("foo",(pCx,a,b)=>a+b); + T.assert(7===db.selectValue("select foo(3,4)")). + assert(5===db.selectValue("select foo(3,?)",2)). + assert(5===db.selectValue("select foo(?,?2)",[1,4])). + assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); + db.createFunction("bar", { + arity: -1, + xFunc: (pCx,...args)=>{ + T.assert(db.pointer === capi.sqlite3_context_db_handle(pCx)); + let rc = 0; + for(const v of args) rc += v; + return rc; + } + }).createFunction({ + name: "asis", + xFunc: (pCx,arg)=>arg + }); + T.assert(0===db.selectValue("select bar()")). + assert(1===db.selectValue("select bar(1)")). + assert(3===db.selectValue("select bar(1,2)")). + assert(-1===db.selectValue("select bar(1,2,-4)")). + assert('hi' === db.selectValue("select asis('hi')")). + assert('hi' === db.selectValue("select ?",'hi')). + assert(null === db.selectValue("select null")). + assert(null === db.selectValue("select asis(null)")). + assert(1 === db.selectValue("select ?",1)). + assert(2 === db.selectValue("select ?",[2])). + assert(3 === db.selectValue("select $a",{$a:3})). + assert(T.eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). + assert(T.eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))); + + let blobArg = new Uint8Array([0x68, 0x69]); + let blobRc = db.selectValue( + "select asis(?1)", + blobArg.buffer/*confirm that ArrayBuffer is handled as a Uint8Array*/ + ); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + blobRc = db.selectValue("select asis(X'6869')"); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + + blobArg = new Int8Array([0x68, 0x69]); + //debug("blobArg=",blobArg); + blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length); + //debug("blobRc=",blobRc); + T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); + + let rc = sqlite3.capi.sqlite3_create_function_v2( + this.db, "foo", 0, -1, 0, 0, 0, 0, 0 + ); + T.assert( + sqlite3.capi.SQLITE_FORMAT === rc, + "For invalid eTextRep argument." + ); + rc = sqlite3.capi.sqlite3_create_function_v2(this.db, "foo", 0); + T.assert( + sqlite3.capi.SQLITE_MISUSE === rc, + "For invalid arg count." + ); + + /* Confirm that we can map and unmap the same function with + multiple arities... */ + const fCounts = [0,0]; + const fArityCheck = function(pCx){ + return ++fCounts[arguments.length-1]; + }; + //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; + rc = capi.sqlite3_create_function_v2( + db, "nary", 0, capi.SQLITE_UTF8, 0, fArityCheck, 0, 0, 0 + ); + T.assert( 0===rc ); + rc = capi.sqlite3_create_function_v2( + db, "nary", 1, capi.SQLITE_UTF8, 0, fArityCheck, 0, 0, 0 + ); + T.assert( 0===rc ); + const sqlFArity0 = "select nary()"; + const sqlFArity1 = "select nary(1)"; + T.assert( 1 === db.selectValue(sqlFArity0) ) + .assert( 1 === fCounts[0] ).assert( 0 === fCounts[1] ); + T.assert( 1 === db.selectValue(sqlFArity1) ) + .assert( 1 === fCounts[0] ).assert( 1 === fCounts[1] ); + capi.sqlite3_create_function_v2( + db, "nary", 0, capi.SQLITE_UTF8, 0, 0, 0, 0, 0 + ); + T.mustThrowMatching((()=>db.selectValue(sqlFArity0)), + (e)=>((e instanceof sqlite3.SQLite3Error) + && e.message.indexOf("wrong number of arguments")>0), + "0-arity variant was uninstalled."); + T.assert( 2 === db.selectValue(sqlFArity1) ) + .assert( 1 === fCounts[0] ).assert( 2 === fCounts[1] ); + capi.sqlite3_create_function_v2( + db, "nary", 1, capi.SQLITE_UTF8, 0, 0, 0, 0, 0 + ); + T.mustThrowMatching((()=>db.selectValue(sqlFArity1)), + (e)=>((e instanceof sqlite3.SQLite3Error) + && e.message.indexOf("no such function")>0), + "1-arity variant was uninstalled."); + //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = false; + } + }) + + //////////////////////////////////////////////////////////////////// + .t({ + name: 'Aggregate UDFs', + //predicate: ()=>false, + test: function(sqlite3){ + const db = this.db; + const sjac = capi.sqlite3_js_aggregate_context; + db.createFunction({ + name: 'summer', + xStep: (pCtx, n)=>{ + const ac = sjac(pCtx, 4); + wasm.poke32(ac, wasm.peek32(ac) + Number(n)); + }, + xFinal: (pCtx)=>{ + const ac = sjac(pCtx, 0); + return ac ? wasm.peek32(ac) : 0; + } + }); + let v = db.selectValue([ + "with cte(v) as (", + "select 3 union all select 5 union all select 7", + ") select summer(v), summer(v+1) from cte" + /* ------------------^^^^^^^^^^^ ensures that we're handling + sqlite3_aggregate_context() properly. */ + ]); + T.assert(15===v); + T.mustThrowMatching(()=>db.selectValue("select summer(1,2)"), + /wrong number of arguments/); + + db.createFunction({ + name: 'summerN', + arity: -1, + xStep: (pCtx, ...args)=>{ + const ac = sjac(pCtx, 4); + let sum = wasm.peek32(ac); + for(const v of args) sum += Number(v); + wasm.poke32(ac, sum); + }, + xFinal: (pCtx)=>{ + const ac = sjac(pCtx, 0); + capi.sqlite3_result_int( pCtx, ac ? wasm.peek32(ac) : 0 ); + // xFinal() may either return its value directly or call + // sqlite3_result_xyz() and return undefined. Both are + // functionally equivalent. + } + }); + T.assert(18===db.selectValue('select summerN(1,8,9), summerN(2,3,4)')); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{ + xFunc: ()=>{}, xStep: ()=>{} + }); + }, /scalar or aggregate\?/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xStep: ()=>{}}); + }, /Missing xFinal/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xFinal: ()=>{}}); + }, /Missing xStep/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{}); + }, /Missing function-type properties/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xFunc:()=>{}, xDestroy:'nope'}); + }, /xDestroy property must be a function/); + T.mustThrowMatching(()=>{ + db.createFunction('nope',{xFunc:()=>{}, pApp:'nope'}); + }, /Invalid value for pApp/); + } + }/*aggregate UDFs*/) + + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'Aggregate UDFs (64-bit)', + predicate: ()=>wasm.bigIntEnabled, + //predicate: ()=>false, + test: function(sqlite3){ + const db = this.db; + const sjac = capi.sqlite3_js_aggregate_context; + db.createFunction({ + name: 'summer64', + xStep: (pCtx, n)=>{ + const ac = sjac(pCtx, 8); + wasm.poke64(ac, wasm.peek64(ac) + BigInt(n)); + }, + xFinal: (pCtx)=>{ + const ac = sjac(pCtx, 0); + return ac ? wasm.peek64(ac) : 0n; + } + }); + let v = db.selectValue([ + "with cte(v) as (", + "select 9007199254740991 union all select 1 union all select 2", + ") select summer64(v), summer64(v+1) from cte" + ]); + T.assert(9007199254740994n===v); + } + }/*aggregate UDFs*/) + + //////////////////////////////////////////////////////////////////// + .t({ + name: 'Window UDFs', + //predicate: ()=>false, + test: function(){ + /* Example window function, table, and results taken from: + https://sqlite.org/windowfunctions.html#udfwinfunc */ + const db = this.db; + const sjac = (cx,n=4)=>capi.sqlite3_js_aggregate_context(cx,n); + const xValueFinal = (pCtx)=>{ + const ac = sjac(pCtx, 0); + return ac ? wasm.peek32(ac) : 0; + }; + const xStepInverse = (pCtx, n)=>{ + const ac = sjac(pCtx); + wasm.poke32(ac, wasm.peek32(ac) + Number(n)); + }; + db.createFunction({ + name: 'winsumint', + xStep: (pCtx, n)=>xStepInverse(pCtx, n), + xInverse: (pCtx, n)=>xStepInverse(pCtx, -n), + xFinal: xValueFinal, + xValue: xValueFinal + }); + db.exec([ + "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", + "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" + ]); + let rc = db.exec({ + returnValue: 'resultRows', + sql:[ + "SELECT x, winsumint(y) OVER (", + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", + ") AS sum_y ", + "FROM twin ORDER BY x;" + ] + }); + T.assert(Array.isArray(rc)) + .assert(5 === rc.length); + let count = 0; + for(const row of rc){ + switch(++count){ + case 1: T.assert('a'===row[0] && 9===row[1]); break; + case 2: T.assert('b'===row[0] && 12===row[1]); break; + case 3: T.assert('c'===row[0] && 16===row[1]); break; + case 4: T.assert('d'===row[0] && 12===row[1]); break; + case 5: T.assert('e'===row[0] && 9===row[1]); break; + default: toss("Too many rows to window function."); + } + } + const resultRows = []; + rc = db.exec({ + resultRows, + returnValue: 'resultRows', + sql:[ + "SELECT x, winsumint(y) OVER (", + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", + ") AS sum_y ", + "FROM twin ORDER BY x;" + ] + }); + T.assert(rc === resultRows) + .assert(5 === rc.length); + + rc = db.exec({ + returnValue: 'saveSql', + sql: "select 1; select 2; -- empty\n; select 3" + }); + T.assert(Array.isArray(rc)) + .assert(3===rc.length) + .assert('select 1;' === rc[0]) + .assert('select 2;' === rc[1]) + .assert('-- empty\n; select 3' === rc[2] + /* Strange but true. */); + T.mustThrowMatching(()=>{ + db.exec({sql:'', returnValue: 'nope'}); + }, /^Invalid returnValue/); + + db.exec("DROP TABLE twin"); + } + }/*window UDFs*/) + + //////////////////////////////////////////////////////////////////// + .t("ATTACH", function(){ + const db = this.db; + const resultRows = []; + db.exec({ + sql:new TextEncoder('utf-8').encode([ + // ^^^ testing string-vs-typedarray handling in exec() + "attach 'session' as foo;", + "create table foo.bar(a);", + "insert into foo.bar(a) values(1),(2),(3);", + "select a from foo.bar order by a;" + ].join('')), + rowMode: 0, + resultRows + }); + T.assert(3===resultRows.length) + .assert(2===resultRows[1]); + T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); + + /** Demonstrate the JS-simplified form of the sqlite3_exec() callback... */ + let colCount = 0, rowCount = 0; + let rc = capi.sqlite3_exec( + db, "select a, a*2 from foo.bar", function(aVals, aNames){ + //console.warn("execCallback(",arguments,")"); + colCount = aVals.length; + ++rowCount; + T.assert(2===aVals.length) + .assert(2===aNames.length) + .assert(+(aVals[1]) === 2 * +(aVals[0])); + }, 0, 0 + ); + T.assert(0===rc).assert(3===rowCount).assert(2===colCount); + rc = capi.sqlite3_exec( + db.pointer, "select a from foo.bar", ()=>{ + tossQuietly("Testing throwing from exec() callback."); + }, 0, 0 + ); + T.assert(capi.SQLITE_ABORT === rc); + + /* Demonstrate how to get access to the "full" callback + signature, as opposed to the simplified JS-specific one... */ + rowCount = colCount = 0; + const pCb = wasm.installFunction('i(pipp)', function(pVoid,nCols,aVals,aCols){ + /* Tip: wasm.cArgvToJs() can be used to convert aVals and + aCols to arrays: const vals = wasm.cArgvToJs(nCols, + aVals); */ + ++rowCount; + colCount = nCols; + T.assert(2 === nCols) + .assert(wasm.isPtr(pVoid)) + .assert(wasm.isPtr(aVals)) + .assert(wasm.isPtr(aCols)) + .assert(+wasm.cstrToJs(wasm.peekPtr(aVals + wasm.ptrSizeof)) + === 2 * +wasm.cstrToJs(wasm.peekPtr(aVals))); + return 0; + }); + try { + T.assert(wasm.isPtr(pCb)); + rc = capi.sqlite3_exec( + db, new TextEncoder('utf-8').encode("select a, a*2 from foo.bar"), + pCb, 0, 0 + ); + T.assert(0===rc) + .assert(3===rowCount) + .assert(2===colCount); + }finally{ + wasm.uninstallFunction(pCb); + } + + // Demonstrate that an OOM result does not propagate through sqlite3_exec()... + rc = capi.sqlite3_exec( + db, ["select a,"," a*2 from foo.bar"], (aVals, aNames)=>{ + sqlite3.WasmAllocError.toss("just testing"); + }, 0, 0 + ); + T.assert(capi.SQLITE_ABORT === rc); + + db.exec("detach foo"); + T.mustThrow(()=>db.exec("select * from foo.bar"), + "Because foo is no longer attached."); + }) + + //////////////////////////////////////////////////////////////////// + .t({ + name: 'C-side WASM tests', + predicate: ()=>(haveWasmCTests() || "Not compiled in."), + test: function(){ + const w = wasm, db = this.db; + const stack = w.scopedAllocPush(); + let ptrInt; + const origValue = 512; + try{ + ptrInt = w.scopedAlloc(4); + w.poke32(ptrInt,origValue); + const cf = w.xGet('sqlite3_wasm_test_intptr'); + const oldPtrInt = ptrInt; + T.assert(origValue === w.peek32(ptrInt)); + const rc = cf(ptrInt); + T.assert(2*origValue === rc). + assert(rc === w.peek32(ptrInt)). + assert(oldPtrInt === ptrInt); + const pi64 = w.scopedAlloc(8)/*ptr to 64-bit integer*/; + const o64 = 0x010203040506/*>32-bit integer*/; + if(w.bigIntEnabled){ + w.poke64(pi64, o64); + //log("pi64 =",pi64, "o64 = 0x",o64.toString(16), o64); + const v64 = ()=>w.peek64(pi64) + T.assert(v64() == o64); + //T.assert(o64 === w.peek64(pi64)); + const cf64w = w.xGet('sqlite3_wasm_test_int64ptr'); + cf64w(pi64); + T.assert(v64() == BigInt(2 * o64)); + cf64w(pi64); + T.assert(v64() == BigInt(4 * o64)); + + const biTimes2 = w.xGet('sqlite3_wasm_test_int64_times2'); + T.assert(BigInt(2 * o64) === + biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError + in the call :/ */)); + + const pMin = w.scopedAlloc(16); + const pMax = pMin + 8; + const g64 = (p)=>w.peek64(p); + w.poke64([pMin, pMax], 0); + const minMaxI64 = [ + w.xCall('sqlite3_wasm_test_int64_min'), + w.xCall('sqlite3_wasm_test_int64_max') + ]; + T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). + assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); + //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); + w.xCall('sqlite3_wasm_test_int64_minmax', pMin, pMax); + T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). + assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); + //log("pMin",g64(pMin), "pMax",g64(pMax)); + w.poke64(pMin, minMaxI64[0]); + T.assert(g64(pMin) === minMaxI64[0]). + assert(minMaxI64[0] === db.selectValue("select ?",g64(pMin))). + assert(minMaxI64[1] === db.selectValue("select ?",g64(pMax))); + const rxRange = /too big/; + T.mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[0] - BigInt(1))}, + rxRange). + mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[1] + BigInt(1))}, + (e)=>rxRange.test(e.message)); + }else{ + log("No BigInt support. Skipping related tests."); + log("\"The problem\" here is that we can manipulate, at the byte level,", + "heap memory to set 64-bit values, but we can't get those values", + "back into JS because of the lack of 64-bit integer support."); + } + }finally{ + const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); + //log("x=",x,"y=",y,"z=",z); // just looking at the alignment + w.scopedAllocPop(stack); + } + } + }/* jaccwabyt-specific tests */) + + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'virtual table #1: eponymous w/ manual exception handling', + predicate: ()=>!!capi.sqlite3_index_info, + test: function(sqlite3){ + const VT = sqlite3.vtab; + const tmplCols = Object.assign(Object.create(null),{ + A: 0, B: 1 + }); + /** + The vtab demonstrated here is a JS-ification of + ext/misc/templatevtab.c. + */ + const tmplMod = new sqlite3.capi.sqlite3_module(); + T.assert(0===tmplMod.$xUpdate); + tmplMod.setupModule({ + catchExceptions: false, + methods: { + xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + try{ + const args = wasm.cArgvToJs(argc, argv); + T.assert(args.length>=3) + .assert(args[0] === 'testvtab') + .assert(args[1] === 'main') + .assert(args[2] === 'testvtab'); + //console.debug("xConnect() args =",args); + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored(a,b)" + ); + if(0===rc){ + const t = VT.xVtab.create(ppVtab); + T.assert(t === VT.xVtab.get(wasm.peekPtr(ppVtab))); + } + return rc; + }catch(e){ + if(!(e instanceof sqlite3.WasmAllocError)){ + wasm.dealloc(wasm.peekPtr, pzErr); + wasm.pokePtr(pzErr, wasm.allocCString(e.message)); + } + return VT.xError('xConnect',e); + } + }, + xCreate: true /* just for testing. Will be removed afterwards. */, + xDisconnect: function(pVtab){ + try { + VT.xVtab.unget(pVtab).dispose(); + return 0; + }catch(e){ + return VT.xError('xDisconnect',e); + } + }, + xOpen: function(pVtab, ppCursor){ + try{ + const t = VT.xVtab.get(pVtab), + c = VT.xCursor.create(ppCursor); + T.assert(t instanceof capi.sqlite3_vtab) + .assert(c instanceof capi.sqlite3_vtab_cursor); + c._rowId = 0; + return 0; + }catch(e){ + return VT.xError('xOpen',e); + } + }, + xClose: function(pCursor){ + try{ + const c = VT.xCursor.unget(pCursor); + T.assert(c instanceof capi.sqlite3_vtab_cursor) + .assert(!VT.xCursor.get(pCursor)); + c.dispose(); + return 0; + }catch(e){ + return VT.xError('xClose',e); + } + }, + xNext: function(pCursor){ + try{ + const c = VT.xCursor.get(pCursor); + ++c._rowId; + return 0; + }catch(e){ + return VT.xError('xNext',e); + } + }, + xColumn: function(pCursor, pCtx, iCol){ + try{ + const c = VT.xCursor.get(pCursor); + switch(iCol){ + case tmplCols.A: + capi.sqlite3_result_int(pCtx, 1000 + c._rowId); + break; + case tmplCols.B: + capi.sqlite3_result_int(pCtx, 2000 + c._rowId); + break; + default: sqlite3.SQLite3Error.toss("Invalid column id",iCol); + } + return 0; + }catch(e){ + return VT.xError('xColumn',e); + } + }, + xRowid: function(pCursor, ppRowid64){ + try{ + const c = VT.xCursor.get(pCursor); + VT.xRowid(ppRowid64, c._rowId); + return 0; + }catch(e){ + return VT.xError('xRowid',e); + } + }, + xEof: function(pCursor){ + const c = VT.xCursor.get(pCursor), + rc = c._rowId>=10; + return rc; + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + try{ + const c = VT.xCursor.get(pCursor); + c._rowId = 0; + const list = capi.sqlite3_values_to_js(argc, argv); + T.assert(argc === list.length); + //log(argc,"xFilter value(s):",list); + return 0; + }catch(e){ + return VT.xError('xFilter',e); + } + }, + xBestIndex: function(pVtab, pIdxInfo){ + try{ + //const t = VT.xVtab.get(pVtab); + const sii = capi.sqlite3_index_info; + const pii = new sii(pIdxInfo); + pii.$estimatedRows = 10; + pii.$estimatedCost = 10.0; + //log("xBestIndex $nConstraint =",pii.$nConstraint); + if(pii.$nConstraint>0){ + // Validate nthConstraint() and nthConstraintUsage() + const max = pii.$nConstraint; + for(let i=0; i < max; ++i ){ + let v = pii.nthConstraint(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthConstraint(i); + T.assert(v instanceof sii.sqlite3_index_constraint) + .assert(v.pointer >= pii.$aConstraint); + v.dispose(); + v = pii.nthConstraintUsage(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthConstraintUsage(i); + T.assert(v instanceof sii.sqlite3_index_constraint_usage) + .assert(v.pointer >= pii.$aConstraintUsage); + v.$argvIndex = i;//just to get some values into xFilter + v.dispose(); + } + } + //log("xBestIndex $nOrderBy =",pii.$nOrderBy); + if(pii.$nOrderBy>0){ + // Validate nthOrderBy() + const max = pii.$nOrderBy; + for(let i=0; i < max; ++i ){ + let v = pii.nthOrderBy(i,true); + T.assert(wasm.isPtr(v)); + v = pii.nthOrderBy(i); + T.assert(v instanceof sii.sqlite3_index_orderby) + .assert(v.pointer >= pii.$aOrderBy); + v.dispose(); + } + } + pii.dispose(); + return 0; + }catch(e){ + return VT.xError('xBestIndex',e); + } + } + } + }); + this.db.onclose.disposeAfter.push(tmplMod); + T.assert(0===tmplMod.$xUpdate) + .assert(tmplMod.$xCreate) + .assert(tmplMod.$xCreate === tmplMod.$xConnect, + "setup() must make these equivalent and "+ + "installMethods() must avoid re-compiling identical functions"); + tmplMod.$xCreate = 0 /* make tmplMod eponymous-only */; + let rc = capi.sqlite3_create_module( + this.db, "testvtab", tmplMod, 0 + ); + this.db.checkRc(rc); + const list = this.db.selectArrays( + "SELECT a,b FROM testvtab where a<9999 and b>1 order by a, b" + /* Query is shaped so that it will ensure that some constraints + end up in xBestIndex(). */ + ); + T.assert(10===list.length) + .assert(1000===list[0][0]) + .assert(2009===list[list.length-1][1]); + } + })/*custom vtab #1*/ + + //////////////////////////////////////////////////////////////////////// + .t({ + name: 'virtual table #2: non-eponymous w/ automated exception wrapping', + predicate: ()=>!!capi.sqlite3_index_info, + test: function(sqlite3){ + const VT = sqlite3.vtab; + const tmplCols = Object.assign(Object.create(null),{ + A: 0, B: 1 + }); + /** + The vtab demonstrated here is a JS-ification of + ext/misc/templatevtab.c. + */ + let throwOnCreate = 1 ? 0 : capi.SQLITE_CANTOPEN + /* ^^^ just for testing exception wrapping. Note that sqlite + always translates errors from a vtable to a generic + SQLITE_ERROR unless it's from xConnect()/xCreate() and that + callback sets an error string. */; + const vtabTrace = 1 + ? ()=>{} + : (methodName,...args)=>console.debug('sqlite3_module::'+methodName+'():',...args); + const modConfig = { + /* catchExceptions changes how the methods are wrapped */ + catchExceptions: true, + name: "vtab2test", + methods:{ + xCreate: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + vtabTrace("xCreate",...arguments); + if(throwOnCreate){ + sqlite3.SQLite3Error.toss( + throwOnCreate, + "Throwing a test exception." + ); + } + const args = wasm.cArgvToJs(argc, argv); + vtabTrace("xCreate","argv:",args); + T.assert(args.length>=3); + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored(a,b)" + ); + if(0===rc){ + const t = VT.xVtab.create(ppVtab); + T.assert(t === VT.xVtab.get(wasm.peekPtr(ppVtab))); + vtabTrace("xCreate",...arguments," ppVtab =",t.pointer); + } + return rc; + }, + xConnect: true, + xDestroy: function(pVtab){ + vtabTrace("xDestroy/xDisconnect",pVtab); + VT.xVtab.dispose(pVtab); + }, + xDisconnect: true, + xOpen: function(pVtab, ppCursor){ + const t = VT.xVtab.get(pVtab), + c = VT.xCursor.create(ppCursor); + T.assert(t instanceof capi.sqlite3_vtab) + .assert(c instanceof capi.sqlite3_vtab_cursor); + vtabTrace("xOpen",...arguments," cursor =",c.pointer); + c._rowId = 0; + }, + xClose: function(pCursor){ + vtabTrace("xClose",...arguments); + const c = VT.xCursor.unget(pCursor); + T.assert(c instanceof capi.sqlite3_vtab_cursor) + .assert(!VT.xCursor.get(pCursor)); + c.dispose(); + }, + xNext: function(pCursor){ + vtabTrace("xNext",...arguments); + const c = VT.xCursor.get(pCursor); + ++c._rowId; + }, + xColumn: function(pCursor, pCtx, iCol){ + vtabTrace("xColumn",...arguments); + const c = VT.xCursor.get(pCursor); + switch(iCol){ + case tmplCols.A: + capi.sqlite3_result_int(pCtx, 1000 + c._rowId); + break; + case tmplCols.B: + capi.sqlite3_result_int(pCtx, 2000 + c._rowId); + break; + default: sqlite3.SQLite3Error.toss("Invalid column id",iCol); + } + }, + xRowid: function(pCursor, ppRowid64){ + vtabTrace("xRowid",...arguments); + const c = VT.xCursor.get(pCursor); + VT.xRowid(ppRowid64, c._rowId); + }, + xEof: function(pCursor){ + vtabTrace("xEof",...arguments); + return VT.xCursor.get(pCursor)._rowId>=10; + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + vtabTrace("xFilter",...arguments); + const c = VT.xCursor.get(pCursor); + c._rowId = 0; + const list = capi.sqlite3_values_to_js(argc, argv); + T.assert(argc === list.length); + }, + xBestIndex: function(pVtab, pIdxInfo){ + vtabTrace("xBestIndex",...arguments); + //const t = VT.xVtab.get(pVtab); + const pii = VT.xIndexInfo(pIdxInfo); + pii.$estimatedRows = 10; + pii.$estimatedCost = 10.0; + pii.dispose(); + } + }/*methods*/ + }; + const tmplMod = VT.setupModule(modConfig); + T.assert(1===tmplMod.$iVersion); + this.db.onclose.disposeAfter.push(tmplMod); + this.db.checkRc(capi.sqlite3_create_module( + this.db.pointer, modConfig.name, tmplMod.pointer, 0 + )); + this.db.exec([ + "create virtual table testvtab2 using ", + modConfig.name, + "(arg1 blah, arg2 bloop)" + ]); + if(0){ + /* If we DROP TABLE then xDestroy() is called. If the + vtab is instead destroyed when the db is closed, + xDisconnect() is called. */ + this.db.onclose.disposeBefore.push(function(db){ + console.debug("Explicitly dropping testvtab2 via disposeBefore handler..."); + db.exec( + /** DROP TABLE is the only way to get xDestroy() to be called. */ + "DROP TABLE testvtab2" + ); + }); + } + let list = this.db.selectArrays( + "SELECT a,b FROM testvtab2 where a<9999 and b>1 order by a, b" + /* Query is shaped so that it will ensure that some + constraints end up in xBestIndex(). */ + ); + T.assert(10===list.length) + .assert(1000===list[0][0]) + .assert(2009===list[list.length-1][1]); + + list = this.db.selectArrays( + "SELECT a,b FROM testvtab2 where a<9999 and b>1 order by b, a limit 5" + ); + T.assert(5===list.length) + .assert(1000===list[0][0]) + .assert(2004===list[list.length-1][1]); + + // Call it as a table-valued function... + list = this.db.selectArrays([ + "SELECT a,b FROM ", modConfig.name, + " where a<9999 and b>1 order by b, a limit 1" + ]); + T.assert(1===list.length) + .assert(1000===list[0][0]) + .assert(2000===list[0][1]); + } + })/*custom vtab #2*/ + //////////////////////////////////////////////////////////////////////// + .t('Custom collation', function(sqlite3){ + let collationCounter = 0; + let myCmp = function(pArg,n1,p1,n2,p2){ + //int (*)(void*,int,const void*,int,const void*) + ++collationCounter; + const rc = wasm.exports.sqlite3_strnicmp(p1,p2,(n1this.db.checkRc(rc), + /SQLITE_UTF8 is the only supported encoding./); + /* + We need to ensure that replacing that collation function does + the right thing. We don't have a handle to the underlying WASM + pointer from here, so cannot verify (without digging through + internal state) that the old one gets uninstalled, but we can + verify that a new one properly replaces it. (That said, + console.warn() output has shown that the uninstallation does + happen.) + */ + collationCounter = 0; + myCmp = function(pArg,n1,p1,n2,p2){ + --collationCounter; + return 0; + }; + rc = capi.sqlite3_create_collation_v2(this.db, "MYCOLLATION", capi.SQLITE_UTF8, + 0, myCmp, 0); + this.db.checkRc(rc); + rc = this.db.selectValue("select 'hi' = 'HI' collate mycollation"); + T.assert(rc>0).assert(-1===collationCounter); + rc = this.db.selectValue("select 'a' = 'b' collate mycollation"); + T.assert(rc>0).assert(-2===collationCounter); + rc = capi.sqlite3_create_collation_v2(this.db, "MYCOLLATION", capi.SQLITE_UTF8, + 0, null, 0); + this.db.checkRc(rc); + rc = 0; + try { + this.db.selectValue("select 'a' = 'b' collate mycollation"); + }catch(e){ + /* Why is e.resultCode not automatically an extended result + code? The DB() class enables those automatically. */ + rc = sqlite3.capi.sqlite3_extended_errcode(this.db); + } + T.assert(capi.SQLITE_ERROR_MISSING_COLLSEQ === rc); + })/*custom collation*/ + + //////////////////////////////////////////////////////////////////////// + .t('Close db', function(){ + T.assert(this.db).assert(wasm.isPtr(this.db.pointer)); + //wasm.sqlite3_wasm_db_reset(this.db); // will leak virtual tables! + this.db.close(); + T.assert(!this.db.pointer); + }) + ;/* end of oo1 checks */ + + //////////////////////////////////////////////////////////////////////// + T.g('kvvfs') + .t({ + name: 'kvvfs is disabled in worker', + predicate: ()=>(isWorker() || "test is only valid in a Worker"), + test: function(sqlite3){ + T.assert( + !capi.sqlite3_vfs_find('kvvfs'), + "Expecting kvvfs to be unregistered." + ); + } + }) + .t({ + name: 'kvvfs in main thread', + predicate: ()=>(isUIThread() + || "local/sessionStorage are unavailable in a Worker"), + test: function(sqlite3){ + const filename = this.kvvfsDbFile = 'session'; + const pVfs = capi.sqlite3_vfs_find('kvvfs'); + T.assert(pVfs); + const JDb = this.JDb = sqlite3.oo1.JsStorageDb; + const unlink = this.kvvfsUnlink = ()=>{JDb.clearStorage(filename)}; + unlink(); + let db = new JDb(filename); + try { + db.exec([ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + db.close(); + db = new JDb(filename); + db.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + }finally{ + db.close(); + } + } + }/*kvvfs sanity checks*/) + .t({ + name: 'kvvfs sqlite3_js_vfs_create_file()', + predicate: ()=>"kvvfs does not currently support this", + test: function(sqlite3){ + let db; + try { + db = new this.JDb(this.kvvfsDbFile); + const exp = capi.sqlite3_js_db_export(db); + db.close(); + this.kvvfsUnlink(); + capi.sqlite3_js_vfs_create_file("kvvfs", this.kvvfsDbFile, exp); + db = new this.JDb(filename); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + }finally{ + db.close(); + this.kvvfsUnlink(); + } + delete this.kvvfsDbFile; + delete this.kvvfsUnlink; + delete this.JDb; + } + }/*kvvfs sqlite3_js_vfs_create_file()*/) + ;/* end kvvfs tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS: Origin-Private File System', + (sqlite3)=>(sqlite3.opfs + ? true : "requires Worker thread in a compatible browser")) + .t({ + name: 'OPFS db sanity checks', + test: async function(sqlite3){ + const filename = this.opfsDbFile = 'sqlite3-tester1.db'; + const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs'); + T.assert(pVfs); + const unlink = this.opfsUnlink = + (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)}; + unlink(); + let db = new sqlite3.oo1.OpfsDb(filename); + try { + db.exec([ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new sqlite3.oo1.OpfsDb(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); + }finally{ + db.close(); + unlink(); + } + } + }/*OPFS db sanity checks*/) + .t({ + name: 'OPFS export/import', + test: async function(sqlite3){ + let db; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp); + const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + } + }/*OPFS export/import*/) + .t({ + name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()', + test: async function(sqlite3){ + const filename = this.opfsDbFile; + const pVfs = this.opfsVfs; + const unlink = this.opfsUnlink; + T.assert(filename && pVfs && !!unlink); + delete this.opfsDbFile; + delete this.opfsVfs; + delete this.opfsUnlink; + unlink(); + // Sanity-test sqlite3_js_vfs_create_file()... + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = 1379; + let sh; + try{ + T.assert(!(await opfs.entryExists(filename))); + capi.sqlite3_js_vfs_create_file( + pVfs, filename, null, fSize + ); + T.assert(await opfs.entryExists(filename)); + let fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + + const ba = new Uint8Array([1,2,3,4,5]); + capi.sqlite3_js_vfs_create_file( + "opfs", filename, ba + ); + T.assert(await opfs.entryExists(filename)); + fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(ba.byteLength === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + + T.mustThrowMatching(()=>{ + capi.sqlite3_js_vfs_create_file( + "no-such-vfs", filename, ba + ); + }, "SQLITE_NOTFOUND: Unknown sqlite3_vfs name: no-such-vfs"); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*OPFS util sanity checks*/) + ;/* end OPFS tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('Session API') + .t({ + name: 'Session API sanity checks', + predicate: ()=>!!capi.sqlite3changegroup_add, + test: function(sqlite3){ + warn("The session API tests could use some expansion."); + const db1 = new sqlite3.oo1.DB(), db2 = new sqlite3.oo1.DB(); + const sqlInit = [ + "create table t(rowid INTEGER PRIMARY KEY,a,b); ", + "insert into t(rowid,a,b) values", + "(1,'a1','b1'),", + "(2,'a2','b2'),", + "(3,'a3','b3');" + ].join(''); + db1.exec(sqlInit); + db2.exec(sqlInit); + T.assert(3 === db1.selectValue("select count(*) from t")) + .assert('b3' === db1.selectValue('select b from t where rowid=3')); + const stackPtr = wasm.pstack.pointer; + try{ + let ppOut = wasm.pstack.allocPtr(); + let rc = capi.sqlite3session_create(db1, "main", ppOut); + T.assert(0===rc); + let pSession = wasm.peekPtr(ppOut); + T.assert(pSession && wasm.isPtr(pSession)); + capi.sqlite3session_table_filter(pSession, (pCtx, tbl)=>{ + T.assert('t' === tbl).assert( 99 === pCtx ); + return 1; + }, 99); + db1.exec([ + "update t set b='bTwo' where rowid=2;", + "update t set a='aThree' where rowid=3;", + "delete from t where rowid=1;", + "insert into t(rowid,a,b) values(4,'a4','b4')" + ]); + T.assert('bTwo' === db1.selectValue("select b from t where rowid=2")) + .assert(undefined === db1.selectValue('select a from t where rowid=1')) + .assert('b4' === db1.selectValue('select b from t where rowid=4')) + .assert(3 === db1.selectValue('select count(*) from t')); + + const testSessionEnable = false; + if(testSessionEnable){ + rc = capi.sqlite3session_enable(pSession, 0); + T.assert( 0 === rc ) + .assert( 0 === capi.sqlite3session_enable(pSession, -1) ); + db1.exec("delete from t where rowid=2;"); + rc = capi.sqlite3session_enable(pSession, 1); + T.assert( rc > 0 ) + .assert( capi.sqlite3session_enable(pSession, -1) > 0 ) + .assert(undefined === db1.selectValue('select a from t where rowid=2')); + }else{ + warn("sqlite3session_enable() tests disabled due to unexpected results.", + "(Possibly a tester misunderstanding, as opposed to a bug.)"); + } + let db1Count = db1.selectValue("select count(*) from t"); + T.assert( db1Count === (testSessionEnable ? 2 : 3) ); + + /* Capture changeset and destroy session. */ + let pnChanges = wasm.pstack.alloc('i32'), + ppChanges = wasm.pstack.allocPtr(); + rc = capi.sqlite3session_changeset(pSession, pnChanges, ppChanges); + T.assert( 0 === rc ); + capi.sqlite3session_delete(pSession); + pSession = 0; + const pChanges = wasm.peekPtr(ppChanges), + nChanges = wasm.peek32(pnChanges); + T.assert( pChanges && wasm.isPtr( pChanges ) ) + .assert( nChanges > 0 ); + + /* Revert db1 via an inverted changeset, but keep pChanges + and nChanges for application to db2. */ + rc = capi.sqlite3changeset_invert( nChanges, pChanges, pnChanges, ppChanges ); + T.assert( 0 === rc ); + rc = capi.sqlite3changeset_apply( + db1, wasm.peek32(pnChanges), wasm.peekPtr(ppChanges), 0, (pCtx, eConflict, pIter)=>{ + return 1; + }, 0 + ); + T.assert( 0 === rc ); + wasm.dealloc( wasm.peekPtr(ppChanges) ); + pnChanges = ppChanges = 0; + T.assert('b2' === db1.selectValue("select b from t where rowid=2")) + .assert('a1' === db1.selectValue('select a from t where rowid=1')) + .assert(undefined === db1.selectValue('select b from t where rowid=4')); + db1Count = db1.selectValue("select count(*) from t"); + T.assert(3 === db1Count); + + /* Apply pre-reverted changeset (pChanges, nChanges) to + db2... */ + rc = capi.sqlite3changeset_apply( + db2, nChanges, pChanges, 0, (pCtx, eConflict, pIter)=>{ + return pCtx ? 1 : 0 + }, 1 + ); + wasm.dealloc( pChanges ); + T.assert( 0 === rc ) + .assert( 'b4' === db2.selectValue('select b from t where rowid=4') ) + .assert( 'aThree' === db2.selectValue('select a from t where rowid=3') ) + .assert( undefined === db2.selectValue('select b from t where rowid=1') ); + if(testSessionEnable){ + T.assert( (undefined === db2.selectValue('select b from t where rowid=2')), + "But... the session was disabled when rowid=2 was deleted?" ); + log("rowids from db2.t:",db2.selectValues('select rowid from t order by rowid')); + T.assert( 3 === db2.selectValue('select count(*) from t') ); + }else{ + T.assert( 'bTwo' === db2.selectValue('select b from t where rowid=2') ) + .assert( 3 === db2.selectValue('select count(*) from t') ); + } + }finally{ + wasm.pstack.restore(stackPtr); + db1.close(); + db2.close(); + } + } + })/*session API sanity tests*/ + ;/*end of session API group*/; + + //////////////////////////////////////////////////////////////////////// + log("Loading and initializing sqlite3 WASM module..."); + if(!self.sqlite3InitModule && !isUIThread()){ + /* Vanilla worker, as opposed to an ES6 module worker */ + /* + If sqlite3.js is in a directory other than this script, in order + to get sqlite3.js to resolve sqlite3.wasm properly, we have to + explicitly tell it where sqlite3.js is being loaded from. We do + that by passing the `sqlite3.dir=theDirName` URL argument to + _this_ script. That URL argument will be seen by the JS/WASM + loader and it will adjust the sqlite3.wasm path accordingly. If + sqlite3.js/.wasm are in the same directory as this script then + that's not needed. + + URL arguments passed as part of the filename via importScripts() + are simply lost, and such scripts see the self.location of + _this_ script. + */ + let sqlite3Js = 'sqlite3.js'; + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; + } + importScripts(sqlite3Js); + } + self.sqlite3InitModule.__isUnderTest = + true /* disables certain API-internal cleanup so that we can + test internal APIs from here */; + self.sqlite3InitModule({ + print: log, + printErr: error + }).then(function(sqlite3){ + //console.log('sqlite3 =',sqlite3); + log("Done initializing WASM/JS bits. Running tests..."); + capi = sqlite3.capi; + wasm = sqlite3.wasm; + log("sqlite3 version:",capi.sqlite3_libversion(), + capi.sqlite3_sourceid()); + if(wasm.bigIntEnabled){ + log("BigInt/int64 support is enabled."); + }else{ + logClass('warning',"BigInt/int64 support is disabled."); + } + if(haveWasmCTests()){ + log("sqlite3_wasm_test_...() APIs are available."); + }else{ + logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); + } + TestUtil.runTests(sqlite3); + }); +})(self); + DELETED ext/wasm/tester1.html Index: ext/wasm/tester1.html ================================================================== --- ext/wasm/tester1.html +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - sqlite3 tester #1 (UI thread) - - - -

    sqlite3 WASM/JS tester #1 (UI thread)

    -
    See tester1-worker.html - for the Worker-thread variant.
    -
    - - -
    -
    - - - - DELETED ext/wasm/tester1.js Index: ext/wasm/tester1.js ================================================================== --- ext/wasm/tester1.js +++ /dev/null @@ -1,1864 +0,0 @@ -/* - 2022-10-12 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - Main functional and regression tests for the sqlite3 WASM API. - - This mini-framework works like so: - - This script adds a series of test groups, each of which contains an - arbitrary number of tests, into a queue. After loading of the - sqlite3 WASM/JS module is complete, that queue is processed. If any - given test fails, the whole thing fails. This script is built such - that it can run from the main UI thread or worker thread. Test - groups and individual tests can be assigned a predicate function - which determines whether to run them or not, and this is - specifically intended to be used to toggle certain tests on or off - for the main/worker threads. - - Each test group defines a state object which gets applied as each - test function's `this`. Test functions can use that to, e.g., set up - a db in an early test and close it in a later test. Each test gets - passed the sqlite3 namespace object as its only argument. -*/ -'use strict'; -(function(){ - /** - Set up our output channel differently depending - on whether we are running in a worker thread or - the main (UI) thread. - */ - let logClass; - /* Predicate for tests/groups. */ - const isUIThread = ()=>(self.window===self && self.document); - /* Predicate for tests/groups. */ - const isWorker = ()=>!isUIThread(); - /* Predicate for tests/groups. */ - const testIsTodo = ()=>false; - const haveWasmCTests = ()=>{ - return !!wasm.exports.sqlite3_wasm_test_intptr; - }; - { - const mapToString = (v)=>{ - switch(typeof v){ - case 'number': case 'string': case 'boolean': - case 'undefined': case 'bigint': - return ''+v; - default: break; - } - if(null===v) return 'null'; - if(v instanceof Error){ - v = { - message: v.message, - stack: v.stack, - errorClass: v.name - }; - } - return JSON.stringify(v,undefined,2); - }; - const normalizeArgs = (args)=>args.map(mapToString); - if( isUIThread() ){ - console.log("Running in the UI thread."); - const logTarget = document.querySelector('#test-output'); - logClass = function(cssClass,...args){ - const ln = document.createElement('div'); - if(cssClass){ - for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ - ln.classList.add(c); - } - } - ln.append(document.createTextNode(normalizeArgs(args).join(' '))); - logTarget.append(ln); - }; - const cbReverse = document.querySelector('#cb-log-reverse'); - const cbReverseKey = 'tester1:cb-log-reverse'; - const cbReverseIt = ()=>{ - logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); - //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); - }; - cbReverse.addEventListener('change', cbReverseIt, true); - /*if(localStorage.getItem(cbReverseKey)){ - cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); - }*/ - cbReverseIt(); - }else{ /* Worker thread */ - console.log("Running in a Worker thread."); - logClass = function(cssClass,...args){ - postMessage({ - type:'log', - payload:{cssClass, args: normalizeArgs(args)} - }); - }; - } - } - const reportFinalTestStatus = function(pass){ - if(isUIThread()){ - const e = document.querySelector('#color-target'); - e.classList.add(pass ? 'tests-pass' : 'tests-fail'); - }else{ - postMessage({type:'test-result', payload:{pass}}); - } - }; - const log = (...args)=>{ - //console.log(...args); - logClass('',...args); - } - const warn = (...args)=>{ - console.warn(...args); - logClass('warning',...args); - } - const error = (...args)=>{ - console.error(...args); - logClass('error',...args); - }; - - const toss = (...args)=>{ - error(...args); - throw new Error(args.join(' ')); - }; - const tossQuietly = (...args)=>{ - throw new Error(args.join(' ')); - }; - - const roundMs = (ms)=>Math.round(ms*100)/100; - - /** - Helpers for writing sqlite3-specific tests. - */ - const TestUtil = { - /** Running total of the number of tests run via - this API. */ - counter: 0, - /* Separator line for log messages. */ - separator: '------------------------------------------------------------', - /** - If expr is a function, it is called and its result - is returned, coerced to a bool, else expr, coerced to - a bool, is returned. - */ - toBool: function(expr){ - return (expr instanceof Function) ? !!expr() : !!expr; - }, - /** Throws if expr is false. If expr is a function, it is called - and its result is evaluated. If passed multiple arguments, - those after the first are a message string which get applied - as an exception message if the assertion fails. The message - arguments are concatenated together with a space between each. - */ - assert: function f(expr, ...msg){ - ++this.counter; - if(!this.toBool(expr)){ - throw new Error(msg.length ? msg.join(' ') : "Assertion failed."); - } - return this; - }, - /** Calls f() and squelches any exception it throws. If it - does not throw, this function throws. */ - mustThrow: function(f, msg){ - ++this.counter; - let err; - try{ f(); } catch(e){err=e;} - if(!err) throw new Error(msg || "Expected exception."); - return this; - }, - /** - Works like mustThrow() but expects filter to be a regex, - function, or string to match/filter the resulting exception - against. If f() does not throw, this test fails and an Error is - thrown. If filter is a regex, the test passes if - filter.test(error.message) passes. If it's a function, the test - passes if filter(error) returns truthy. If it's a string, the - test passes if the filter matches the exception message - precisely. In all other cases the test fails, throwing an - Error. - - If it throws, msg is used as the error report unless it's falsy, - in which case a default is used. - */ - mustThrowMatching: function(f, filter, msg){ - ++this.counter; - let err; - try{ f(); } catch(e){err=e;} - if(!err) throw new Error(msg || "Expected exception."); - let pass = false; - if(filter instanceof RegExp) pass = filter.test(err.message); - else if(filter instanceof Function) pass = filter(err); - else if('string' === typeof filter) pass = (err.message === filter); - if(!pass){ - throw new Error(msg || ("Filter rejected this exception: "+err.message)); - } - return this; - }, - /** Throws if expr is truthy or expr is a function and expr() - returns truthy. */ - throwIf: function(expr, msg){ - ++this.counter; - if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); - return this; - }, - /** Throws if expr is falsy or expr is a function and expr() - returns falsy. */ - throwUnless: function(expr, msg){ - ++this.counter; - if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); - return this; - }, - eqApprox: (v1,v2,factor=0.05)=>(v1>=(v2-factor) && v1<=(v2+factor)), - TestGroup: (function(){ - let groupCounter = 0; - const TestGroup = function(name, predicate){ - this.number = ++groupCounter; - this.name = name; - this.predicate = predicate; - this.tests = []; - }; - TestGroup.prototype = { - addTest: function(testObj){ - this.tests.push(testObj); - return this; - }, - run: async function(sqlite3){ - log(TestUtil.separator); - logClass('group-start',"Group #"+this.number+':',this.name); - const indent = ' '; - if(this.predicate && !this.predicate(sqlite3)){ - logClass('warning',indent, - "SKIPPING group because predicate says to."); - return; - } - const assertCount = TestUtil.counter; - const groupState = Object.create(null); - const skipped = []; - let runtime = 0, i = 0; - for(const t of this.tests){ - ++i; - const n = this.number+"."+i; - log(indent, n+":", t.name); - if(t.predicate && !t.predicate(sqlite3)){ - logClass('warning', indent, indent, - 'SKIPPING because predicate says to'); - skipped.push( n+': '+t.name ); - }else{ - const tc = TestUtil.counter, now = performance.now(); - await t.test.call(groupState, sqlite3); - const then = performance.now(); - runtime += then - now; - logClass('faded',indent, indent, - TestUtil.counter - tc, 'assertion(s) in', - roundMs(then-now),'ms'); - } - } - logClass('green', - "Group #"+this.number+":",(TestUtil.counter - assertCount), - "assertion(s) in",roundMs(runtime),"ms"); - if(skipped.length){ - logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped); - } - } - }; - return TestGroup; - })()/*TestGroup*/, - testGroups: [], - currentTestGroup: undefined, - addGroup: function(name, predicate){ - this.testGroups.push( this.currentTestGroup = - new this.TestGroup(name, predicate) ); - return this; - }, - addTest: function(name, callback){ - let predicate; - if(1===arguments.length){ - const opt = arguments[0]; - predicate = opt.predicate; - name = opt.name; - callback = opt.test; - } - this.currentTestGroup.addTest({ - name, predicate, test: callback - }); - return this; - }, - runTests: async function(sqlite3){ - return new Promise(async function(pok,pnok){ - try { - let runtime = 0; - for(let g of this.testGroups){ - const now = performance.now(); - await g.run(sqlite3); - runtime += performance.now() - now; - } - log(TestUtil.separator); - logClass(['strong','green'], - "Done running tests.",TestUtil.counter,"assertions in", - roundMs(runtime),'ms'); - pok(); - reportFinalTestStatus(true); - }catch(e){ - error(e); - pnok(e); - reportFinalTestStatus(false); - } - }.bind(this)); - } - }/*TestUtil*/; - const T = TestUtil; - T.g = T.addGroup; - T.t = T.addTest; - let capi, wasm/*assigned after module init*/; - //////////////////////////////////////////////////////////////////////// - // End of infrastructure setup. Now define the tests... - //////////////////////////////////////////////////////////////////////// - - //////////////////////////////////////////////////////////////////// - T.g('Basic sanity checks') - .t('Namespace object checks', function(sqlite3){ - const wasmCtypes = wasm.ctype; - T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). - assert(wasmCtypes.structs[0].members.szOsFile.sizeof>=4). - assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ - ].members.xFileSize.offset>0); - [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', - 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', - 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' - ].forEach((k)=>T.assert('number' === typeof capi[k])); - [/* Spot-check a few of the WASM API methods. */ - 'alloc', 'dealloc', 'installFunction' - ].forEach((k)=>T.assert(wasm[k] instanceof Function)); - - T.assert(capi.sqlite3_errstr(capi.SQLITE_IOERR_ACCESS).indexOf("I/O")>=0). - assert(capi.sqlite3_errstr(capi.SQLITE_CORRUPT).indexOf('malformed')>0). - assert(capi.sqlite3_errstr(capi.SQLITE_OK) === 'not an error'); - - try { - throw new sqlite3.WasmAllocError; - }catch(e){ - T.assert(e instanceof Error) - .assert(e instanceof sqlite3.WasmAllocError) - .assert("Allocation failed." === e.message); - } - try { - throw new sqlite3.WasmAllocError("test",{ - cause: 3 - }); - }catch(e){ - T.assert(3 === e.cause) - .assert("test" === e.message); - } - try {throw new sqlite3.WasmAllocError("test","ing",".")} - catch(e){T.assert("test ing ." === e.message)} - - try{ throw new sqlite3.SQLite3Error(capi.SQLITE_SCHEMA) } - catch(e){ T.assert('SQLITE_SCHEMA' === e.message) } - try{ sqlite3.SQLite3Error.toss(capi.SQLITE_CORRUPT,{cause: true}) } - catch(e){ - T.assert('SQLITE_CORRUPT'===e.message) - .assert(true===e.cause); - } - }) - //////////////////////////////////////////////////////////////////// - .t('strglob/strlike', function(sqlite3){ - T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). - assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). - assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). - assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); - }) - //////////////////////////////////////////////////////////////////// - ;/*end of basic sanity checks*/ - - //////////////////////////////////////////////////////////////////// - T.g('C/WASM Utilities') - .t('sqlite3.wasm namespace', function(sqlite3){ - const w = wasm; - const chr = (x)=>x.charCodeAt(0); - //log("heap getters..."); - { - const li = [8, 16, 32]; - if(w.bigIntEnabled) li.push(64); - for(const n of li){ - const bpe = n/8; - const s = w.heapForSize(n,false); - T.assert(bpe===s.BYTES_PER_ELEMENT). - assert(w.heapForSize(s.constructor) === s); - const u = w.heapForSize(n,true); - T.assert(bpe===u.BYTES_PER_ELEMENT). - assert(s!==u). - assert(w.heapForSize(u.constructor) === u); - } - } - - // isPtr32() - { - const ip = w.isPtr32; - T.assert(ip(0)) - .assert(!ip(-1)) - .assert(!ip(1.1)) - .assert(!ip(0xffffffff)) - .assert(ip(0x7fffffff)) - .assert(!ip()) - .assert(!ip(null)/*might change: under consideration*/) - ; - } - - //log("jstrlen()..."); - { - T.assert(3 === w.jstrlen("abc")).assert(4 === w.jstrlen("äbc")); - } - - //log("jstrcpy()..."); - { - const fillChar = 10; - let ua = new Uint8Array(8), rc, - refill = ()=>ua.fill(fillChar); - refill(); - rc = w.jstrcpy("hello", ua); - T.assert(6===rc).assert(0===ua[5]).assert(chr('o')===ua[4]); - refill(); - ua[5] = chr('!'); - rc = w.jstrcpy("HELLO", ua, 0, -1, false); - T.assert(5===rc).assert(chr('!')===ua[5]).assert(chr('O')===ua[4]); - refill(); - rc = w.jstrcpy("the end", ua, 4); - //log("rc,ua",rc,ua); - T.assert(4===rc).assert(0===ua[7]). - assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); - refill(); - rc = w.jstrcpy("the end", ua, 4, -1, false); - T.assert(4===rc).assert(chr(' ')===ua[7]). - assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); - refill(); - rc = w.jstrcpy("", ua, 0, 1, true); - //log("rc,ua",rc,ua); - T.assert(1===rc).assert(0===ua[0]); - refill(); - rc = w.jstrcpy("x", ua, 0, 1, true); - //log("rc,ua",rc,ua); - T.assert(1===rc).assert(0===ua[0]); - refill(); - rc = w.jstrcpy('äbä', ua, 0, 1, true); - T.assert(1===rc, 'Must not write partial multi-byte char.') - .assert(0===ua[0]); - refill(); - rc = w.jstrcpy('äbä', ua, 0, 2, true); - T.assert(1===rc, 'Must not write partial multi-byte char.') - .assert(0===ua[0]); - refill(); - rc = w.jstrcpy('äbä', ua, 0, 2, false); - T.assert(2===rc).assert(fillChar!==ua[1]).assert(fillChar===ua[2]); - }/*jstrcpy()*/ - - //log("cstrncpy()..."); - { - const scope = w.scopedAllocPush(); - try { - let cStr = w.scopedAllocCString("hello"); - const n = w.cstrlen(cStr); - let cpy = w.scopedAlloc(n+10); - let rc = w.cstrncpy(cpy, cStr, n+10); - T.assert(n+1 === rc). - assert("hello" === w.cstringToJs(cpy)). - assert(chr('o') === w.getMemValue(cpy+n-1)). - assert(0 === w.getMemValue(cpy+n)); - let cStr2 = w.scopedAllocCString("HI!!!"); - rc = w.cstrncpy(cpy, cStr2, 3); - T.assert(3===rc). - assert("HI!lo" === w.cstringToJs(cpy)). - assert(chr('!') === w.getMemValue(cpy+2)). - assert(chr('l') === w.getMemValue(cpy+3)); - }finally{ - w.scopedAllocPop(scope); - } - } - - //log("jstrToUintArray()..."); - { - let a = w.jstrToUintArray("hello", false); - T.assert(5===a.byteLength).assert(chr('o')===a[4]); - a = w.jstrToUintArray("hello", true); - T.assert(6===a.byteLength).assert(chr('o')===a[4]).assert(0===a[5]); - a = w.jstrToUintArray("äbä", false); - T.assert(5===a.byteLength).assert(chr('b')===a[2]); - a = w.jstrToUintArray("äbä", true); - T.assert(6===a.byteLength).assert(chr('b')===a[2]).assert(0===a[5]); - } - - //log("allocCString()..."); - { - const cstr = w.allocCString("hällo, world"); - const n = w.cstrlen(cstr); - T.assert(13 === n) - .assert(0===w.getMemValue(cstr+n)) - .assert(chr('d')===w.getMemValue(cstr+n-1)); - } - - //log("scopedAlloc() and friends..."); - { - const alloc = w.alloc, dealloc = w.dealloc; - w.alloc = w.dealloc = null; - T.assert(!w.scopedAlloc.level) - .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) - .mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); - w.alloc = alloc; - T.mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); - w.dealloc = dealloc; - T.mustThrowMatching(()=>w.scopedAllocPop(), /^Invalid state/) - .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) - .mustThrowMatching(()=>w.scopedAlloc.level=0, /read-only/); - const asc = w.scopedAllocPush(); - let asc2; - try { - const p1 = w.scopedAlloc(16), - p2 = w.scopedAlloc(16); - T.assert(1===w.scopedAlloc.level) - .assert(Number.isFinite(p1)) - .assert(Number.isFinite(p2)) - .assert(asc[0] === p1) - .assert(asc[1]===p2); - asc2 = w.scopedAllocPush(); - const p3 = w.scopedAlloc(16); - T.assert(2===w.scopedAlloc.level) - .assert(Number.isFinite(p3)) - .assert(2===asc.length) - .assert(p3===asc2[0]); - - const [z1, z2, z3] = w.scopedAllocPtr(3); - T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) - .assert(0===w.getMemValue(z1,'i32'), 'allocPtr() must zero the targets') - .assert(0===w.getMemValue(z3,'i32')); - }finally{ - // Pop them in "incorrect" order to make sure they behave: - w.scopedAllocPop(asc); - T.assert(0===asc.length); - T.mustThrowMatching(()=>w.scopedAllocPop(asc), - /^Invalid state object/); - if(asc2){ - T.assert(2===asc2.length,'Should be p3 and z1'); - w.scopedAllocPop(asc2); - T.assert(0===asc2.length); - T.mustThrowMatching(()=>w.scopedAllocPop(asc2), - /^Invalid state object/); - } - } - T.assert(0===w.scopedAlloc.level); - w.scopedAllocCall(function(){ - T.assert(1===w.scopedAlloc.level); - const [cstr, n] = w.scopedAllocCString("hello, world", true); - T.assert(12 === n) - .assert(0===w.getMemValue(cstr+n)) - .assert(chr('d')===w.getMemValue(cstr+n-1)); - }); - }/*scopedAlloc()*/ - - //log("xCall()..."); - { - const pJson = w.xCall('sqlite3_wasm_enum_json'); - T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); - } - - //log("xWrap()..."); - { - T.mustThrowMatching(()=>w.xWrap('sqlite3_libversion',null,'i32'), - /requires 0 arg/). - assert(w.xWrap.resultAdapter('i32') instanceof Function). - assert(w.xWrap.argAdapter('i32') instanceof Function); - let fw = w.xWrap('sqlite3_libversion','utf8'); - T.mustThrowMatching(()=>fw(1), /requires 0 arg/); - let rc = fw(); - T.assert('string'===typeof rc).assert(rc.length>5); - rc = w.xCallWrapped('sqlite3_wasm_enum_json','*'); - T.assert(rc>0 && Number.isFinite(rc)); - rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); - T.assert('string'===typeof rc).assert(rc.length>300); - if(haveWasmCTests()){ - fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:free',['i32']); - rc = fw(0); - T.assert('hello'===rc); - rc = fw(1); - T.assert(null===rc); - - if(w.bigIntEnabled){ - w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); - w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); - fw = w.xWrap('sqlite3_wasm_test_int64_times2','thrice','twice'); - rc = fw(1); - T.assert(12n===rc); - - w.scopedAllocCall(function(){ - let pI1 = w.scopedAlloc(8), pI2 = pI1+4; - w.setMemValue(pI1, 0,'*')(pI2, 0, '*'); - let f = w.xWrap('sqlite3_wasm_test_int64_minmax',undefined,['i64*','i64*']); - let r1 = w.getMemValue(pI1, 'i64'), r2 = w.getMemValue(pI2, 'i64'); - T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); - }); - } - } - } - }/*WhWasmUtil*/) - - //////////////////////////////////////////////////////////////////// - .t('sqlite3.StructBinder (jaccwabyt)', function(sqlite3){ - const S = sqlite3, W = S.wasm; - const MyStructDef = { - sizeof: 16, - members: { - p4: {offset: 0, sizeof: 4, signature: "i"}, - pP: {offset: 4, sizeof: 4, signature: "P"}, - ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, - cstr: {offset: 12, sizeof: 4, signature: "s"} - } - }; - if(W.bigIntEnabled){ - const m = MyStructDef; - m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; - m.sizeof += m.members.p8.sizeof; - } - const StructType = S.StructBinder.StructType; - const K = S.StructBinder('my_struct',MyStructDef); - T.mustThrowMatching(()=>K(), /via 'new'/). - mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); - const k1 = new K(), k2 = new K(); - try { - T.assert(k1.constructor === K). - assert(K.isA(k1)). - assert(k1 instanceof K). - assert(K.prototype.lookupMember('p4').key === '$p4'). - assert(K.prototype.lookupMember('$p4').name === 'p4'). - mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). - assert(undefined === K.prototype.lookupMember('nope',false)). - assert(k1 instanceof StructType). - assert(StructType.isA(k1)). - assert(K.resolveToInstance(k1.pointer)===k1). - mustThrowMatching(()=>K.resolveToInstance(null,true), /is-not-a my_struct/). - assert(k1 === StructType.instanceForPointer(k1.pointer)). - mustThrowMatching(()=>k1.$ro = 1, /read-only/); - Object.keys(MyStructDef.members).forEach(function(key){ - key = K.memberKey(key); - T.assert(0 == k1[key], - "Expecting allocation to zero the memory "+ - "for "+key+" but got: "+k1[key]+ - " from "+k1.memoryDump()); - }); - T.assert('number' === typeof k1.pointer). - mustThrowMatching(()=>k1.pointer = 1, /pointer/). - assert(K.instanceForPointer(k1.pointer) === k1); - k1.$p4 = 1; k1.$pP = 2; - T.assert(1 === k1.$p4).assert(2 === k1.$pP); - if(MyStructDef.members.$p8){ - k1.$p8 = 1/*must not throw despite not being a BigInt*/; - k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); - T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); - } - T.assert(!k1.ondispose); - k1.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(k1.ondispose)). - assert(k1.ondispose[0] === k1.$cstr). - assert('number' === typeof k1.$cstr). - assert('A C-string.' === k1.memberToJsString('cstr')); - k1.$pP = k2; - T.assert(k1.$pP === k2); - k1.$pP = null/*null is special-cased to 0.*/; - T.assert(0===k1.$pP); - let ptr = k1.pointer; - k1.dispose(); - T.assert(undefined === k1.pointer). - assert(undefined === K.instanceForPointer(ptr)). - mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); - const k3 = new K(); - ptr = k3.pointer; - T.assert(k3 === K.instanceForPointer(ptr)); - K.disposeAll(); - T.assert(ptr). - assert(undefined === k2.pointer). - assert(undefined === k3.pointer). - assert(undefined === K.instanceForPointer(ptr)); - }finally{ - k1.dispose(); - k2.dispose(); - } - - if(!W.bigIntEnabled){ - log("Skipping WasmTestStruct tests: BigInt not enabled."); - return; - } - - const WTStructDesc = - W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; - const autoResolvePtr = true /* EXPERIMENTAL */; - if(autoResolvePtr){ - WTStructDesc.members.ppV.signature = 'P'; - } - const WTStruct = S.StructBinder(WTStructDesc); - //log(WTStruct.structName, WTStruct.structInfo); - const wts = new WTStruct(); - //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); - try{ - T.assert(wts.constructor === WTStruct). - assert(WTStruct.memberKeys().indexOf('$ppV')>=0). - assert(wts.memberKeys().indexOf('$v8')>=0). - assert(!K.isA(wts)). - assert(WTStruct.isA(wts)). - assert(wts instanceof WTStruct). - assert(wts instanceof StructType). - assert(StructType.isA(wts)). - assert(wts === StructType.instanceForPointer(wts.pointer)); - T.assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). - assert(0===wts.$ppV).assert(0===wts.$xFunc). - assert(WTStruct.instanceForPointer(wts.pointer) === wts); - const testFunc = - W.xGet('sqlite3_wasm_test_struct'/*name gets mangled in -O3 builds!*/); - let counter = 0; - //log("wts.pointer =",wts.pointer); - const wtsFunc = function(arg){ - /*log("This from a JS function called from C, "+ - "which itself was called from JS. arg =",arg);*/ - ++counter; - T.assert(WTStruct.instanceForPointer(arg) === wts); - if(3===counter){ - tossQuietly("Testing exception propagation."); - } - } - wts.$v4 = 10; wts.$v8 = 20; - wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) - T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) - .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) - .assert(0 === wts.$cstr) - .assert(wts.memberIsString('$cstr')) - .assert(!wts.memberIsString('$v4')) - .assert(null === wts.memberToJsString('$cstr')) - .assert(W.functionEntry(wts.$xFunc) instanceof Function); - /* It might seem silly to assert that the values match - what we just set, but recall that all of those property - reads and writes are, via property interceptors, - actually marshaling their data to/from a raw memory - buffer, so merely reading them back is actually part of - testing the struct-wrapping API. */ - - testFunc(wts.pointer); - //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); - T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) - .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)) - .assert('string' === typeof wts.memberToJsString('cstr')) - .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) - .mustThrowMatching(()=>wts.memberToJsString('xFunc'), - /Invalid member type signature for C-string/) - ; - testFunc(wts.pointer); - T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) - .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)); - /** The 3rd call to wtsFunc throw from JS, which is called - from C, which is called from JS. Let's ensure that - that exception propagates back here... */ - T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); - W.uninstallFunction(wts.$xFunc); - wts.$xFunc = 0; - if(autoResolvePtr){ - wts.$ppV = 0; - T.assert(!wts.$ppV); - //WTStruct.debugFlags(0x03); - wts.$ppV = wts; - T.assert(wts === wts.$ppV) - //WTStruct.debugFlags(0); - } - wts.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(wts.ondispose)). - assert(wts.ondispose[0] === wts.$cstr). - assert('A C-string.' === wts.memberToJsString('cstr')); - const ptr = wts.pointer; - wts.dispose(); - T.assert(ptr).assert(undefined === wts.pointer). - assert(undefined === WTStruct.instanceForPointer(ptr)) - }finally{ - wts.dispose(); - } - }/*StructBinder*/) - - //////////////////////////////////////////////////////////////////// - .t('sqlite3.StructBinder part 2', function(sqlite3){ - // https://www.sqlite.org/c3ref/vfs.html - // https://www.sqlite.org/c3ref/io_methods.html - const sqlite3_io_methods = capi.sqlite3_io_methods, - sqlite3_vfs = capi.sqlite3_vfs, - sqlite3_file = capi.sqlite3_file; - //log("struct sqlite3_file", sqlite3_file.memberKeys()); - //log("struct sqlite3_vfs", sqlite3_vfs.memberKeys()); - //log("struct sqlite3_io_methods", sqlite3_io_methods.memberKeys()); - const installMethod = function callee(tgt, name, func){ - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.ondisposeRemoveFunc = function(){ - if(this.__ondispose){ - const who = this; - this.__ondispose.forEach( - (v)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - }else{/*wasm function wrapper property*/ - delete who[v]; - } - } - ); - delete this.__ondispose; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name), - memKey = tgt.memberKey(name); - //log("installMethod",tgt, name, sigN); - if(!tgt.__ondispose){ - T.assert(undefined === tgt.ondispose); - tgt.ondispose = [callee.ondisposeRemoveFunc]; - tgt.__ondispose = []; - } - const fProxy = callee.argcProxy(func, sigN); - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - /** - ACHTUNG: function pointer IDs are from a different pool than - allocation IDs, starting at 1 and incrementing in steps of 1, - so if we set tgt[memKey] to those values, we'd very likely - later misinterpret them as plain old pointer addresses unless - unless we use some silly heuristic like "all values <5k are - presumably function pointers," or actually perform a function - lookup on every pointer to first see if it's a function. That - would likely work just fine, but would be kludgy. - - It turns out that "all values less than X are functions" is - essentially how it works in wasm: a function pointer is - reported to the client as its index into the - __indirect_function_table. - - So... once jaccwabyt can be told how to access the - function table, it could consider all pointer values less - than that table's size to be functions. As "real" pointer - values start much, much higher than the function table size, - that would likely work reasonably well. e.g. the object - pointer address for sqlite3's default VFS is (in this local - setup) 65104, whereas the function table has fewer than 600 - entries. - */ - const wrapperKey = '$'+memKey; - tgt[wrapperKey] = fProxy; - tgt.__ondispose.push(pFunc, wrapperKey); - //log("tgt.__ondispose =",tgt.__ondispose); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - - const installIOMethods = function instm(iom){ - (iom instanceof capi.sqlite3_io_methods) || toss("Invalid argument type."); - if(!instm._requireFileArg){ - instm._requireFileArg = function(arg,methodName){ - arg = capi.sqlite3_file.resolveToInstance(arg); - if(!arg){ - err("sqlite3_io_methods::xClose() was passed a non-sqlite3_file."); - } - return arg; - }; - instm._methods = { - // https://sqlite.org/c3ref/io_methods.html - xClose: /*i(P)*/function(f){ - /* int (*xClose)(sqlite3_file*) */ - log("xClose(",f,")"); - if(!(f = instm._requireFileArg(f,'xClose'))) return capi.SQLITE_MISUSE; - f.dispose(/*noting that f has externally-owned memory*/); - return 0; - }, - xRead: /*i(Ppij)*/function(f,dest,n,offset){ - /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ - log("xRead(",arguments,")"); - if(!(f = instm._requireFileArg(f))) return capi.SQLITE_MISUSE; - wasm.heap8().fill(0, dest + offset, n); - return 0; - }, - xWrite: /*i(Ppij)*/function(f,dest,n,offset){ - /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ - log("xWrite(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xWrite'))) return capi.SQLITE_MISUSE; - return 0; - }, - xTruncate: /*i(Pj)*/function(f){ - /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ - log("xTruncate(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xTruncate'))) return capi.SQLITE_MISUSE; - return 0; - }, - xSync: /*i(Pi)*/function(f){ - /* int (*xSync)(sqlite3_file*, int flags) */ - log("xSync(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xSync'))) return capi.SQLITE_MISUSE; - return 0; - }, - xFileSize: /*i(Pp)*/function(f,pSz){ - /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ - log("xFileSize(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xFileSize'))) return capi.SQLITE_MISUSE; - wasm.setMemValue(pSz, 0/*file size*/); - return 0; - }, - xLock: /*i(Pi)*/function(f){ - /* int (*xLock)(sqlite3_file*, int) */ - log("xLock(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xLock'))) return capi.SQLITE_MISUSE; - return 0; - }, - xUnlock: /*i(Pi)*/function(f){ - /* int (*xUnlock)(sqlite3_file*, int) */ - log("xUnlock(",arguments,")"); - if(!(f=instm._requireFileArg(f,'xUnlock'))) return capi.SQLITE_MISUSE; - return 0; - }, - xCheckReservedLock: /*i(Pp)*/function(){ - /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ - log("xCheckReservedLock(",arguments,")"); - return 0; - }, - xFileControl: /*i(Pip)*/function(){ - /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ - log("xFileControl(",arguments,")"); - return capi.SQLITE_NOTFOUND; - }, - xSectorSize: /*i(P)*/function(){ - /* int (*xSectorSize)(sqlite3_file*) */ - log("xSectorSize(",arguments,")"); - return 0/*???*/; - }, - xDeviceCharacteristics:/*i(P)*/function(){ - /* int (*xDeviceCharacteristics)(sqlite3_file*) */ - log("xDeviceCharacteristics(",arguments,")"); - return 0; - } - }; - }/*static init*/ - iom.$iVersion = 1; - Object.keys(instm._methods).forEach( - (k)=>installMethod(iom, k, instm._methods[k]) - ); - }/*installIOMethods()*/; - - const iom = new sqlite3_io_methods, sfile = new sqlite3_file; - const err = console.error.bind(console); - try { - const IOM = sqlite3_io_methods, S3F = sqlite3_file; - //log("iom proto",iom,iom.constructor.prototype); - //log("sfile",sfile,sfile.constructor.prototype); - T.assert(0===sfile.$pMethods).assert(iom.pointer > 0); - //log("iom",iom); - sfile.$pMethods = iom.pointer; - T.assert(iom.pointer === sfile.$pMethods) - .assert(IOM.resolveToInstance(iom)) - .assert(undefined ===IOM.resolveToInstance(sfile)) - .mustThrow(()=>IOM.resolveToInstance(0,true)) - .assert(S3F.resolveToInstance(sfile.pointer)) - .assert(undefined===S3F.resolveToInstance(iom)) - .assert(iom===IOM.resolveToInstance(sfile.$pMethods)); - T.assert(0===iom.$iVersion); - installIOMethods(iom); - T.assert(1===iom.$iVersion); - //log("iom.__ondispose",iom.__ondispose); - T.assert(Array.isArray(iom.__ondispose)).assert(iom.__ondispose.length>10); - }finally{ - iom.dispose(); - T.assert(undefined === iom.__ondispose); - } - - const dVfs = new sqlite3_vfs(capi.sqlite3_vfs_find(null)); - try { - const SB = sqlite3.StructBinder; - T.assert(dVfs instanceof SB.StructType) - .assert(dVfs.pointer) - .assert('sqlite3_vfs' === dVfs.structName) - .assert(!!dVfs.structInfo) - .assert(SB.StructType.hasExternalPointer(dVfs)) - .assert(dVfs.$iVersion>0) - .assert('number'===typeof dVfs.$zName) - .assert('number'===typeof dVfs.$xSleep) - .assert(wasm.functionEntry(dVfs.$xOpen)) - .assert(dVfs.memberIsString('zName')) - .assert(dVfs.memberIsString('$zName')) - .assert(!dVfs.memberIsString('pAppData')) - .mustThrowMatching(()=>dVfs.memberToJsString('xSleep'), - /Invalid member type signature for C-string/) - .mustThrowMatching(()=>dVfs.memberSignature('nope'), /nope is not a mapped/) - .assert('string' === typeof dVfs.memberToJsString('zName')) - .assert(dVfs.memberToJsString('zName')===dVfs.memberToJsString('$zName')) - ; - //log("Default VFS: @",dVfs.pointer); - Object.keys(sqlite3_vfs.structInfo.members).forEach(function(mname){ - const mk = sqlite3_vfs.memberKey(mname), mbr = sqlite3_vfs.structInfo.members[mname], - addr = dVfs[mk], prefix = 'defaultVfs.'+mname; - if(1===mbr.signature.length){ - let sep = '?', val = undefined; - switch(mbr.signature[0]){ - // TODO: move this into an accessor, e.g. getPreferredValue(member) - case 'i': case 'j': case 'f': case 'd': sep = '='; val = dVfs[mk]; break - case 'p': case 'P': sep = '@'; val = dVfs[mk]; break; - case 's': sep = '='; - val = dVfs.memberToJsString(mname); - break; - } - //log(prefix, sep, val); - }else{ - //log(prefix," = funcptr @",addr, wasm.functionEntry(addr)); - } - }); - }finally{ - dVfs.dispose(); - T.assert(undefined===dVfs.pointer); - } - }/*StructBinder part 2*/) - - //////////////////////////////////////////////////////////////////// - .t('sqlite3.wasm.pstack', function(sqlite3){ - const P = wasm.pstack; - const isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; - const stack = P.pointer; - T.assert(0===stack % 8 /* must be 8-byte aligned */); - try{ - const remaining = P.remaining; - T.assert(P.quota >= 4096) - .assert(remaining === P.quota) - .mustThrowMatching(()=>P.alloc(0), isAllocErr) - .mustThrowMatching(()=>P.alloc(-1), isAllocErr); - let p1 = P.alloc(12); - T.assert(p1 === stack - 16/*8-byte aligned*/) - .assert(P.pointer === p1); - let p2 = P.alloc(7); - T.assert(p2 === p1-8/*8-byte aligned, stack grows downwards*/) - .mustThrowMatching(()=>P.alloc(remaining), isAllocErr) - .assert(24 === stack - p2) - .assert(P.pointer === p2); - let n = remaining - (stack - p2); - let p3 = P.alloc(n); - T.assert(p3 === stack-remaining) - .mustThrowMatching(()=>P.alloc(1), isAllocErr); - }finally{ - P.restore(stack); - } - - T.assert(P.pointer === stack); - try { - const [p1, p2, p3] = P.allocChunks(3,4); - T.assert(P.pointer === stack-16/*always rounded to multiple of 8*/) - .assert(p2 === p1 + 4) - .assert(p3 === p2 + 4); - T.mustThrowMatching(()=>P.allocChunks(1024, 1024 * 16), - (e)=>e instanceof sqlite3.WasmAllocError) - }finally{ - P.restore(stack); - } - - T.assert(P.pointer === stack); - try { - let [p1, p2, p3] = P.allocPtr(3,false); - let sPos = stack-16/*always rounded to multiple of 8*/; - T.assert(P.pointer === sPos) - .assert(p2 === p1 + 4) - .assert(p3 === p2 + 4); - [p1, p2, p3] = P.allocPtr(3); - T.assert(P.pointer === sPos-24/*3 x 8 bytes*/) - .assert(p2 === p1 + 8) - .assert(p3 === p2 + 8); - p1 = P.allocPtr(); - T.assert('number'===typeof p1); - }finally{ - P.restore(stack); - } - }/*pstack tests*/) - - //////////////////////////////////////////////////////////////////// - ;/*end of C/WASM utils checks*/ - - T.g('sqlite3_randomness()') - .t('To memory buffer', function(sqlite3){ - const stack = wasm.pstack.pointer; - try{ - const n = 520; - const p = wasm.pstack.alloc(n); - T.assert(0===wasm.getMemValue(p)) - .assert(0===wasm.getMemValue(p+n-1)); - T.assert(undefined === capi.sqlite3_randomness(n - 10, p)); - let j, check = 0; - const heap = wasm.heap8u(); - for(j = 0; j < 10 && 0===check; ++j){ - check += heap[p + j]; - } - T.assert(check > 0); - check = 0; - // Ensure that the trailing bytes were not modified... - for(j = n - 10; j < n && 0===check; ++j){ - check += heap[p + j]; - } - T.assert(0===check); - }finally{ - wasm.pstack.restore(stack); - } - }) - .t('To byte array', function(sqlite3){ - const ta = new Uint8Array(117); - let i, n = 0; - for(i=0; i0); - const t0 = new Uint8Array(0); - T.assert(t0 === capi.sqlite3_randomness(t0), - "0-length array is a special case"); - }) - ;/*end sqlite3_randomness() checks*/ - - //////////////////////////////////////////////////////////////////////// - T.g('sqlite3.oo1') - .t('Create db', function(sqlite3){ - const dbFile = '/tester1.db'; - wasm.sqlite3_wasm_vfs_unlink(0, dbFile); - const db = this.db = new sqlite3.oo1.DB(dbFile); - T.assert(Number.isInteger(db.pointer)) - .mustThrowMatching(()=>db.pointer=1, /read-only/) - .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) - .assert('main'===db.dbName(0)) - .assert('string' === typeof db.dbVfsName()); - // Custom db error message handling via sqlite3_prepare_v2/v3() - let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); - T.assert(capi.SQLITE_MISUSE === rc) - .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) - .assert(dbFile === db.dbFilename()) - .assert(!db.dbFilename('nope')); - }) - - //////////////////////////////////////////////////////////////////// - .t('DB.Stmt', function(S){ - let st = this.db.prepare( - new TextEncoder('utf-8').encode("select 3 as a") - ); - //debug("statement =",st); - try { - T.assert(Number.isInteger(st.pointer)) - .mustThrowMatching(()=>st.pointer=1, /read-only/) - .assert(1===this.db.openStatementCount()) - .assert(!st._mayGet) - .assert('a' === st.getColumnName(0)) - .assert(1===st.columnCount) - .assert(0===st.parameterCount) - .mustThrow(()=>st.bind(1,null)) - .assert(true===st.step()) - .assert(3 === st.get(0)) - .mustThrow(()=>st.get(1)) - .mustThrow(()=>st.get(0,~capi.SQLITE_INTEGER)) - .assert(3 === st.get(0,capi.SQLITE_INTEGER)) - .assert(3 === st.getInt(0)) - .assert('3' === st.get(0,capi.SQLITE_TEXT)) - .assert('3' === st.getString(0)) - .assert(3.0 === st.get(0,capi.SQLITE_FLOAT)) - .assert(3.0 === st.getFloat(0)) - .assert(3 === st.get({}).a) - .assert(3 === st.get([])[0]) - .assert(3 === st.getJSON(0)) - .assert(st.get(0,capi.SQLITE_BLOB) instanceof Uint8Array) - .assert(1===st.get(0,capi.SQLITE_BLOB).length) - .assert(st.getBlob(0) instanceof Uint8Array) - .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) - .assert(st._mayGet) - .assert(false===st.step()) - .assert(!st._mayGet) - ; - T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). - assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). - assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). - assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); - }finally{ - st.finalize(); - } - T.assert(!st.pointer) - .assert(0===this.db.openStatementCount()); - }) - - //////////////////////////////////////////////////////////////////////// - .t('sqlite3_js_...()', function(){ - const db = this.db; - if(1){ - const vfsList = capi.sqlite3_js_vfs_list(); - T.assert(vfsList.length>1); - T.assert('string'===typeof vfsList[0]); - //log("vfsList =",vfsList); - for(const v of vfsList){ - T.assert('string' === typeof v) - .assert(capi.sqlite3_vfs_find(v) > 0); - } - } - /** - Trivia: the magic db name ":memory:" does not actually use the - "memdb" VFS unless "memdb" is _explicitly_ provided as the VFS - name. Instead, it uses the default VFS with an in-memory btree. - Thus this.db's VFS may not be memdb even though it's an in-memory - db. - */ - const pVfsMem = capi.sqlite3_vfs_find('memdb'), - pVfsDflt = capi.sqlite3_vfs_find(0), - pVfsDb = capi.sqlite3_js_db_vfs(db.pointer); - T.assert(pVfsMem > 0) - .assert(pVfsDflt > 0) - .assert(pVfsDb > 0) - .assert(pVfsMem !== pVfsDflt - /* memdb lives on top of the default vfs */) - .assert(pVfsDb === pVfsDflt || pVfsdb === pVfsMem) - ; - /*const vMem = new capi.sqlite3_vfs(pVfsMem), - vDflt = new capi.sqlite3_vfs(pVfsDflt), - vDb = new capi.sqlite3_vfs(pVfsDb);*/ - const duv = capi.sqlite3_js_db_uses_vfs; - T.assert(pVfsDflt === duv(db.pointer, 0) - || pVfsMem === duv(db.pointer,0)) - .assert(!duv(db.pointer, "foo")) - ; - }/*sqlite3_js_...()*/) - - //////////////////////////////////////////////////////////////////// - .t('Table t', function(sqlite3){ - const db = this.db; - let list = []; - let rc = db.exec({ - sql:['CREATE TABLE t(a,b);', - // ^^^ using TEMP TABLE breaks the db export test - "INSERT INTO t(a,b) VALUES(1,2),(3,4),", - "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for - off-by-one bug in string-to-WASM conversion*/], - saveSql: list, - bind: [5,6] - }); - //debug("Exec'd SQL:", list); - T.assert(rc === db) - .assert(2 === list.length) - .assert('string'===typeof list[1]) - .assert(4===db.changes()); - if(wasm.bigIntEnabled){ - T.assert(4n===db.changes(false,true)); - } - let blob = db.selectValue("select b from t where a='blob'"); - T.assert(blob instanceof Uint8Array). - assert(0x68===blob[0] && 0x69===blob[1]); - blob = null; - let counter = 0, colNames = []; - list.length = 0; - db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ - rowMode: 'object', - resultRows: list, - columnNames: colNames, - callback: function(row,stmt){ - ++counter; - T.assert((row.a%2 && row.a<6) || 'blob'===row.a); - } - }); - T.assert(2 === colNames.length) - .assert('a' === colNames[0]) - .assert(4 === counter) - .assert(4 === list.length); - list.length = 0; - db.exec("SELECT a a, b b FROM t",{ - rowMode: 'array', - callback: function(row,stmt){ - ++counter; - T.assert(Array.isArray(row)) - .assert((0===row[1]%2 && row[1]<7) - || (row[1] instanceof Uint8Array)); - } - }); - T.assert(8 === counter); - T.assert(Number.MIN_SAFE_INTEGER === - db.selectValue("SELECT "+Number.MIN_SAFE_INTEGER)). - assert(Number.MAX_SAFE_INTEGER === - db.selectValue("SELECT "+Number.MAX_SAFE_INTEGER)); - if(wasm.bigIntEnabled && haveWasmCTests()){ - const mI = wasm.xCall('sqlite3_wasm_test_int64_max'); - const b = BigInt(Number.MAX_SAFE_INTEGER * 2); - T.assert(b === db.selectValue("SELECT "+b)). - assert(b === db.selectValue("SELECT ?", b)). - assert(mI == db.selectValue("SELECT $x", {$x:mI})); - }else{ - /* Curiously, the JS spec seems to be off by one with the definitions - of MIN/MAX_SAFE_INTEGER: - - https://github.com/emscripten-core/emscripten/issues/17391 */ - T.mustThrow(()=>db.selectValue("SELECT "+(Number.MAX_SAFE_INTEGER+1))). - mustThrow(()=>db.selectValue("SELECT "+(Number.MIN_SAFE_INTEGER-1))); - } - - let st = db.prepare("update t set b=:b where a='blob'"); - try { - const ndx = st.getParamIndex(':b'); - T.assert(1===ndx); - st.bindAsBlob(ndx, "ima blob").reset(true); - } finally { - st.finalize(); - } - - try { - db.prepare("/*empty SQL*/"); - toss("Must not be reached."); - }catch(e){ - T.assert(e instanceof sqlite3.SQLite3Error) - .assert(0==e.message.indexOf('Cannot prepare empty')); - } - }) - - //////////////////////////////////////////////////////////////////////// - .t('selectArray/Object()', function(sqlite3){ - const db = this.db; - let rc = db.selectArray('select a, b from t where a=?', 5); - T.assert(Array.isArray(rc)) - .assert(2===rc.length) - .assert(5===rc[0] && 6===rc[1]); - rc = db.selectArray('select a, b from t where b=-1'); - T.assert(undefined === rc); - rc = db.selectObject('select a A, b b from t where b=?', 6); - T.assert(rc && 'object'===typeof rc) - .assert(5===rc.A) - .assert(6===rc.b); - rc = db.selectArray('select a, b from t where b=-1'); - T.assert(undefined === rc); - }) - - //////////////////////////////////////////////////////////////////////// - .t('sqlite3_js_db_export()', function(){ - const db = this.db; - const xp = capi.sqlite3_js_db_export(db.pointer); - T.assert(xp instanceof Uint8Array) - .assert(xp.byteLength>0) - .assert(0 === xp.byteLength % 512); - }/*sqlite3_js_db_export()*/) - - //////////////////////////////////////////////////////////////////// - .t('Scalar UDFs', function(sqlite3){ - const db = this.db; - db.createFunction("foo",(pCx,a,b)=>a+b); - T.assert(7===db.selectValue("select foo(3,4)")). - assert(5===db.selectValue("select foo(3,?)",2)). - assert(5===db.selectValue("select foo(?,?2)",[1,4])). - assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); - db.createFunction("bar", { - arity: -1, - xFunc: (pCx,...args)=>{ - let rc = 0; - for(const v of args) rc += v; - return rc; - } - }).createFunction({ - name: "asis", - xFunc: (pCx,arg)=>arg - }); - T.assert(0===db.selectValue("select bar()")). - assert(1===db.selectValue("select bar(1)")). - assert(3===db.selectValue("select bar(1,2)")). - assert(-1===db.selectValue("select bar(1,2,-4)")). - assert('hi' === db.selectValue("select asis('hi')")). - assert('hi' === db.selectValue("select ?",'hi')). - assert(null === db.selectValue("select null")). - assert(null === db.selectValue("select asis(null)")). - assert(1 === db.selectValue("select ?",1)). - assert(2 === db.selectValue("select ?",[2])). - assert(3 === db.selectValue("select $a",{$a:3})). - assert(T.eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). - assert(T.eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))); - - let blobArg = new Uint8Array(2); - blobArg.set([0x68, 0x69], 0); - let blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - blobRc = db.selectValue("select asis(X'6869')"); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - - blobArg = new Int8Array(2); - blobArg.set([0x68, 0x69]); - //debug("blobArg=",blobArg); - blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length); - //debug("blobRc=",blobRc); - T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); - }) - - //////////////////////////////////////////////////////////////////// - .t({ - name: 'Aggregate UDFs', - test: function(sqlite3){ - const db = this.db; - const sjac = capi.sqlite3_js_aggregate_context; - db.createFunction({ - name: 'summer', - xStep: (pCtx, n)=>{ - const ac = sjac(pCtx, 4); - wasm.setMemValue(ac, wasm.getMemValue(ac,'i32') + Number(n), 'i32'); - }, - xFinal: (pCtx)=>{ - const ac = sjac(pCtx, 0); - return ac ? wasm.getMemValue(ac,'i32') : 0; - } - }); - let v = db.selectValue([ - "with cte(v) as (", - "select 3 union all select 5 union all select 7", - ") select summer(v), summer(v+1) from cte" - /* ------------------^^^^^^^^^^^ ensures that we're handling - sqlite3_aggregate_context() properly. */ - ]); - T.assert(15===v); - T.mustThrowMatching(()=>db.selectValue("select summer(1,2)"), - /wrong number of arguments/); - - db.createFunction({ - name: 'summerN', - arity: -1, - xStep: (pCtx, ...args)=>{ - const ac = sjac(pCtx, 4); - let sum = wasm.getMemValue(ac, 'i32'); - for(const v of args) sum += Number(v); - wasm.setMemValue(ac, sum, 'i32'); - }, - xFinal: (pCtx)=>{ - const ac = sjac(pCtx, 0); - capi.sqlite3_result_int( pCtx, ac ? wasm.getMemValue(ac,'i32') : 0 ); - // xFinal() may either return its value directly or call - // sqlite3_result_xyz() and return undefined. Both are - // functionally equivalent. - } - }); - T.assert(18===db.selectValue('select summerN(1,8,9), summerN(2,3,4)')); - T.mustThrowMatching(()=>{ - db.createFunction('nope',{ - xFunc: ()=>{}, xStep: ()=>{} - }); - }, /scalar or aggregate\?/); - T.mustThrowMatching(()=>{ - db.createFunction('nope',{xStep: ()=>{}}); - }, /Missing xFinal/); - T.mustThrowMatching(()=>{ - db.createFunction('nope',{xFinal: ()=>{}}); - }, /Missing xStep/); - T.mustThrowMatching(()=>{ - db.createFunction('nope',{}); - }, /Missing function-type properties/); - T.mustThrowMatching(()=>{ - db.createFunction('nope',{xFunc:()=>{}, xDestroy:'nope'}); - }, /xDestroy property must be a function/); - T.mustThrowMatching(()=>{ - db.createFunction('nope',{xFunc:()=>{}, pApp:'nope'}); - }, /Invalid value for pApp/); - } - }/*aggregate UDFs*/) - - //////////////////////////////////////////////////////////////////////// - .t({ - name: 'Aggregate UDFs (64-bit)', - predicate: ()=>wasm.bigIntEnabled, - test: function(sqlite3){ - const db = this.db; - const sjac = capi.sqlite3_js_aggregate_context; - db.createFunction({ - name: 'summer64', - xStep: (pCtx, n)=>{ - const ac = sjac(pCtx, 8); - wasm.setMemValue(ac, wasm.getMemValue(ac,'i64') + BigInt(n), 'i64'); - }, - xFinal: (pCtx)=>{ - const ac = sjac(pCtx, 0); - return ac ? wasm.getMemValue(ac,'i64') : 0n; - } - }); - let v = db.selectValue([ - "with cte(v) as (", - "select 9007199254740991 union all select 1 union all select 2", - ") select summer64(v), summer64(v+1) from cte" - ]); - T.assert(9007199254740994n===v); - } - }/*aggregate UDFs*/) - - //////////////////////////////////////////////////////////////////// - .t({ - name: 'Window UDFs', - test: function(){ - /* Example window function, table, and results taken from: - https://sqlite.org/windowfunctions.html#udfwinfunc */ - const db = this.db; - const sjac = (cx,n=4)=>capi.sqlite3_js_aggregate_context(cx,n); - const xValueFinal = (pCtx)=>{ - const ac = sjac(pCtx, 0); - return ac ? wasm.getMemValue(ac,'i32') : 0; - }; - const xStepInverse = (pCtx, n)=>{ - const ac = sjac(pCtx); - wasm.setMemValue(ac, wasm.getMemValue(ac,'i32') + Number(n), 'i32'); - }; - db.createFunction({ - name: 'winsumint', - xStep: (pCtx, n)=>xStepInverse(pCtx, n), - xInverse: (pCtx, n)=>xStepInverse(pCtx, -n), - xFinal: xValueFinal, - xValue: xValueFinal - }); - db.exec([ - "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", - "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" - ]); - let rc = db.exec({ - returnValue: 'resultRows', - sql:[ - "SELECT x, winsumint(y) OVER (", - "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", - ") AS sum_y ", - "FROM twin ORDER BY x;" - ] - }); - T.assert(Array.isArray(rc)) - .assert(5 === rc.length); - let count = 0; - for(const row of rc){ - switch(++count){ - case 1: T.assert('a'===row[0] && 9===row[1]); break; - case 2: T.assert('b'===row[0] && 12===row[1]); break; - case 3: T.assert('c'===row[0] && 16===row[1]); break; - case 4: T.assert('d'===row[0] && 12===row[1]); break; - case 5: T.assert('e'===row[0] && 9===row[1]); break; - default: toss("Too many rows to window function."); - } - } - const resultRows = []; - rc = db.exec({ - resultRows, - returnValue: 'resultRows', - sql:[ - "SELECT x, winsumint(y) OVER (", - "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING", - ") AS sum_y ", - "FROM twin ORDER BY x;" - ] - }); - T.assert(rc === resultRows) - .assert(5 === rc.length); - - rc = db.exec({ - returnValue: 'saveSql', - sql: "select 1; select 2; -- empty\n; select 3" - }); - T.assert(Array.isArray(rc)) - .assert(3===rc.length) - .assert('select 1;' === rc[0]) - .assert('select 2;' === rc[1]) - .assert('-- empty\n; select 3' === rc[2] - /* Strange but true. */); - - T.mustThrowMatching(()=>{ - db.exec({sql:'', returnValue: 'nope'}); - }, /^Invalid returnValue/); - - db.exec("DROP TABLE twin"); - } - }/*window UDFs*/) - - //////////////////////////////////////////////////////////////////// - .t("ATTACH", function(){ - const db = this.db; - const resultRows = []; - db.exec({ - sql:new TextEncoder('utf-8').encode([ - // ^^^ testing string-vs-typedarray handling in exec() - "attach 'session' as foo;", - "create table foo.bar(a);", - "insert into foo.bar(a) values(1),(2),(3);", - "select a from foo.bar order by a;" - ].join('')), - rowMode: 0, - resultRows - }); - T.assert(3===resultRows.length) - .assert(2===resultRows[1]); - T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); - let colCount = 0, rowCount = 0; - const execCallback = function(pVoid, nCols, aVals, aNames){ - colCount = nCols; - ++rowCount; - T.assert(2===aVals.length) - .assert(2===aNames.length) - .assert(+(aVals[1]) === 2 * +(aVals[0])); - }; - let rc = capi.sqlite3_exec( - db.pointer, "select a, a*2 from foo.bar", execCallback, - 0, 0 - ); - T.assert(0===rc).assert(3===rowCount).assert(2===colCount); - rc = capi.sqlite3_exec( - db.pointer, "select a from foo.bar", ()=>{ - tossQuietly("Testing throwing from exec() callback."); - }, 0, 0 - ); - T.assert(capi.SQLITE_ABORT === rc); - db.exec("detach foo"); - T.mustThrow(()=>db.exec("select * from foo.bar")); - }) - - //////////////////////////////////////////////////////////////////// - .t({ - name: 'C-side WASM tests (if compiled in)', - predicate: haveWasmCTests, - test: function(){ - const w = wasm, db = this.db; - const stack = w.scopedAllocPush(); - let ptrInt; - const origValue = 512; - const ptrValType = 'i32'; - try{ - ptrInt = w.scopedAlloc(4); - w.setMemValue(ptrInt,origValue, ptrValType); - const cf = w.xGet('sqlite3_wasm_test_intptr'); - const oldPtrInt = ptrInt; - //log('ptrInt',ptrInt); - //log('getMemValue(ptrInt)',w.getMemValue(ptrInt)); - T.assert(origValue === w.getMemValue(ptrInt, ptrValType)); - const rc = cf(ptrInt); - //log('cf(ptrInt)',rc); - //log('ptrInt',ptrInt); - //log('getMemValue(ptrInt)',w.getMemValue(ptrInt,ptrValType)); - T.assert(2*origValue === rc). - assert(rc === w.getMemValue(ptrInt,ptrValType)). - assert(oldPtrInt === ptrInt); - const pi64 = w.scopedAlloc(8)/*ptr to 64-bit integer*/; - const o64 = 0x010203040506/*>32-bit integer*/; - const ptrType64 = 'i64'; - if(w.bigIntEnabled){ - w.setMemValue(pi64, o64, ptrType64); - //log("pi64 =",pi64, "o64 = 0x",o64.toString(16), o64); - const v64 = ()=>w.getMemValue(pi64,ptrType64) - //log("getMemValue(pi64)",v64()); - T.assert(v64() == o64); - //T.assert(o64 === w.getMemValue(pi64, ptrType64)); - const cf64w = w.xGet('sqlite3_wasm_test_int64ptr'); - cf64w(pi64); - //log("getMemValue(pi64)",v64()); - T.assert(v64() == BigInt(2 * o64)); - cf64w(pi64); - T.assert(v64() == BigInt(4 * o64)); - - const biTimes2 = w.xGet('sqlite3_wasm_test_int64_times2'); - T.assert(BigInt(2 * o64) === - biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError - in the call :/ */)); - - const pMin = w.scopedAlloc(16); - const pMax = pMin + 8; - const g64 = (p)=>w.getMemValue(p,ptrType64); - w.setMemValue(pMin, 0, ptrType64); - w.setMemValue(pMax, 0, ptrType64); - const minMaxI64 = [ - w.xCall('sqlite3_wasm_test_int64_min'), - w.xCall('sqlite3_wasm_test_int64_max') - ]; - T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). - assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); - //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); - w.xCall('sqlite3_wasm_test_int64_minmax', pMin, pMax); - T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). - assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); - //log("pMin",g64(pMin), "pMax",g64(pMax)); - w.setMemValue(pMin, minMaxI64[0], ptrType64); - T.assert(g64(pMin) === minMaxI64[0]). - assert(minMaxI64[0] === db.selectValue("select ?",g64(pMin))). - assert(minMaxI64[1] === db.selectValue("select ?",g64(pMax))); - const rxRange = /too big/; - T.mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[0] - BigInt(1))}, - rxRange). - mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[1] + BigInt(1))}, - (e)=>rxRange.test(e.message)); - }else{ - log("No BigInt support. Skipping related tests."); - log("\"The problem\" here is that we can manipulate, at the byte level,", - "heap memory to set 64-bit values, but we can't get those values", - "back into JS because of the lack of 64-bit integer support."); - } - }finally{ - const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); - //log("x=",x,"y=",y,"z=",z); // just looking at the alignment - w.scopedAllocPop(stack); - } - } - }/* jaccwabyt-specific tests */) - - .t('Close db', function(){ - T.assert(this.db).assert(Number.isInteger(this.db.pointer)); - wasm.exports.sqlite3_wasm_db_reset(this.db.pointer); - this.db.close(); - T.assert(!this.db.pointer); - }) - ;/* end of oo1 checks */ - - //////////////////////////////////////////////////////////////////////// - T.g('kvvfs') - .t('kvvfs sanity checks', function(sqlite3){ - if(isWorker()){ - T.assert( - !capi.sqlite3_vfs_find('kvvfs'), - "Expecting kvvfs to be unregistered." - ); - log("kvvfs is (correctly) unavailable in a Worker."); - return; - } - const filename = 'session'; - const pVfs = capi.sqlite3_vfs_find('kvvfs'); - T.assert(pVfs); - const JDb = sqlite3.oo1.JsStorageDb; - const unlink = ()=>JDb.clearStorage(filename); - unlink(); - let db = new JDb(filename); - try { - db.exec([ - 'create table kvvfs(a);', - 'insert into kvvfs(a) values(1),(2),(3)' - ]); - T.assert(3 === db.selectValue('select count(*) from kvvfs')); - db.close(); - db = new JDb(filename); - db.exec('insert into kvvfs(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from kvvfs')); - }finally{ - db.close(); - unlink(); - } - }/*kvvfs sanity checks*/) - ;/* end kvvfs tests */ - - //////////////////////////////////////////////////////////////////////// - T.g('OPFS (Worker thread only and only in supported browsers)', - (sqlite3)=>{return !!sqlite3.opfs}) - .t({ - name: 'OPFS sanity checks', - test: async function(sqlite3){ - const opfs = sqlite3.opfs; - const filename = 'sqlite3-tester1.db'; - const pVfs = capi.sqlite3_vfs_find('opfs'); - T.assert(pVfs); - const unlink = (fn=filename)=>wasm.sqlite3_wasm_vfs_unlink(pVfs,fn); - unlink(); - let db = new opfs.OpfsDb(filename); - try { - db.exec([ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]); - T.assert(3 === db.selectValue('select count(*) from p')); - db.close(); - db = new opfs.OpfsDb(filename); - db.exec('insert into p(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from p')); - }finally{ - db.close(); - unlink(); - } - - if(1){ - // Sanity-test sqlite3_wasm_vfs_create_file()... - const fSize = 1379; - let sh; - try{ - T.assert(!(await opfs.entryExists(filename))); - let rc = wasm.sqlite3_wasm_vfs_create_file( - pVfs, filename, null, fSize - ); - T.assert(0===rc) - .assert(await opfs.entryExists(filename)); - const fh = await opfs.rootDirectory.getFileHandle(filename); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - }finally{ - if(sh) await sh.close(); - unlink(); - } - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); - } - }/*OPFS sanity checks*/) - ;/* end OPFS tests */ - - //////////////////////////////////////////////////////////////////////// - log("Loading and initializing sqlite3 WASM module..."); - if(!isUIThread()){ - /* - If sqlite3.js is in a directory other than this script, in order - to get sqlite3.js to resolve sqlite3.wasm properly, we have to - explicitly tell it where sqlite3.js is being loaded from. We do - that by passing the `sqlite3.dir=theDirName` URL argument to - _this_ script. That URL argument will be seen by the JS/WASM - loader and it will adjust the sqlite3.wasm path accordingly. If - sqlite3.js/.wasm are in the same directory as this script then - that's not needed. - - URL arguments passed as part of the filename via importScripts() - are simply lost, and such scripts see the self.location of - _this_ script. - */ - let sqlite3Js = 'sqlite3.js'; - const urlParams = new URL(self.location.href).searchParams; - if(urlParams.has('sqlite3.dir')){ - sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; - } - importScripts(sqlite3Js); - } - self.sqlite3InitModule({ - print: log, - printErr: error - }).then(function(sqlite3){ - //console.log('sqlite3 =',sqlite3); - log("Done initializing WASM/JS bits. Running tests..."); - capi = sqlite3.capi; - wasm = sqlite3.wasm; - log("sqlite3 version:",capi.sqlite3_libversion(), - capi.sqlite3_sourceid()); - if(wasm.bigIntEnabled){ - log("BigInt/int64 support is enabled."); - }else{ - logClass('warning',"BigInt/int64 support is disabled."); - } - if(haveWasmCTests()){ - log("sqlite3_wasm_test_...() APIs are available."); - }else{ - logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); - } - TestUtil.runTests(sqlite3); - }); -})(); ADDED ext/wasm/tests/opfs/concurrency/index.html Index: ext/wasm/tests/opfs/concurrency/index.html ================================================================== --- /dev/null +++ ext/wasm/tests/opfs/concurrency/index.html @@ -0,0 +1,50 @@ + + + + + + + + sqlite3 OPFS Worker concurrency tester + + + +

    +

    + OPFS concurrency tester using multiple independent Workers. + Disclaimer: concurrency in OPFS is currently a pain point! +

    +

    + URL flags: pass a number of workers using + the workers=N URL flag. Set the time between each + workload with interval=N (milliseconds). Set the + number of worker iterations with iterations=N. + Enable OPFS VFS verbosity with verbose=1-3 (output + goes to the dev console). Enable/disable "unlock ASAP" mode + (higher concurrency, lower speed) + with unlock-asap=0-1. +

    +

    Achtung: if it does not start to do anything within a couple of + seconds, check the dev console: Chrome sometimes fails to load + the wasm module due to "cannot allocate WasmMemory." Closing and + re-opening the tab usually resolves it, but sometimes restarting + the browser is required. +

    +
    + + +
    +
    + + + + ADDED ext/wasm/tests/opfs/concurrency/test.js Index: ext/wasm/tests/opfs/concurrency/test.js ================================================================== --- /dev/null +++ ext/wasm/tests/opfs/concurrency/test.js @@ -0,0 +1,136 @@ +(async function(self){ + + const logCss = (function(){ + const mapToString = (v)=>{ + switch(typeof v){ + case 'number': case 'string': case 'boolean': + case 'undefined': case 'bigint': + return ''+v; + default: break; + } + if(null===v) return 'null'; + if(v instanceof Error){ + v = { + message: v.message, + stack: v.stack, + errorClass: v.name + }; + } + return JSON.stringify(v,undefined,2); + }; + const normalizeArgs = (args)=>args.map(mapToString); + const logTarget = document.querySelector('#test-output'); + const logCss = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass){ + for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ + ln.classList.add(c); + } + } + ln.append(document.createTextNode(normalizeArgs(args).join(' '))); + logTarget.append(ln); + }; + const cbReverse = document.querySelector('#cb-log-reverse'); + const cbReverseKey = 'tester1:cb-log-reverse'; + const cbReverseIt = ()=>{ + logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + }; + cbReverse.addEventListener('change', cbReverseIt, true); + if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + } + cbReverseIt(); + return logCss; + })(); + const stdout = (...args)=>logCss('',...args); + const stderr = (...args)=>logCss('error',...args); + + const wait = async (ms)=>{ + return new Promise((resolve)=>setTimeout(resolve,ms)); + }; + + const urlArgsJs = new URL(document.currentScript.src).searchParams; + const urlArgsHtml = new URL(self.location.href).searchParams; + const options = Object.create(null); + options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); + options.workerCount = ( + urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 + ) || 4; + options.opfsVerbose = ( + urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 + ) || 1; + options.interval = ( + urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 1000 + ) || 1000; + options.iterations = ( + urlArgsHtml.has('iterations') ? +urlArgsHtml.get('iterations') : 10 + ) || 10; + options.unlockAsap = ( + urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 + ) || 0; + options.noUnlink = !!urlArgsHtml.has('no-unlink'); + const workers = []; + workers.post = (type,...args)=>{ + for(const w of workers) w.postMessage({type, payload:args}); + }; + workers.counts = {loaded: 0, passed: 0, failed: 0}; + const checkFinished = function(){ + if(workers.counts.passed + workers.counts.failed !== workers.length){ + return; + } + if(workers.counts.failed>0){ + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + }else{ + logCss('tests-pass',"All",workers.length,"workers finished."); + } + }; + workers.onmessage = function(msg){ + msg = msg.data; + const prefix = 'Worker #'+msg.worker+':'; + switch(msg.type){ + case 'loaded': + stdout(prefix,"loaded"); + if(++workers.counts.loaded === workers.length){ + stdout("All",workers.length,"workers loaded. Telling them to run..."); + workers.post('run'); + } + break; + case 'stdout': stdout(prefix,...msg.payload); break; + case 'stderr': stderr(prefix,...msg.payload); break; + case 'error': stderr(prefix,"ERROR:",...msg.payload); break; + case 'finished': + ++workers.counts.passed; + logCss('tests-pass',prefix,...msg.payload); + checkFinished(); + break; + case 'failed': + ++workers.counts.failed; + logCss('tests-fail',prefix,"FAILED:",...msg.payload); + checkFinished(); + break; + default: logCss('error',"Unhandled message type:",msg); break; + } + }; + + stdout("Launching",options.workerCount,"workers. Options:",options); + workers.uri = ( + 'worker.js?' + + 'sqlite3.dir='+options.sqlite3Dir + + '&interval='+options.interval + + '&iterations='+options.iterations + + '&opfs-verbose='+options.opfsVerbose + + '&opfs-unlock-asap='+options.unlockAsap + ); + for(let i = 0; i < options.workerCount; ++i){ + stdout("Launching worker..."); + workers.push(new Worker( + workers.uri+'&workerId='+(i+1)+( + (i || options.noUnlink) ? '' : '&unlink-db' + ) + )); + } + // Have to delay onmessage assignment until after the loop + // to avoid that early workers get an undue head start. + workers.forEach((w)=>w.onmessage = workers.onmessage); +})(self); ADDED ext/wasm/tests/opfs/concurrency/worker.js Index: ext/wasm/tests/opfs/concurrency/worker.js ================================================================== --- /dev/null +++ ext/wasm/tests/opfs/concurrency/worker.js @@ -0,0 +1,113 @@ +importScripts( + (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' +); +self.sqlite3InitModule().then(async function(sqlite3){ + const urlArgs = new URL(self.location.href).searchParams; + const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + }; + const wPost = (type,...payload)=>{ + postMessage({type, worker: options.workerName, payload}); + }; + const stdout = (...args)=>wPost('stdout',...args); + const stderr = (...args)=>wPost('stderr',...args); + if(!sqlite3.opfs){ + stderr("OPFS support not detected. Aborting."); + return; + } + + const wait = async (ms)=>{ + return new Promise((resolve)=>setTimeout(resolve,ms)); + }; + + const dbName = 'concurrency-tester.db'; + if(urlArgs.has('unlink-db')){ + await sqlite3.opfs.unlink(dbName); + stdout("Unlinked",dbName); + } + wPost('loaded'); + let db; + const interval = Object.assign(Object.create(null),{ + delay: urlArgs.has('interval') ? (+urlArgs.get('interval') || 750) : 750, + handle: undefined, + count: 0 + }); + const finish = ()=>{ + if(db){ + if(!db.pointer) return; + db.close(); + } + if(interval.error){ + wPost('failed',"Ending work after interval #"+interval.count, + "due to error:",interval.error); + }else{ + wPost('finished',"Ending work after",interval.count,"intervals."); + } + }; + const run = async function(){ + db = new sqlite3.oo1.OpfsDb({ + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, + flags: 'c' + }); + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); + + const maxIterations = + urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; + stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); + const doWork = async ()=>{ + const tm = new Date().getTime(); + ++interval.count; + const prefix = "v(#"+interval.count+")"; + stdout("Setting",prefix,"=",tm); + try{ + db.exec({ + sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", + bind: [options.workerName, new Date().getTime()] + }); + //stdout("Set",prefix); + }catch(e){ + interval.error = e; + } + }; + if(1){/*use setInterval()*/ + setTimeout(async function timer(){ + await doWork(); + if(interval.error || maxIterations === interval.count){ + finish(); + }else{ + setTimeout(timer, interval.delay); + } + }, interval.delay); + }else{ + /*This approach provides no concurrency whatsoever: each worker + is run to completion before any others can work.*/ + let i; + for(i = 0; i < maxIterations; ++i){ + await doWork(); + if(interval.error) break; + await wait(interval.ms); + } + finish(); + } + }/*run()*/; + + self.onmessage = function({data}){ + switch(data.type){ + case 'run': run().catch((e)=>{ + if(!interval.error) interval.error = e; + finish(); + }); + break; + default: + stderr("Unhandled message type '"+data.type+"'."); + break; + } + }; +}); Index: ext/wasm/wasmfs.make ================================================================== --- ext/wasm/wasmfs.make +++ ext/wasm/wasmfs.make @@ -13,101 +13,120 @@ # is loaded from any directory other than the one in which the # containing HTML lives. Similarly, they cannot be loaded from a # Worker to an Emscripten quirk regarding loading nested Workers. dir.wasmfs := $(dir.wasm) sqlite3-wasmfs.js := $(dir.wasmfs)/sqlite3-wasmfs.js +sqlite3-wasmfs.mjs := $(dir.wasmfs)/sqlite3-wasmfs.mjs sqlite3-wasmfs.wasm := $(dir.wasmfs)/sqlite3-wasmfs.wasm CLEAN_FILES += $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.wasm) \ - $(subst .js,.worker.js,$(sqlite3-wasmfs.js)) + $(subst .js,.worker.js,$(sqlite3-wasmfs.js)) \ + $(sqlite3-wasmfs.mjs) \ + $(subst .mjs,.worker.mjs,$(sqlite3-wasmfs.mjs)) ######################################################################## # emcc flags for .c/.o. -sqlite3-wasmfs.cflags := -sqlite3-wasmfs.cflags += -std=c99 -fPIC -sqlite3-wasmfs.cflags += -pthread -sqlite3-wasmfs.cflags += $(cflags.common) -sqlite3-wasmfs.cflags += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS +cflags.sqlite3-wasmfs := +cflags.sqlite3-wasmfs += -std=c99 -fPIC +cflags.sqlite3-wasmfs += -pthread +cflags.sqlite3-wasmfs += $(cflags.speedtest1) +cflags.sqlite3-wasmfs += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS ######################################################################## # emcc flags specific to building the final .js/.wasm file... -sqlite3-wasmfs.jsflags := -fPIC -sqlite3-wasmfs.jsflags += --no-entry -sqlite3-wasmfs.jsflags += --minify 0 -sqlite3-wasmfs.jsflags += -sMODULARIZE -sqlite3-wasmfs.jsflags += -sSTRICT_JS -sqlite3-wasmfs.jsflags += -sDYNAMIC_EXECUTION=0 -sqlite3-wasmfs.jsflags += -sNO_POLYFILL -sqlite3-wasmfs.jsflags += -sEXPORTED_FUNCTIONS=@$(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) -sqlite3-wasmfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack - # wasmMemory ==> for -sIMPORTED_MEMORY - # allocateUTF8OnStack ==> wasmfs internals -sqlite3-wasmfs.jsflags += -sUSE_CLOSURE_COMPILER=0 -sqlite3-wasmfs.jsflags += -sIMPORTED_MEMORY -#sqlite3-wasmfs.jsflags += -sINITIAL_MEMORY=13107200 -#sqlite3-wasmfs.jsflags += -sTOTAL_STACK=4194304 -sqlite3-wasmfs.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) -sqlite3-wasmfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. -#sqlite3-wasmfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API -# Perhaps the wasmfs build doesn't? -#sqlite3-wasmfs.jsflags += -sABORTING_MALLOC -sqlite3-wasmfs.jsflags += -sALLOW_TABLE_GROWTH -sqlite3-wasmfs.jsflags += -Wno-limited-postlink-optimizations +emcc.flags.sqlite3-wasmfs := -fPIC +emcc.flags.sqlite3-wasmfs += --no-entry +emcc.flags.sqlite3-wasmfs += --minify 0 +emcc.flags.sqlite3-wasmfs += -sMODULARIZE +emcc.flags.sqlite3-wasmfs += -sEXPORT_NAME=$(sqlite3.js.init-func) +emcc.flags.sqlite3-wasmfs += -sSTRICT_JS +emcc.flags.sqlite3-wasmfs += -sDYNAMIC_EXECUTION=0 +emcc.flags.sqlite3-wasmfs += -sNO_POLYFILL +emcc.flags.sqlite3-wasmfs += -sWASM_BIGINT=$(emcc.WASM_BIGINT) +emcc.flags.sqlite3-wasmfs += -sEXPORTED_FUNCTIONS=@$(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +emcc.flags.sqlite3-wasmfs += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack + # wasmMemory ==> for -sIMPORTED_MEMORY + # allocateUTF8OnStack ==> wasmfs internals +emcc.flags.sqlite3-wasmfs += -sUSE_CLOSURE_COMPILER=0 +emcc.flags.sqlite3-wasmfs += -Wno-limited-postlink-optimizations # ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. -sqlite3-wasmfs.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 -sqlite3-wasmfs.jsflags += -sLLD_REPORT_UNDEFINED -#sqlite3-wasmfs.jsflags += --import-undefined -sqlite3-wasmfs.jsflags += -sMEMORY64=0 -sqlite3-wasmfs.jsflags += -sINITIAL_MEMORY=128450560 +emcc.flags.sqlite3-wasmfs += -sALLOW_TABLE_GROWTH +emcc.flags.sqlite3-wasmfs += -sSTACK_SIZE=512KB +emcc.flags.sqlite3-wasmfs += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. +emcc.flags.sqlite3-wasmfs += -sMEMORY64=0 +emcc.flags.sqlite3-wasmfs += -sIMPORTED_MEMORY +emcc.flags.sqlite3-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) # ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js -sqlite3-wasmfs.fsflags := -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2 -sENVIRONMENT=web,worker -# -sPTHREAD_POOL_SIZE values of 2 or higher trigger that bug. -sqlite3-wasmfs.jsflags += $(sqlite3-wasmfs.fsflags) -#sqlite3-wasmfs.jsflags += -sALLOW_MEMORY_GROWTH +sqlite3-wasmfs.fsflags := -pthread -sWASMFS \ + -sPTHREAD_POOL_SIZE=2 -sENVIRONMENT=web,worker \ + -sERROR_ON_UNDEFINED_SYMBOLS=0 -sLLD_REPORT_UNDEFINED +# ^^^^^ why undefined symbols are necessary for the wasmfs build is anyone's guess. +emcc.flags.sqlite3-wasmfs += $(sqlite3-wasmfs.fsflags) +#emcc.flags.sqlite3-wasmfs += -sALLOW_MEMORY_GROWTH #^^^ using ALLOW_MEMORY_GROWTH produces a warning from emcc: # USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, # see https://github.com/WebAssembly/design/issues/1271 [-Wpthreads-mem-growth] -sqlite3-wasmfs.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) -$(eval $(call call-make-pre-js,sqlite3-wasmfs)) -sqlite3-wasmfs.jsflags += $(pre-post-common.flags) $(pre-post-sqlite3-wasmfs.flags) -$(sqlite3-wasmfs.js): $(sqlite3-wasm.c) \ - $(EXPORTED_FUNCTIONS.api) $(MAKEFILE) $(MAKEFILE.wasmfs) \ - $(pre-post-sqlite3-wasmfs.deps) +# And, indeed, it runs slowly if memory is permitted to grow. +emcc.flags.sqlite3-wasmfs.vanilla := +emcc.flags.sqlite3-wasmfs.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META +$(eval $(call call-make-pre-js,sqlite3-wasmfs,vanilla)) +$(eval $(call call-make-pre-js,sqlite3-wasmfs,esm)) +Xemcc.flags.sqlite3-wasmfs.vanilla += \ + $(pre-post-common.flags.vanilla) \ + $(pre-post-sqlite3-wasmfs.flags.vanilla) +Xemcc.flags.sqlite3-wasmfs.esm += \ + $(pre-post-common.flags.esm) \ + $(pre-post-sqlite3-wasmfs.flags.esm) +$(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(sqlite3-wasm.c) \ + $(EXPORTED_FUNCTIONS.api) $(MAKEFILE) $(MAKEFILE.wasmfs) +$(sqlite3-wasmfs.js): $(pre-post-sqlite3-wasmfs.deps.vanilla) +$(sqlite3-wasmfs.mjs): $(pre-post-sqlite3-wasmfs.deps.esm) +# SQLITE3-WASMFS.xJS.RECIPE is the wasmfs-specific counterpart +# of SQLITE3.xJS.RECIPE from the main makefile. +define SQLITE3-WASMFS.xJS.RECIPE @echo "Building $@ ..." $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ - $(sqlite3-wasmfs.cflags) $(sqlite3-wasmfs.jsflags) \ + $(cflags.sqlite3-wasmfs) \ + $(emcc.flags.sqlite3-wasmfs) $(emcc.flags.sqlite3-wasmfs.$(1)) \ + $(pre-post-sqlite3-wasmfs.flags.$(1)) \ $(sqlite3-wasm.c) + @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(1)) chmod -x $(sqlite3-wasmfs.wasm) $(maybe-wasm-strip) $(sqlite3-wasmfs.wasm) - @ls -la $@ $(sqlite3-wasmfs.wasm) + @ls -la $(sqlite3-wasmfs.wasm) sqlite3-wasmfs*js +endef +$(sqlite3-wasmfs.js): + $(call SQLITE3-WASMFS.xJS.RECIPE,vanilla) +$(sqlite3-wasmfs.mjs): $(sqlite3-wasmfs.js) + $(call SQLITE3-WASMFS.xJS.RECIPE,esm) $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) -wasmfs: $(sqlite3-wasmfs.js) -all: wasmfs +wasmfs: $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs) +#all: wasmfs ######################################################################## # speedtest1 for wasmfs. speedtest1-wasmfs.js := $(dir.wasmfs)/speedtest1-wasmfs.js speedtest1-wasmfs.wasm := $(subst .js,.wasm,$(speedtest1-wasmfs.js)) -speedtest1-wasmfs.eflags := $(sqlite3-wasmfs.fsflags) -speedtest1-wasmfs.eflags += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS -speedtest1-wasmfs.eflags += -sALLOW_MEMORY_GROWTH=0 -speedtest1-wasmfs.eflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) -$(eval $(call call-make-pre-js,speedtest1-wasmfs)) +emcc.flags.speedtest1-wasmfs := $(sqlite3-wasmfs.fsflags) +emcc.flags.speedtest1-wasmfs += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS +emcc.flags.speedtest1-wasmfs += -sALLOW_MEMORY_GROWTH=0 +emcc.flags.speedtest1-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) +#$(eval $(call call-make-pre-js,speedtest1-wasmfs,vanilla)) $(speedtest1-wasmfs.js): $(speedtest1.cses) $(sqlite3-wasmfs.js) \ $(MAKEFILE) $(MAKEFILE.wasmfs) \ - $(pre-post-speedtest1-wasmfs.deps) \ + $(pre-post-sqlite3-wasmfs.deps) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(speedtest1-wasmfs.eflags) $(speedtest1-common.eflags) \ - $(pre-post-speedtest1-wasmfs.flags) \ - $(speedtest1.cflags) \ - $(sqlite3-wasmfs.cflags) \ + $(emcc.speedtest1.common) $(emcc.flags.speedtest1-wasmfs) \ + $(pre-post-sqlite3-wasmfs.flags.vanilla) \ + $(cflags.sqlite3-wasmfs) \ -o $@ $(speedtest1.cses) -lm $(maybe-wasm-strip) $(speedtest1-wasmfs.wasm) ls -la $@ $(speedtest1-wasmfs.wasm) -speedtest1: $(speedtest1-wasmfs.js) +#speedtest1: $(speedtest1-wasmfs.js) +wasmfs: $(speedtest1-wasmfs.js) CLEAN_FILES += $(speedtest1-wasmfs.js) $(speedtest1-wasmfs.wasm) \ $(subst .js,.worker.js,$(speedtest1-wasmfs.js)) # end speedtest1.js ######################################################################## Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -361,10 +361,11 @@ # Extensions to be statically loaded. # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/basexx.c \ $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/cksumvfs.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/csv.c \ $(TOP)/ext/misc/decimal.c \ @@ -755,10 +756,12 @@ # Source files that go into making shell.c SHELL_SRC = \ $(TOP)/src/shell.c.in \ $(TOP)/ext/misc/appendvfs.c \ $(TOP)/ext/misc/completion.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ $(TOP)/ext/misc/decimal.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -739,17 +739,18 @@ static void renameTokenCheckAll(Parse *pParse, const void *pPtr){ assert( pParse==pParse->db->pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( pParse->nErr==0 ){ const RenameToken *p; - u8 i = 0; + u32 i = 1; for(p=pParse->pRename; p; p=p->pNext){ if( p->p ){ assert( p->p!=pPtr ); - i += *(u8*)(p->p); + i += *(u8*)(p->p) | 1; } } + assert( i>0 ); } } #else # define renameTokenCheckAll(x,y) #endif Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -1898,66 +1898,71 @@ ** and initialize fields of the MemPage structure accordingly. ** ** Only the following combinations are supported. Anything different ** indicates a corrupt database files: ** -** PTF_ZERODATA -** PTF_ZERODATA | PTF_LEAF -** PTF_LEAFDATA | PTF_INTKEY -** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +** PTF_ZERODATA (0x02, 2) +** PTF_LEAFDATA | PTF_INTKEY (0x05, 5) +** PTF_ZERODATA | PTF_LEAF (0x0a, 10) +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF (0x0d, 13) */ static int decodeFlags(MemPage *pPage, int flagByte){ BtShared *pBt; /* A copy of pPage->pBt */ assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); - flagByte &= ~PTF_LEAF; - pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; - if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an - ** interior table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a - ** leaf table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); - pPage->intKey = 1; - if( pPage->leaf ){ + pPage->max1bytePayload = pBt->max1bytePayload; + if( flagByte>=(PTF_ZERODATA | PTF_LEAF) ){ + pPage->childPtrSize = 0; + pPage->leaf = 1; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF) ){ pPage->intKeyLeaf = 1; pPage->xCellSize = cellSizePtrTableLeaf; pPage->xParseCell = btreeParseCellPtr; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); + } + }else{ + pPage->childPtrSize = 4; + pPage->leaf = 0; + if( flagByte==(PTF_ZERODATA) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKeyLeaf = 0; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; - } - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an - ** interior index b-tree page. */ - assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a - ** leaf index b-tree page. */ - assert( (PTF_ZERODATA|PTF_LEAF)==10 ); - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; - pPage->xParseCell = btreeParseCellPtrIndex; - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - }else{ - /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is - ** an error. */ - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; - pPage->xParseCell = btreeParseCellPtrIndex; - return SQLITE_CORRUPT_PAGE(pPage); - } - pPage->max1bytePayload = pBt->max1bytePayload; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); + } + } return SQLITE_OK; } /* ** Compute the amount of freespace on the page. In other words, fill @@ -5493,13 +5498,29 @@ /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. */ +static SQLITE_NOINLINE int btreeLast(BtCursor *pCur, int *pRes){ + int rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ - int rc; - assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ @@ -5516,27 +5537,11 @@ assert( pCur->pPage->leaf ); #endif *pRes = 0; return SQLITE_OK; } - - rc = moveToRoot(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); - if( rc==SQLITE_OK ){ - pCur->curFlags |= BTCF_AtLast; - }else{ - pCur->curFlags &= ~BTCF_AtLast; - } - }else if( rc==SQLITE_EMPTY ){ - assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); - *pRes = 1; - rc = SQLITE_OK; - } - return rc; + return btreeLast(pCur, pRes); } /* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) ** table near the key intKey. Return a success code. ** @@ -6599,11 +6604,11 @@ } /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } /* Now manipulate the actual database free-list structure. There are two @@ -7039,28 +7044,24 @@ ** pTemp is not null. Regardless of pTemp, allocate a new entry ** in pPage->apOvfl[] and make it point to the cell content (either ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. -** -** *pRC must be SQLITE_OK when this routine is called. */ -static void insertCell( +static int insertCell( MemPage *pPage, /* Page into which we are copying */ int i, /* New cell becomes the i-th cell of the page */ u8 *pCell, /* Content of the new cell */ int sz, /* Bytes of content in pCell */ u8 *pTemp, /* Temp storage space for pCell, if needed */ - Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ - int *pRC /* Read and write return code from here */ + Pgno iChild /* If non-zero, replace first 4 bytes with this value */ ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); @@ -7091,18 +7092,17 @@ assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ - *pRC = rc; - return; + return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); - if( rc ){ *pRC = rc; return; } + if( rc ){ return rc; } /* The allocateSpace() routine guarantees the following properties ** if it returns successfully */ assert( idx >= 0 ); assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); @@ -7125,17 +7125,20 @@ /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; } #endif } + return SQLITE_OK; } /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -7232,18 +7235,20 @@ /* ** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been ** computed. */ static void populateCellCache(CellArray *p, int idx, int N){ + MemPage *pRef = p->pRef; + u16 *szCell = p->szCell; assert( idx>=0 && idx+N<=p->nCell ); while( N>0 ){ assert( p->apCell[idx]!=0 ); - if( p->szCell[idx]==0 ){ - p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + if( szCell[idx]==0 ){ + szCell[idx] = pRef->xCellSize(pRef, p->apCell[idx]); }else{ assert( CORRUPT_DB || - p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + szCell[idx]==pRef->xCellSize(pRef, p->apCell[idx]) ); } idx++; N--; } } @@ -7441,12 +7446,12 @@ u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; int iEnd = iFirst + nCell; - u8 *pFree = 0; - int szFree = 0; + u8 *pFree = 0; /* \__ Parameters for pending call to */ + int szFree = 0; /* / freeSpace() */ for(i=iFirst; iapCell[i]; if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; @@ -7463,10 +7468,13 @@ szFree = sz; if( pFree+sz>pEnd ){ return 0; } }else{ + /* The current cell is adjacent to and before the pFree cell. + ** Combine the two regions into one to reduce the number of calls + ** to freeSpace(). */ pFree = pCell; szFree += sz; } nRet++; } @@ -7670,11 +7678,11 @@ ** of the parent page are still manipulated by thh code below. ** That is Ok, at this point the parent page is guaranteed to ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); } } @@ -7698,12 +7706,12 @@ pStop = &pCell[9]; while( ((*(pOut++) = *(pCell++))&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + rc = insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno); } /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); @@ -7808,11 +7816,11 @@ } /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ *pRC = setChildPtrmaps(pTo); } } } @@ -8232,19 +8240,21 @@ r = cntNew[i-1] - 1; d = r + 1 - leafData; (void)cachedCellSize(&b, d); do{ + int szR, szD; assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + && (bBulk || szRight+szD+2 > szLeft-(szR+(i==k-1?0:2)))){ break; } - szRight += b.szCell[d] + 2; - szLeft -= b.szCell[r] + 2; + szRight += szD + 2; + szLeft -= szR + 2; cntNew[i-1] = r; r--; d--; }while( r>=0 ); szNew[i] = szRight; @@ -8294,11 +8304,11 @@ apNew[i] = pNew; nNew++; cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; } } @@ -8387,11 +8397,11 @@ ** If the sibling pages are not leaves, then the pointer map entry ** associated with the right-child of each sibling may also need to be ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ MemPage *pOld; MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; int iNew = 0; int iOld = 0; @@ -8484,11 +8494,11 @@ pSrcEnd = b.apEnd[k]; if( SQLITE_WITHIN(pSrcEnd, pCell, pCell+sz) ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + rc = insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno); if( rc!=SQLITE_OK ) goto balance_cleanup; assert( sqlite3PagerIswriteable(pParent->pDbPage) ); } /* Now update the actual sibling pages. The order in which they are updated @@ -8580,11 +8590,11 @@ - apNew[0]->nCell*2) || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM && !leafCorrection ){ + }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ for(i=0; iaData[8]); @@ -8601,11 +8611,11 @@ for(i=nNew; iisInit ){ + if( ISAUTOVACUUM(pBt) && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); @@ -8663,11 +8673,11 @@ */ rc = sqlite3PagerWrite(pRoot->pDbPage); if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } if( rc ){ *ppChild = 0; @@ -8995,11 +9005,10 @@ int loc = seekResult; /* -1: before desired location +1: after */ int szNew = 0; int idx; MemPage *pPage; Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; unsigned char *oldCell; unsigned char *newCell = 0; assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND|BTREE_PREFORMAT))==flags ); assert( (flags & BTREE_PREFORMAT)==0 || seekResult || pCur->pKeyInfo==0 ); @@ -9014,11 +9023,11 @@ ** that the cursor is already where it needs to be and returns without ** doing any work. To avoid thwarting these optimizations, it is important ** not to clear the cursor here. */ if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + rc = saveAllCursors(p->pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; if( loc && pCur->iPage<0 ){ /* This can only happen if the schema is corrupt such that there is more ** than one table or index with the same root page as used by the cursor. ** Which can only happen if the SQLITE_NoSchemaError flag was set when @@ -9038,12 +9047,12 @@ if( rc && rc!=SQLITE_EMPTY ) return rc; } assert( cursorOwnsBtShared(pCur) ); assert( (pCur->curFlags & BTCF_WriteFlag)!=0 - && pBt->inTransaction==TRANS_WRITE - && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + && p->pBt->inTransaction==TRANS_WRITE + && (p->pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); /* Assert that the caller has been consistent. If this cursor was opened ** expecting an index b-tree, then the caller should be inserting blob ** keys with no associated data. If the cursor was opened expecting an @@ -9156,30 +9165,32 @@ TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit || CORRUPT_DB ); - newCell = pBt->pTmpSpace; + newCell = p->pBt->pTmpSpace; assert( newCell!=0 ); + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; - szNew = pBt->nPreformatSize; + szNew = p->pBt->nPreformatSize; if( szNew<4 ) szNew = 4; - if( ISAUTOVACUUM && szNew>pPage->maxLocal ){ + if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); if( info.nPayload!=info.nLocal ){ Pgno ovfl = get4byte(&newCell[szNew-4]); - ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + ptrmapPut(p->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + if( NEVER(rc) ) goto end_insert; } } }else{ rc = fillInCell(pPage, newCell, pX, &szNew); + if( rc ) goto end_insert; } - if( rc ) goto end_insert; assert( szNew==pPage->xCellSize(pPage, newCell) ); - assert( szNew <= MX_CELL_SIZE(pBt) ); + assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; if( loc==0 ){ CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ @@ -9195,11 +9206,11 @@ } BTREE_CLEAR_CELL(rc, pPage, oldCell, info); testcase( pCur->curFlags & BTCF_ValidOvfl ); invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload - && (!ISAUTOVACUUM || szNewminLocal) + && (!ISAUTOVACUUM(p->pBt) || szNewminLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add ** the leftover space to the free list. But experiments show that ** doing that is no faster then skipping this optimization and just @@ -9225,11 +9236,11 @@ idx = ++pCur->ix; pCur->curFlags &= ~BTCF_ValidNKey; }else{ assert( pPage->leaf ); } - insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + rc = insertCell(pPage, idx, newCell, szNew, 0, 0); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move @@ -9298,11 +9309,10 @@ ** calling sqlite3BtreeInsert() with the BTREE_PREFORMAT flag specified. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ - int rc = SQLITE_OK; BtShared *pBt = pDest->pBt; u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ const u8 *aIn; /* Pointer to next input buffer */ u32 nIn; /* Size of input buffer aIn[] */ u32 nRem; /* Bytes of data still to copy */ @@ -9321,11 +9331,13 @@ } nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ memcpy(aOut, aIn, nIn); pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + return SQLITE_OK; }else{ + int rc = SQLITE_OK; Pager *pSrcPager = pSrc->pBt->pPager; u8 *pPgnoOut = 0; Pgno ovflIn = 0; DbPage *pPageIn = 0; MemPage *pPageOut = 0; @@ -9373,11 +9385,11 @@ if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ Pgno pgnoNew; MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); - if( ISAUTOVACUUM && pPageOut ){ + if( ISAUTOVACUUM(pBt) && pPageOut ){ ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); } releasePage(pPageOut); pPageOut = pNew; if( pPageOut ){ @@ -9389,13 +9401,12 @@ } }while( nRem>0 && rc==SQLITE_OK ); releasePage(pPageOut); sqlite3PagerUnref(pPageIn); + return rc; } - - return rc; } /* ** Delete the entry that the cursor is pointing to. ** @@ -9546,11 +9557,11 @@ assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); if( rc==SQLITE_OK ){ - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n); } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; } Index: src/btree.h ================================================================== --- src/btree.h +++ src/btree.h @@ -181,11 +181,11 @@ ** to prefetch content from remote machines - to provide those ** implementations with limits on what needs to be prefetched and thereby ** reduce network bandwidth. ** ** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by -** standard SQLite. The other hints are provided for extentions that use +** standard SQLite. The other hints are provided for extensions that use ** the SQLite parser and code generator but substitute their own storage ** engine. */ #define BTREE_HINT_RANGE 0 /* Range constraints on queries */ Index: src/btreeInt.h ================================================================== --- src/btreeInt.h +++ src/btreeInt.h @@ -672,13 +672,13 @@ ** within an expression that is an argument to another macro ** (sqliteMallocRaw), it is not possible to use conditional compilation. ** So, this macro is defined instead. */ #ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) +#define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else -#define ISAUTOVACUUM 0 +#define ISAUTOVACUUM(pBt) 0 #endif /* ** This structure is passed around through all the sanity checking routines Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -2002,10 +2002,17 @@ assert( TF_HasStored==COLFLAG_STORED ); pTab->tabFlags |= eType; if( pCol->colFlags & COLFLAG_PRIMKEY ){ makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ } + if( ALWAYS(pExpr) && pExpr->op==TK_ID ){ + /* The value of a generated column needs to be a real expression, not + ** just a reference to another column, in order for covering index + ** optimizations to work correctly. So if the value is not an expression, + ** turn it into one by adding a unary "+" operator. */ + pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); + } sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); pExpr = 0; goto generated_done; generated_error: @@ -2138,11 +2145,12 @@ static const char * const azType[] = { /* SQLITE_AFF_BLOB */ "", /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", - /* SQLITE_AFF_REAL */ " REAL" + /* SQLITE_AFF_REAL */ " REAL", + /* SQLITE_AFF_FLEXNUM */ " NUM", }; int len; const char *zType; sqlite3_snprintf(n-k, &zStmt[k], zSep); @@ -2154,14 +2162,16 @@ testcase( pCol->affinity==SQLITE_AFF_BLOB ); testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); + testcase( pCol->affinity==SQLITE_AFF_FLEXNUM ); zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_BLOB + assert( pCol->affinity==SQLITE_AFF_BLOB + || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; assert( k<=n ); } @@ -2572,10 +2582,11 @@ ** they should not be changed. Expressions attached to a table or ** index definition are tagged this way to help ensure that we do ** not pass them into code generator routines by mistake. */ static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + (void)pWalker; ExprSetVVAProperty(pExpr, EP_Immutable); return WRC_Continue; } static void markExprListImmutable(ExprList *pList){ if( pList ){ @@ -3138,12 +3149,11 @@ &pTable->nCol, &pTable->aCol); if( pParse->nErr==0 && pTable->nCol==pSel->pEList->nExpr ){ assert( db->mallocFailed==0 ); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, - SQLITE_AFF_NONE); + sqlite3SubqueryColumnTypes(pParse, pTable, pSel, SQLITE_AFF_NONE); } }else{ /* CREATE VIEW name AS... without an argument list. Construct ** the column names from the SELECT statement that defines the view. */ Index: src/dbpage.c ================================================================== --- src/dbpage.c +++ src/dbpage.c @@ -70,10 +70,14 @@ sqlite3_vtab **ppVtab, char **pzErr ){ DbpageTable *pTab = 0; int rc = SQLITE_OK; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); rc = sqlite3_declare_vtab(db, "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ @@ -108,10 +112,11 @@ ** 3 schema=?1, pgno=?2 */ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; int iPlan = 0; + (void)tab; /* If there is a schema= constraint, it must be honored. Report a ** ridiculously large estimated cost if the schema= constraint is ** unavailable */ @@ -223,10 +228,12 @@ DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; int rc; sqlite3 *db = pTab->db; Btree *pBt; + (void)idxStr; + /* Default setting is no rows of result */ pCsr->pgno = 1; pCsr->mxPgno = 0; if( idxNum & 2 ){ @@ -318,10 +325,11 @@ int iDb; Btree *pBt; Pager *pPager; int szPage; + (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ zErr = "read-only"; goto update_fail; } if( argc==1 ){ Index: src/dbstat.c ================================================================== --- src/dbstat.c +++ src/dbstat.c @@ -161,10 +161,11 @@ char **pzErr ){ StatTable *pTab = 0; int rc = SQLITE_OK; int iDb; + (void)pAux; if( argc>=4 ){ Token nm; sqlite3TokenInit(&nm, (char*)argv[3]); iDb = sqlite3FindDb(db, &nm); @@ -214,10 +215,11 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; int iSchema = -1; int iName = -1; int iAgg = -1; + (void)tab; /* Look for a valid schema=? constraint. If found, change the idxNum to ** 1 and request the value of that constraint be sent to xFilter. And ** lower the cost estimate to encourage the constrained version to be ** used. @@ -739,10 +741,12 @@ sqlite3_str *pSql; /* Query of btrees to analyze */ char *zSql; /* String value of pSql */ int iArg = 0; /* Count of argv[] parameters used so far */ int rc = SQLITE_OK; /* Result of this operation */ const char *zName = 0; /* Only provide analysis of this table */ + (void)argc; + (void)idxStr; statResetCsr(pCsr); sqlite3_finalize(pCsr->pStmt); pCsr->pStmt = 0; if( idxNum & 0x01 ){ @@ -822,28 +826,28 @@ if( !pCsr->isAgg ){ sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC); } break; case 4: /* ncell */ - sqlite3_result_int(ctx, pCsr->nCell); + sqlite3_result_int64(ctx, pCsr->nCell); break; case 5: /* payload */ - sqlite3_result_int(ctx, pCsr->nPayload); + sqlite3_result_int64(ctx, pCsr->nPayload); break; case 6: /* unused */ - sqlite3_result_int(ctx, pCsr->nUnused); + sqlite3_result_int64(ctx, pCsr->nUnused); break; case 7: /* mx_payload */ - sqlite3_result_int(ctx, pCsr->nMxPayload); + sqlite3_result_int64(ctx, pCsr->nMxPayload); break; case 8: /* pgoffset */ if( !pCsr->isAgg ){ sqlite3_result_int64(ctx, pCsr->iOffset); } break; case 9: /* pgsize */ - sqlite3_result_int(ctx, pCsr->szPage); + sqlite3_result_int64(ctx, pCsr->szPage); break; case 10: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); int iDb = pCsr->iDb; sqlite3_result_text(ctx, db->aDb[iDb].zDbSName, -1, SQLITE_STATIC); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -42,51 +42,124 @@ ** SELECT a AS b FROM t1 WHERE b; ** SELECT * FROM t1 WHERE (select a from t1); */ char sqlite3ExprAffinity(const Expr *pExpr){ int op; - while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ - assert( pExpr->op==TK_COLLATE - || pExpr->op==TK_IF_NULL_ROW - || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); - pExpr = pExpr->pLeft; - assert( pExpr!=0 ); - } op = pExpr->op; - if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ - assert( ExprUseYTab(pExpr) ); - assert( pExpr->y.pTab!=0 ); - return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - } - if( op==TK_SELECT ){ - assert( ExprUseXSelect(pExpr) ); - assert( pExpr->x.pSelect!=0 ); - assert( pExpr->x.pSelect->pEList!=0 ); - assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); - return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); - } + while( 1 /* exit-by-break */ ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ + assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + } + if( op==TK_SELECT ){ + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( pExpr->x.pSelect->pEList!=0 ); + assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } #ifndef SQLITE_OMIT_CAST - if( op==TK_CAST ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - return sqlite3AffinityType(pExpr->u.zToken, 0); - } + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } #endif - if( op==TK_SELECT_COLUMN ){ - assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); - assert( pExpr->iColumn < pExpr->iTable ); - assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); - return sqlite3ExprAffinity( - pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr - ); - } - if( op==TK_VECTOR ){ - assert( ExprUseXList(pExpr) ); - return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); + assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); + } + if( op==TK_VECTOR ){ + assert( ExprUseXList(pExpr) ); + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + if( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ + assert( pExpr->op==TK_COLLATE + || pExpr->op==TK_IF_NULL_ROW + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); + pExpr = pExpr->pLeft; + op = pExpr->op; + continue; + } + if( op!=TK_REGISTER || (op = pExpr->op2)==TK_REGISTER ) break; } return pExpr->affExpr; } + +/* +** Make a guess at all the possible datatypes of the result that could +** be returned by an expression. Return a bitmask indicating the answer: +** +** 0x01 Numeric +** 0x02 Text +** 0x04 Blob +** +** If the expression must return NULL, then 0x00 is returned. +*/ +int sqlite3ExprDataType(const Expr *pExpr){ + while( pExpr ){ + switch( pExpr->op ){ + case TK_COLLATE: + case TK_IF_NULL_ROW: + case TK_UPLUS: { + pExpr = pExpr->pLeft; + break; + } + case TK_NULL: { + pExpr = 0; + break; + } + case TK_STRING: { + return 0x02; + } + case TK_BLOB: { + return 0x04; + } + case TK_CONCAT: { + return 0x06; + } + case TK_VARIABLE: + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + return 0x07; + } + case TK_COLUMN: + case TK_AGG_COLUMN: + case TK_SELECT: + case TK_CAST: + case TK_SELECT_COLUMN: + case TK_VECTOR: { + int aff = sqlite3ExprAffinity(pExpr); + if( aff>=SQLITE_AFF_NUMERIC ) return 0x05; + if( aff==SQLITE_AFF_TEXT ) return 0x06; + return 0x07; + } + case TK_CASE: { + int res = 0; + int ii; + ExprList *pList = pExpr->x.pList; + assert( ExprUseXList(pExpr) && pList!=0 ); + assert( pList->nExpr > 0); + for(ii=1; iinExpr; ii+=2){ + res |= sqlite3ExprDataType(pList->a[ii].pExpr); + } + if( pList->nExpr % 2 ){ + res |= sqlite3ExprDataType(pList->a[pList->nExpr-1].pExpr); + } + return res; + } + default: { + return 0x01; + } + } /* End of switch(op) */ + } /* End of while(pExpr) */ + return 0x00; +} /* ** Set the collating sequence for expression pExpr to be the collating ** sequence named by pToken. Return a pointer to a new Expr node that ** implements the COLLATE operator. @@ -171,11 +244,13 @@ CollSeq *pColl = 0; const Expr *p = pExpr; while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ int j; assert( ExprUseYTab(p) ); assert( p->y.pTab!=0 ); if( (j = p->iColumn)>=0 ){ const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); @@ -3253,10 +3328,13 @@ int rReg = 0; /* Register storing resulting */ Select *pSel; /* SELECT statement to encode */ SelectDest dest; /* How to deal with SELECT result */ int nReg; /* Registers to allocate */ Expr *pLimit; /* New limit expression */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif Vdbe *v = pParse->pVdbe; assert( v!=0 ); if( pParse->nErr ) return 0; testcase( pExpr->op==TK_EXISTS ); @@ -3305,12 +3383,13 @@ ** into a register and return that register number. ** ** In both cases, the query is augmented with "LIMIT 1". Any ** preexisting limit is discarded in place of the new LIMIT 1. */ - ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", addrOnce?"":"CORRELATED ", pSel->selId)); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, -1); nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); pParse->nMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; @@ -3349,10 +3428,11 @@ pExpr->iTable = rReg = dest.iSDParm; ExprSetVVAProperty(pExpr, EP_NoReduce); if( addrOnce ){ sqlite3VdbeJumpHere(v, addrOnce); } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn || pParse->nErr ); @@ -4035,11 +4115,11 @@ } return target; } /* -** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. ** If it is, then resolve the expression by reading from the index and ** return the register into which the value has been read. If pExpr is ** not an indexed expression, then return negative. */ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( @@ -4047,11 +4127,11 @@ Expr *pExpr, /* The expression to potentially bypass */ int target /* Where to store the result of the expression */ ){ IndexedExpr *p; Vdbe *v; - for(p=pParse->pIdxExpr; p; p=p->pIENext){ + for(p=pParse->pIdxEpr; p; p=p->pIENext){ int iDataCur = p->iDataCur; if( iDataCur<0 ) continue; if( pParse->iSelfTab ){ if( p->iDataCur!=pParse->iSelfTab-1 ) continue; iDataCur = -1; @@ -4067,14 +4147,14 @@ sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); sqlite3VdbeGoto(v, 0); - p = pParse->pIdxExpr; - pParse->pIdxExpr = 0; + p = pParse->pIdxEpr; + pParse->pIdxEpr = 0; sqlite3ExprCode(pParse, pExpr, target); - pParse->pIdxExpr = p; + pParse->pIdxEpr = p; sqlite3VdbeJumpHere(v, addr+2); }else{ sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); } @@ -4109,11 +4189,11 @@ assert( v!=0 ); expr_code_doover: if( pExpr==0 ){ op = TK_NULL; - }else if( pParse->pIdxExpr!=0 + }else if( pParse->pIdxEpr!=0 && !ExprHasProperty(pExpr, EP_Leaf) && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 ){ return r1; }else{ @@ -4126,26 +4206,32 @@ struct AggInfo_col *pCol; assert( pAggInfo!=0 ); assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( ALWAYS(pTab!=0) ){ + }else{ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, target); } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through } case TK_COLUMN: { @@ -4162,11 +4248,11 @@ iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); if( aff>SQLITE_AFF_BLOB ){ - static const char zAff[] = "B\000C\000D\000E"; + static const char zAff[] = "B\000C\000D\000E\000F"; assert( SQLITE_AFF_BLOB=='A' ); assert( SQLITE_AFF_TEXT=='B' ); sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, &zAff[(aff-'B')*2], P4_STATIC); } @@ -4439,11 +4525,11 @@ || NEVER(pExpr->iAgg>=pInfo->nFunc) ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } case TK_FUNCTION: { ExprList *pFarg; /* List of function arguments */ @@ -4728,11 +4814,11 @@ u8 okConstFactor = pParse->okConstFactor; AggInfo *pAggInfo = pExpr->pAggInfo; if( pAggInfo ){ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); if( !pAggInfo->directMode ){ - inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); break; } if( pExpr->pAggInfo->useSortingIdx ){ sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pAggInfo->aCol[pExpr->iAgg].iSorterColumn, @@ -6159,14 +6245,12 @@ ** For Expr nodes that contain pAggInfo pointers, make sure the AggInfo ** object that is referenced does not refer directly to the Expr. If ** it does, make a copy. This is done because the pExpr argument is ** subject to change. ** -** The copy is stored on pParse->pConstExpr with a register number of 0. -** This will cause the expression to be deleted automatically when the -** Parse object is destroyed, but the zero register number means that it -** will not generate any code in the preamble. +** The copy is scheduled for deletion using the sqlite3ExprDeferredDelete() +** which builds on the sqlite3ParserAddCleanup() mechanism. */ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) && pExpr->pAggInfo!=0 ){ @@ -6173,11 +6257,10 @@ AggInfo *pAggInfo = pExpr->pAggInfo; int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; if( pExpr->op!=TK_AGG_FUNCTION ){ - assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_IF_NULL_ROW ); assert( iAgg>=0 && iAggnColumn ); if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; @@ -6239,10 +6322,77 @@ &pInfo->nFunc, &i ); return i; } + +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an idex of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + assert( pAggInfo->iFirstReg==0 ); + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( pExpr->pAggInfo==0 || pExpr->pAggInfo==pAggInfo ); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} /* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates ** for additional information. @@ -6253,11 +6403,41 @@ Parse *pParse = pNC->pParse; SrcList *pSrcList = pNC->pSrcList; AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + assert( pParse->iSelfTab==0 ); + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; + if( pExpr->pAggInfo!=0 ) break; /* Already resolved by outer context */ + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_COLUMN ); @@ -6265,71 +6445,13 @@ /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable - && pCol->iColumn==pExpr->iColumn - && pExpr->op!=TK_IF_NULL_ROW - ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN - && pE->iTable==pExpr->iTable - && pE->iColumn==pExpr->iColumn - ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - if( pExpr->op==TK_COLUMN ){ - pExpr->op = TK_AGG_COLUMN; - } - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } return WRC_Prune; @@ -6355,11 +6477,10 @@ i = addAggInfoFunc(pParse->db, pAggInfo); if( i>=0 ){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); pItem->pFunc = sqlite3FindFunction(pParse->db, pExpr->u.zToken, pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); if( pExpr->flags & EP_Distinct ){ Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -1080,11 +1080,11 @@ sqlite3_str_appendf(pStr, "%lld", sqlite3_value_int64(pValue)); break; } case SQLITE_BLOB: { char const *zBlob = sqlite3_value_blob(pValue); - int nBlob = sqlite3_value_bytes(pValue); + i64 nBlob = sqlite3_value_bytes(pValue); assert( zBlob==sqlite3_value_blob(pValue) ); /* No encoding change */ sqlite3StrAccumEnlarge(pStr, nBlob*2 + 4); if( pStr->accError==0 ){ char *zText = pStr->zText; int i; @@ -1438,10 +1438,13 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ /* no-op */ + (void)context; + (void)argc; + (void)argv; } #endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ /* IMP: R-25361-16150 This function is omitted from SQLite by default. It @@ -2104,21 +2107,19 @@ default: return; } ans = log(x)/b; }else{ - ans = log(x); switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ case 1: - /* Convert from natural logarithm to log base 10 */ - ans /= M_LN10; + ans = log10(x); break; case 2: - /* Convert from natural logarithm to log base 2 */ - ans /= M_LN2; + ans = log2(x); break; default: + ans = log(x); break; } } sqlite3_result_double(context, ans); } @@ -2183,10 +2184,11 @@ sqlite3_context *context, int argc, sqlite3_value **argv ){ assert( argc==0 ); + (void)argv; sqlite3_result_double(context, M_PI); } #endif /* SQLITE_ENABLE_MATH_FUNCTIONS */ Index: src/hwtime.h ================================================================== --- src/hwtime.h +++ src/hwtime.h @@ -46,13 +46,13 @@ #endif #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; } #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -1791,10 +1791,11 @@ /* no break */ deliberate_fall_through case OE_Rollback: case OE_Fail: { char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, pCol->zCnName); + testcase( zMsg==0 && db->mallocFailed==0 ); sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, iReg); sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); VdbeCoverage(v); Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -1564,10 +1564,11 @@ case SQLITE_ROW: zName = "SQLITE_ROW"; break; case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; case SQLITE_NOTICE_RECOVER_ROLLBACK: zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_NOTICE_RBU: zName = "SQLITE_NOTICE_RBU"; break; case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; case SQLITE_DONE: zName = "SQLITE_DONE"; break; } } Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -277,11 +277,11 @@ ** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). ** ** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 ** This provides a 256-byte safety margin for defense against 32-bit ** signed integer overflow bugs when computing memory allocation sizes. -** Parnoid applications might want to reduce the maximum allocation size +** Paranoid applications might want to reduce the maximum allocation size ** further for an even larger safety margin. 0x3fffffff or 0x0fffffff ** or even smaller would be reasonable upper bounds on the size of a memory ** allocations for most applications. */ #ifndef SQLITE_MAX_ALLOCATION_SIZE @@ -791,13 +791,18 @@ ** SQL statement. Make a copy of this phrase in space obtained form ** sqlite3DbMalloc(). Omit leading and trailing whitespace. */ char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ int n; +#ifdef SQLITE_DEBUG + /* Because of the way the parser works, the span is guaranteed to contain + ** at least one non-space character */ + for(n=0; sqlite3Isspace(zStart[n]); n++){ assert( &zStart[n]0) && sqlite3Isspace(zStart[n-1]) ) n--; + while( sqlite3Isspace(zStart[n-1]) ) n--; return sqlite3DbStrNDup(db, zStart, n); } /* ** Free any prior content in *pz and replace it with a copy of zNew. Index: src/mem5.c ================================================================== --- src/mem5.c +++ src/mem5.c @@ -422,17 +422,13 @@ int iFullSz; if( n<=mem5.szAtom*2 ){ if( n<=mem5.szAtom ) return mem5.szAtom; return mem5.szAtom*2; } - if( n>0x10000000 ){ - if( n>0x40000000 ) return 0; - if( n>0x20000000 ) return 0x40000000; - return 0x20000000; - } + if( n>0x40000000 ) return 0; for(iFullSz=mem5.szAtom*8; iFullSz=(i64)n ) return iFullSz/2; + if( (iFullSz/2)>=n ) return iFullSz/2; return iFullSz; } /* ** Return the ceiling of the logarithm base 2 of iValue. Index: src/memdb.c ================================================================== --- src/memdb.c +++ src/memdb.c @@ -897,10 +897,17 @@ sqlite3_free(pData); } sqlite3_mutex_leave(db->mutex); return rc; } + +/* +** Return true if the VFS is the memvfs. +*/ +int sqlite3IsMemdb(const sqlite3_vfs *pVfs){ + return pVfs==&memdb_vfs; +} /* ** This routine is called when the extension is loaded. ** Register the new VFS. */ Index: src/os_common.h ================================================================== --- src/os_common.h +++ src/os_common.h @@ -33,16 +33,10 @@ ** Macros for performance tracing. Normally turned off. Only works ** on i486 hardware. */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - static sqlite_uint64 g_start; static sqlite_uint64 g_elapsed; #define TIMER_START g_start=sqlite3Hwtime() #define TIMER_END g_elapsed=sqlite3Hwtime()-g_start #define TIMER_ELAPSED g_elapsed Index: src/os_kv.c ================================================================== --- src/os_kv.c +++ src/os_kv.c @@ -50,11 +50,13 @@ int isJournal; /* True if this is a journal file */ unsigned int nJrnl; /* Space allocated for aJrnl[] */ char *aJrnl; /* Journal content */ int szPage; /* Last known page size */ sqlite3_int64 szDb; /* Database file size. -1 means unknown */ + char *aData; /* Buffer to hold page data */ }; +#define SQLITE_KVOS_SZ 133073 /* ** Methods for KVVfsFile */ static int kvvfsClose(sqlite3_file*); @@ -491,10 +493,11 @@ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, pFile->isJournal ? "journal" : "db")); sqlite3_free(pFile->aJrnl); + sqlite3_free(pFile->aData); return SQLITE_OK; } /* ** Read from the -journal file. @@ -539,11 +542,11 @@ ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; unsigned int pgno; int got, n; char zKey[30]; - char aData[133073]; + char *aData = pFile->aData; assert( iOfst>=0 ); assert( iAmt>=0 ); SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); if( iOfst+iAmt>=512 ){ if( (iOfst % iAmt)!=0 ){ @@ -556,19 +559,20 @@ pgno = 1 + iOfst/iAmt; }else{ pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ aData[got] = 0; if( iOfst+iAmt<512 ){ int k = iOfst+iAmt; aData[k*2] = 0; - n = kvvfsDecode(aData, &aData[2000], sizeof(aData)-2000); + n = kvvfsDecode(aData, &aData[2000], SQLITE_KVOS_SZ-2000); if( n>=iOfst+iAmt ){ memcpy(zBuf, &aData[2000+iOfst], iAmt); n = iAmt; }else{ n = 0; @@ -623,11 +627,11 @@ sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; unsigned int pgno; char zKey[30]; - char aData[131073]; + char *aData = pFile->aData; SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); assert( iAmt>=512 && iAmt<=65536 ); assert( (iAmt & (iAmt-1))==0 ); assert( pFile->szPage<0 || pFile->szPage==iAmt ); pFile->szPage = iAmt; @@ -831,10 +835,14 @@ } if( zName[0]=='s' ){ pFile->zClass = "session"; }else{ pFile->zClass = "local"; + } + pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); + if( pFile->aData==0 ){ + return SQLITE_NOMEM; } pFile->aJrnl = 0; pFile->nJrnl = 0; pFile->szPage = -1; pFile->szDb = -1; Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -6677,11 +6677,11 @@ ** requested from the underlying operating system, a number which ** might be greater than or equal to the argument, but not less ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS +#if OS_VXWORKS || _POSIX_C_SOURCE >= 199309L struct timespec sp; sp.tv_sec = microseconds / 1000000; sp.tv_nsec = (microseconds % 1000000) * 1000; nanosleep(&sp, NULL); Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -7007,11 +7007,15 @@ ** The return value to this routine is always safe to use with ** sqlite3_uri_parameter() and sqlite3_filename_database() and friends. */ const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; - return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; + if( nullIfMemDb && (pPager->memDb || sqlite3IsMemdb(pPager->pVfs)) ){ + return &zFake[4]; + }else{ + return pPager->zFilename; + } } /* ** Return the VFS structure for the pager. */ Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -518,12 +518,12 @@ ** value stored as part of the in-memory schema representation, ** set Parse.rc to SQLITE_SCHEMA. */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + if( DbHasProperty(db, iDb, DB_SchemaLoaded) ) pParse->rc = SQLITE_SCHEMA; sqlite3ResetOneSchema(db, iDb); - pParse->rc = SQLITE_SCHEMA; } /* Close the transaction, if one was opened. */ if( openedTransaction ){ sqlite3BtreeCommit(pBt); Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -734,17 +734,30 @@ buf[3] = 0x80 + (u8)(ch & 0x3f); length = 4; } } if( precision>1 ){ + i64 nPrior = 1; width -= precision-1; if( width>1 && !flag_leftjustify ){ sqlite3_str_appendchar(pAccum, width-1, ' '); width = 0; } - while( precision-- > 1 ){ - sqlite3_str_append(pAccum, buf, length); + sqlite3_str_append(pAccum, buf, length); + precision--; + while( precision > 1 ){ + i64 nCopyBytes; + if( nPrior > precision-1 ) nPrior = precision - 1; + nCopyBytes = length*nPrior; + if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ + sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + } + if( pAccum->accError ) break; + sqlite3_str_append(pAccum, + &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); + precision -= nPrior; + nPrior *= 2; } } bufpt = buf; flag_altform2 = 1; goto adjust_width_for_utf8; @@ -968,13 +981,13 @@ ** able to accept at least N more bytes of text. ** ** Return the number of bytes of text that StrAccum is able to accept ** after the attempted enlargement. The value returned might be zero. */ -int sqlite3StrAccumEnlarge(StrAccum *p, int N){ +int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ char *zNew; - assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ testcase(p->accError==SQLITE_TOOBIG); testcase(p->accError==SQLITE_NOMEM); return 0; } @@ -981,12 +994,11 @@ if( p->mxAlloc==0 ){ sqlite3StrAccumSetError(p, SQLITE_TOOBIG); return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; - i64 szNew = p->nChar; - szNew += (sqlite3_int64)N + 1; + i64 szNew = p->nChar + N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, ** to avoid having to call this routine too often */ szNew += p->nChar; } @@ -1012,11 +1024,12 @@ sqlite3_str_reset(p); sqlite3StrAccumSetError(p, SQLITE_NOMEM); return 0; } } - return N; + assert( N>=0 && N<=0x7fffffff ); + return (int)N; } /* ** Append N copies of character c to the given string buffer. */ Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -63,10 +63,14 @@ int iCsr; /* Cursor number for table */ int nKey; /* Number of PK columns for table pTab (>=1) */ } aDefer[4]; #endif struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrPush; /* First instruction to push data into sorter */ + int addrPushEnd; /* Last instruction that pushes data into sorter */ +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure. Deallocate the structure @@ -718,10 +722,14 @@ ** SortCtx.pDeferredRowLoad optimiation. In any of these cases ** regOrigData is 0 to prevent this routine from trying to copy ** values that might not yet exist. */ assert( nData==1 || regData==regOrigData || regOrigData==0 ); + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPush = sqlite3VdbeCurrentAddr(v); +#endif if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nPrefixReg; }else{ @@ -819,10 +827,13 @@ regBase+nOBSat, nBase-nOBSat); if( iSkip ){ sqlite3VdbeChangeP2(v, iSkip, pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPushEnd = sqlite3VdbeCurrentAddr(v)-1; +#endif } /* ** Add code to implement the OFFSET */ @@ -1285,13 +1296,10 @@ testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); - if( pDest->zAffSdst ){ - sqlite3VdbeChangeP4(v, -1, pDest->zAffSdst, nResultCol); - } #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open ** on an ephemeral index. If the current row is already present ** in the index, do not write it to the output. If not, add the @@ -1645,10 +1653,20 @@ int iSortTab; /* Sorter cursor to read from */ int i; int bSeq; /* True if sorter record includes seq. no. */ int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif + + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") + ); + sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); + assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeGoto(v, addrBreak); @@ -1757,10 +1775,11 @@ } sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); VdbeComment((v, "%s", aOutEx[i].zEName)); } } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); switch( eDest ){ case SRT_Table: case SRT_EphemTab: { sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); @@ -1818,10 +1837,11 @@ if( pSort->sortFlags & SORTFLAG_UseSorter ){ sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + sqlite3VdbeScanStatusRange(v, addrExplain, sqlite3VdbeCurrentAddr(v)-1, -1); if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); } /* @@ -1848,10 +1868,11 @@ #ifdef SQLITE_ENABLE_COLUMN_METADATA # define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E) #else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ # define columnType(A,B,C,D,E) columnTypeImpl(A,B) #endif +#ifndef SQLITE_OMIT_DECLTYPE static const char *columnTypeImpl( NameContext *pNC, #ifndef SQLITE_ENABLE_COLUMN_METADATA Expr *pExpr #else @@ -1878,11 +1899,11 @@ ** database table or a subquery. */ Table *pTab = 0; /* Table structure column is extracted from */ Select *pS = 0; /* Select the column is extracted from */ int iCol = pExpr->iColumn; /* Index of column in pTab */ - while( pNC && !pTab ){ + while( ALWAYS(pNC) && !pTab ){ SrcList *pTabList = pNC->pSrcList; for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( jnSrc ){ pTab = pTabList->a[j].pTab; pS = pTabList->a[j].pSelect; @@ -1889,11 +1910,11 @@ }else{ pNC = pNC->pNext; } } - if( pTab==0 ){ + if( NEVER(pTab==0) ){ /* At one time, code such as "SELECT new.x" within a trigger would ** cause this condition to run. Since then, we have restructured how ** trigger code is generated and so this condition is no longer ** possible. However, it can still be true for statements like ** the following: @@ -1994,10 +2015,11 @@ *pzOrigCol = zOrigCol; } #endif return zType; } +#endif /* !defined(SQLITE_OMIT_DECLTYPE) */ /* ** Generate code that will tell the VDBE the declaration types of columns ** in the result set. */ @@ -2089,11 +2111,11 @@ #endif if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; - SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + TREETRACE(0x80,pParse,pSelect,("generating column names\n")); pTabList = pSelect->pSrc; pEList = pSelect->pEList; assert( v!=0 ); assert( pTabList!=0 ); pParse->colNamesSet = 1; @@ -2265,51 +2287,80 @@ } return SQLITE_OK; } /* -** Add type and collation information to a column list based on -** a SELECT statement. -** -** The column list presumably came from selectColumnNamesFromExprList(). -** The column list has only names, not types or collations. This -** routine goes through and adds the types and collations. -** -** This routine requires that all identifiers in the SELECT -** statement be resolved. +** pTab is a transient Table object that represents a subquery of some +** kind (maybe a parenthesized subquery in the FROM clause of a larger +** query, or a VIEW, or a CTE). This routine computes type information +** for that Table object based on the Select object that implements the +** subquery. For the purposes of this routine, "type infomation" means: +** +** * The datatype name, as it might appear in a CREATE TABLE statement +** * Which collating sequence to use for the column +** * The affinity of the column */ -void sqlite3SelectAddColumnTypeAndCollation( - Parse *pParse, /* Parsing contexts */ - Table *pTab, /* Add column type information to this table */ - Select *pSelect, /* SELECT used to determine types and collations */ - char aff /* Default affinity for columns */ +void sqlite3SubqueryColumnTypes( + Parse *pParse, /* Parsing contexts */ + Table *pTab, /* Add column type information to this table */ + Select *pSelect, /* SELECT used to determine types and collations */ + char aff /* Default affinity. */ ){ sqlite3 *db = pParse->db; - NameContext sNC; Column *pCol; CollSeq *pColl; - int i; + int i,j; Expr *p; struct ExprList_item *a; assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); + assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed ) return; - memset(&sNC, 0, sizeof(sNC)); - sNC.pSrcList = pSelect->pSrc; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; - i64 n, m; + i64 n; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; - zType = columnType(&sNC, p, 0, 0, 0); /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + if( pCol->affinity<=SQLITE_AFF_NONE ){ + pCol->affinity = aff; + }else if( pCol->affinity>=SQLITE_AFF_NUMERIC && p->op==TK_CAST ){ + pCol->affinity = SQLITE_AFF_FLEXNUM; + } + if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ + int m = 0; + Select *pS2; + for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + } + if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + }else + if( pCol->affinity>=SQLITE_AFF_NUMERIC && (m&0x02)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + } + } + if( pCol->affinity==SQLITE_AFF_NUMERIC + || pCol->affinity==SQLITE_AFF_FLEXNUM + ){ + zType = "NUM"; + }else{ + zType = 0; + for(j=1; jaffinity ){ + zType = sqlite3StdType[j]; + break; + } + } + } if( zType ){ - m = sqlite3Strlen30(zType); + i64 m = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); if( pCol->zCnName ){ memcpy(&pCol->zCnName[n+1], zType, m+1); pCol->colFlags |= COLFLAG_HASTYPE; @@ -2316,11 +2367,10 @@ }else{ testcase( pCol->colFlags & COLFLAG_HASTYPE ); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); } } - if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl ){ assert( pTab->pIndex==0 ); sqlite3ColumnSetColl(db, pCol, pColl->zName); } @@ -2350,11 +2400,11 @@ } pTab->nTabRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); + sqlite3SubqueryColumnTypes(pParse, pTab, pSelect, aff); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); return 0; } @@ -2875,11 +2925,11 @@ int nLimit = 0; /* Initialize to suppress harmless compiler warning */ assert( !pPrior->pLimit ); pPrior->iLimit = p->iLimit; pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); rc = sqlite3Select(pParse, pPrior, &dest); pPrior->pLimit = 0; if( rc ){ goto multi_select_end; } @@ -2893,11 +2943,11 @@ sqlite3VdbeAddOp3(v, OP_OffsetLimit, p->iLimit, p->iOffset+1, p->iOffset); } } ExplainQueryPlan((pParse, 1, "UNION ALL")); - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); @@ -2946,11 +2996,11 @@ /* Code the SELECT statements to our left */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; } @@ -2966,11 +3016,11 @@ pLimit = p->pLimit; p->pLimit = 0; uniondest.eDest = op; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); assert( p->pOrderBy==0 ); pDelete = p->pPrior; p->pPrior = pPrior; @@ -3027,11 +3077,11 @@ assert( p->pEList ); /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; } @@ -3044,11 +3094,11 @@ pLimit = p->pLimit; p->pLimit = 0; intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; if( p->nSelectRow>pPrior->nSelectRow ){ @@ -4047,10 +4097,38 @@ while( pSel->pPrior ){ pSel = pSel->pPrior; } return pSel->pEList; } + +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; iinExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. ** This routine returns 1 if it makes changes and 0 if no flattening occurs. @@ -4151,11 +4229,12 @@ ** (17f) the subquery must not be the RHS of a LEFT JOIN. ** (17g) either the subquery is the first element of the outer ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) ** (17h) The corresponding result set expressions in all arms of the -** compound must have the same affinity. +** compound must have the same affinity. (See restriction (9) +** on the push-down optimization.) ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, ** LIMIT and OFFSET clauses. The subquery cannot use any compound ** operator other than UNION ALL because all the other compound @@ -4370,23 +4449,11 @@ /* Restriction (23) */ if( (p->selFlags & SF_Recursive) ) return 0; /* Restriction (17h) */ - for(ii=0; iipEList->nExpr; ii++){ - char aff; - assert( pSub->pEList->a[ii].pExpr!=0 ); - aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr); - for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){ - assert( pSub1->pEList!=0 ); - assert( pSub1->pEList->nExpr>ii ); - assert( pSub1->pEList->a[ii].pExpr!=0 ); - if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ - return 0; - } - } - } + if( compoundHasDifferentAffinities(pSub) ) return 0; if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0; aCsrMap = sqlite3DbMallocZero(db, ((i64)pParse->nTab+1)*sizeof(int)); @@ -4393,11 +4460,11 @@ if( aCsrMap ) aCsrMap[0] = pParse->nTab; } } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + TREETRACE(0x4,pParse,p,("flatten %u.%p from term %d\n", pSub->selId, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); @@ -4472,11 +4539,11 @@ } pNew->pPrior = pPrior; if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; - SELECTTRACE(2,pParse,p,("compound-subquery flattener" + TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } assert( pSubitem->pSelect==0 ); } sqlite3DbFree(db, aCsrMap); @@ -4652,12 +4719,12 @@ sqlite3AggInfoPersistWalkerInit(&w, pParse); sqlite3WalkSelect(&w,pSub1); sqlite3SelectDelete(db, pSub1); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + if( sqlite3TreeTrace & 0x4 ){ + TREETRACE(0x4,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif return 1; @@ -5027,16 +5094,18 @@ ** ** (7) The inner query is a Common Table Expression (CTE) that should ** be materialized. (This restriction is implemented in the calling ** routine.) ** -** (8) The subquery may not be a compound that uses UNION, INTERSECT, -** or EXCEPT. (We could, perhaps, relax this restriction to allow -** this case if none of the comparisons operators between left and -** right arms of the compound use a collation other than BINARY. -** But it is a lot of work to check that case for an obscure and -** minor optimization, so we omit it for now.) +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. +** +** (9) If the subquery is a compound, then all arms of the compound must +** have the same affinity. (This is the same as restriction (17h) +** for query flattening.) +** ** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ static int pushDownWhereTerms( @@ -5049,24 +5118,48 @@ int nChng = 0; if( pWhere==0 ) return 0; if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0; -#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pPrior ){ Select *pSel; + int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ u8 op = pSel->op; assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); - if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */ + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif + } + if( compoundHasDifferentAffinities(pSubq) ){ + return 0; /* restriction (9) */ + } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; iinExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; - } #endif + } #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make ** sure no other terms are marked SF_Recursive in case something changes ** in the future. @@ -6066,12 +6159,12 @@ if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){ p->selFlags |= SF_ComplexResult; } } #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After result-set wildcard expansion:\n")); + if( sqlite3TreeTrace & 0x8 ){ + TREETRACE(0x8,pParse,p,("After result-set wildcard expansion:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif return WRC_Continue; } @@ -6118,18 +6211,18 @@ #ifndef SQLITE_OMIT_SUBQUERY /* ** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() ** interface. ** -** For each FROM-clause subquery, add Column.zType and Column.zColl -** information to the Table structure that represents the result set -** of that subquery. +** For each FROM-clause subquery, add Column.zType, Column.zColl, and +** Column.affinity information to the Table structure that represents +** the result set of that subquery. ** ** The Table structure that represents the result set was constructed -** by selectExpander() but the type and collation information was omitted -** at that point because identifiers had not yet been resolved. This -** routine is called after identifier resolution. +** by selectExpander() but the type and collation and affinity information +** was omitted at that point because identifiers had not yet been resolved. +** This routine is called after identifier resolution. */ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; int i; SrcList *pTabList; @@ -6145,13 +6238,11 @@ assert( pTab!=0 ); if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, - SQLITE_AFF_NONE); + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } } #endif @@ -6201,10 +6292,177 @@ if( pParse->nErr ) return; sqlite3ResolveSelectNames(pParse, p, pOuterNC); if( pParse->nErr ) return; sqlite3SelectAddTypeInfo(pParse, p); } + +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->iFirstReg+pAggInfo->nColumn+ii); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ + if( pAggInfo->nColumn==0 ){ + pAggInfo->nSortingColumn = 0; + }else{ + pAggInfo->nSortingColumn = + pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + } + } + analyzeAggFuncArgs(pAggInfo, pNC); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + IndexedExpr *pIEpr; + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + UNUSED_PARAMETER(pSelect); + UNUSED_PARAMETER(pParse); +#endif +} + +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + UNUSED_PARAMETER(pWalker); + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After clling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; +} /* ** Reset the aggregate accumulator. ** ** The aggregate accumulator is a set of memory cells that hold @@ -6215,28 +6473,17 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); assert( pParse->db->pParse==pParse ); assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; if( pParse->nErr ) return; -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; assert( ExprUseXList(pE) ); if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ @@ -6264,19 +6511,20 @@ struct AggInfo_func *pF; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator ** registers if register regAcc contains 0. The caller will take care ** of setting and clearing regAcc. @@ -6292,10 +6540,12 @@ int regHit = 0; int addrHitTest = 0; struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; int addrNext = 0; int regAgg; @@ -6352,11 +6602,11 @@ pColl = pParse->db->pDfltColl; } if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); @@ -6367,11 +6617,11 @@ } if( regHit ){ addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; if( addrHitTest ){ sqlite3VdbeJumpHereOrPopInst(v, addrHitTest); @@ -6463,30 +6713,35 @@ sWalker.xExprCallback = havingToWhereExprCb; sWalker.u.pSelect = p; sqlite3WalkExpr(&sWalker, p->pHaving); #if TREETRACE_ENABLED if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){ - SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + TREETRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif } /* -** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcItem for the prior view. If it is not, -** then return 0. +** Check to see if the pThis entry of pTabList is a self-join of another view. +** Search FROM-clause entries in the range of iFirst..iEnd, including iFirst +** but stopping before iEnd. +** +** If pThis is a self-join, then return the SrcItem for the first other +** instance of that view found. If pThis is not a self-join then return 0. */ static SrcItem *isSelfJoinView( SrcList *pTabList, /* Search for self-joins in this FROM clause */ - SrcItem *pThis /* Search for prior reference to this subquery */ + SrcItem *pThis, /* Search for prior reference to this subquery */ + int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; assert( pThis->pSelect!=0 ); if( pThis->pSelect->selFlags & SF_PushDown ) return 0; - for(pItem = pTabList->a; pItema[iFirst++]; if( pItem->pSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; assert( pItem->pTab!=0 ); assert( pThis->pTab!=0 ); @@ -6595,12 +6850,12 @@ } p->pEList->a[0].pExpr = pExpr; p->selFlags &= ~SF_Aggregate; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); + if( sqlite3TreeTrace & 0x200 ){ + TREETRACE(0x200,pParse,p,("After count-of-view optimization:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif return 1; } @@ -6626,10 +6881,66 @@ return 1; } } return 0; } + +/* +** Return TRUE (non-zero) if the i-th entry in the pTabList SrcList can +** be implemented as a co-routine. The i-th entry is guaranteed to be +** a subquery. +** +** The subquery is implemented as a co-routine if all of the following are +** true: +** +** (1) The subquery will likely be implemented in the outer loop of +** the query. This will be the case if any one of the following +** conditions hold: +** (a) The subquery is the only term in the FROM clause +** (b) The subquery is the left-most term and a CROSS JOIN or similar +** requires it to be the outer loop +** (c) All of the following are true: +** (i) The subquery is the left-most subquery in the FROM clause +** (ii) There is nothing that would prevent the subquery from +** being used as the outer loop if the sqlite3WhereBegin() +** routine nominates it to that position. +** (iii) The query is not a UPDATE ... FROM +** (2) The subquery is not a CTE that should be materialized because of +** the AS MATERIALIZED keywords +** (3) The subquery is not part of a left operand for a RIGHT JOIN +** (4) The SQLITE_Coroutine optimization disable flag is not set +** (5) The subquery is not self-joined +*/ +static int fromClauseTermCanBeCoroutine( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* FROM clause */ + int i, /* Which term of the FROM clause holds the subquery */ + int selFlags /* Flags on the SELECT statement */ +){ + SrcItem *pItem = &pTabList->a[i]; + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ) return 0;/* (2) */ + if( pTabList->a[0].fg.jointype & JT_LTORJ ) return 0; /* (3) */ + if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4) */ + if( isSelfJoinView(pTabList, pItem, i+1, pTabList->nSrc)!=0 ){ + return 0; /* (5) */ + } + if( i==0 ){ + if( pTabList->nSrc==1 ) return 1; /* (1a) */ + if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1; /* (1b) */ + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + return 1; + } + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + while( 1 /*exit-by-break*/ ){ + if( pItem->fg.jointype & (JT_OUTER|JT_CROSS) ) return 0; /* (1c-ii) */ + if( i==0 ) break; + i--; + pItem--; + if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + } + return 1; +} /* ** Generate code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. @@ -6672,12 +6983,12 @@ return 1; } assert( db->mallocFailed==0 ); if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); - if( sqlite3TreeTrace & 0x10100 ){ + TREETRACE(0x1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3TreeTrace & 0x10000 ){ if( (sqlite3TreeTrace & 0x10001)==0x10000 ){ sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d", __FILE__, __LINE__); } sqlite3ShowSelect(p); @@ -6693,12 +7004,12 @@ pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ #if TREETRACE_ENABLED - SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); - if( sqlite3TreeTrace & 0x100 ){ + TREETRACE(0x800,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3TreeTrace & 0x800 ){ sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } #endif sqlite3ParserAddCleanup(pParse, (void(*)(sqlite3*,void*))sqlite3ExprListDelete, @@ -6714,12 +7025,12 @@ goto select_end; } assert( db->mallocFailed==0 ); assert( p->pEList!=0 ); #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x104 ){ - SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); + if( sqlite3TreeTrace & 0x10 ){ + TREETRACE(0x10,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif /* If the SF_UFSrcCheck flag is set, then this function is being called @@ -6756,12 +7067,12 @@ if( sqlite3WindowRewrite(pParse, p) ){ assert( pParse->nErr ); goto select_end; } #if TREETRACE_ENABLED - if( p->pWin && (sqlite3TreeTrace & 0x108)!=0 ){ - SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + if( p->pWin && (sqlite3TreeTrace & 0x40)!=0 ){ + TREETRACE(0x40,pParse,p, ("after window rewrite:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif #endif /* SQLITE_OMIT_WINDOWFUNC */ pTabList = p->pSrc; @@ -6788,11 +7099,11 @@ */ if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==JT_LEFT && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x1000,pParse,p, ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); assert( pItem->iCursor>=0 ); unsetJoinExpr(p->pWhere, pItem->iCursor, pTabList->a[0].fg.jointype & JT_LTORJ); @@ -6844,11 +7155,11 @@ && pSub->pLimit==0 /* Condition (1) */ && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */ && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); sqlite3ParserAddCleanup(pParse, (void(*)(sqlite3*,void*))sqlite3ExprListDelete, pSub->pOrderBy); pSub->pOrderBy = 0; @@ -6899,12 +7210,12 @@ ** procedure. */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x400,pParse,p,("end compound-select processing\n")); + if( (sqlite3TreeTrace & 0x400)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif if( p->pNext==0 ) ExplainQueryPlanPop(pParse); return rc; @@ -6920,17 +7231,17 @@ && p->pWhere->op==TK_AND && OptimizationEnabled(db, SQLITE_PropagateConst) && propagateConstants(pParse, p) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + if( sqlite3TreeTrace & 0x2000 ){ + TREETRACE(0x2000,pParse,p,("After constant propagation:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif }else{ - SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } #ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) @@ -6999,40 +7310,27 @@ && (pItem->fg.isCte==0 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem) ){ #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p, + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; /* Generate code to implement the subquery - ** - ** The subquery is implemented as a co-routine if all of the following are - ** true: - ** - ** (1) the subquery is guaranteed to be the outer loop (so that - ** it does not need to be computed more than once), and - ** (2) the subquery is not a CTE that should be materialized - ** (3) the subquery is not part of a left operand for a RIGHT JOIN */ - if( i==0 - && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) /* (1) */ - && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */ - && (pTabList->a[0].fg.jointype & JT_LTORJ)==0 /* (3) */ - ){ + if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; @@ -7059,11 +7357,11 @@ if( pItem->iCursor!=pCteUse->iCur ){ sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur); VdbeComment((v, "%!S", pItem)); } pSub->nSelectRow = pCteUse->nRowEst; - }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){ + }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in ** this same FROM clause. Reuse it. */ if( pPrior->addrFillSub ){ sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); } @@ -7073,10 +7371,13 @@ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of ** the same view can reuse the materialization. */ int topAddr; int onceAddr = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); pItem->addrFillSub = topAddr+1; pItem->fg.isMaterialized = 1; @@ -7088,19 +7389,18 @@ VdbeComment((v, "materialize %!S", pItem)); }else{ VdbeNoopComment((v, "materialize %!S", pItem)); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); - dest.zAffSdst = sqlite3TableAffinityStr(db, pItem->pTab); + + ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - sqlite3DbFree(db, dest.zAffSdst); - dest.zAffSdst = 0; pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; pCteUse->addrM9e = pItem->addrFillSub; @@ -7122,12 +7422,12 @@ pGroupBy = p->pGroupBy; pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + if( sqlite3TreeTrace & 0x8000 ){ + TREETRACE(0x8000,pParse,p,("After all FROM-clause analysis:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and @@ -7159,12 +7459,12 @@ ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); sDistinct.isTnct = 2; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + if( sqlite3TreeTrace & 0x20000 ){ + TREETRACE(0x20000,pParse,p,("Transform DISTINCT into GROUP BY:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif } @@ -7246,11 +7546,11 @@ #endif assert( WHERE_USE_LIMIT==SF_FixedLimit ); /* Begin the database scan. */ - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, p->pEList, p, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -7263,11 +7563,11 @@ sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ @@ -7302,11 +7602,11 @@ sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); /* End the database scan loop. */ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); } }else{ /* This case when there exist aggregate functions or a GROUP BY clause ** or both */ @@ -7383,16 +7683,18 @@ } if( db->mallocFailed ){ goto select_end; } pAggInfo->selId = p->selId; +#ifdef SQLITE_DEBUG + pAggInfo->pSelect = p; +#endif memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ @@ -7409,49 +7711,21 @@ if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){ minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pFExpr, &pMinMaxOrderBy); }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } - pAggInfo->mxReg = pParse->nMem; + analyzeAggFuncArgs(pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; #if TREETRACE_ENABLED - if( sqlite3TreeTrace & 0x400 ){ - int ii; - SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } - for(ii=0; iinColumn; ii++){ - struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; - sqlite3DebugPrintf( - "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" - " iSorterColumn=%d\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", - pCol->iTable, pCol->iColumn, pCol->iMem, - pCol->iSorterColumn); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } + printAggInfo(pAggInfo); } #endif /* Processing for aggregates with GROUP BY is very different and @@ -7516,21 +7790,25 @@ ** This might involve two separate loops with an OP_Sort in between, or ** it might be a single loop that uses an index to extract information ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); + } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo */ @@ -7575,19 +7853,36 @@ regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); pAggInfo->useSortingIdx = 1; } + + /* If there entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif + } /* If the index or temporary table used by the GROUP BY sort ** will naturally deliver rows in the order required by the ORDER BY ** clause, cancel the ephemeral table open coded earlier. ** @@ -7653,11 +7948,11 @@ */ if( groupBySort ){ sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } sqlite3ExprListDelete(db, pDistinct); @@ -7763,11 +8058,12 @@ /* Open a read-only cursor, execute the OP_Count, close the cursor. */ sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, (int)iRoot, iDb, 1); if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + assignAggregateRegisters(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; @@ -7799,10 +8095,11 @@ }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){ assert( ExprUseXList(pAggInfo->aFunc[0].pFExpr) ); pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ @@ -7815,17 +8112,17 @@ ** be an appropriate ORDER BY expression for the optimization. */ assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); eDist = sqlite3WhereIsDistinct(pWInfo); updateAccumulator(pParse, regAcc, pAggInfo, eDist); if( eDist!=WHERE_DISTINCT_NOOP ){ struct AggInfo_func *pF = pAggInfo->aFunc; if( pF ){ @@ -7835,11 +8132,11 @@ if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); } sSort.pOrderBy = 0; @@ -7857,12 +8154,10 @@ /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, - sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } /* Jump here to skip this query @@ -7882,11 +8177,11 @@ sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG if( pAggInfo && !db->mallocFailed ){ for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } for(i=0; inFunc; i++){ Expr *pExpr = pAggInfo->aFunc[i].pFExpr; @@ -7896,13 +8191,13 @@ } } #endif #if TREETRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end processing\n")); - if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + TREETRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3TreeTrace & 0x40000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif ExplainQueryPlanPop(pParse); return rc; } Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -19,11 +19,11 @@ typedef unsigned int u32; typedef unsigned short int u16; /* ** Optionally #include a user-defined header, whereby compilation options -** may be set prior to where they take effect, but after platform setup. +** may be set prior to where they take effect, but after platform setup. ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include ** file. Note that this macro has a like effect on sqlite3.c compilation. */ # define SHELL_STRINGIFY_(f) #f # define SHELL_STRINGIFY(f) SHELL_STRINGIFY_(f) @@ -465,12 +465,100 @@ /* ** Prompt strings. Initialized in main. Settable with ** .prompt main continue */ -static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ -static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ +#define PROMPT_LEN_MAX 20 +/* First line prompt. default: "sqlite> " */ +static char mainPrompt[PROMPT_LEN_MAX]; +/* Continuation prompt. default: " ...> " */ +static char continuePrompt[PROMPT_LEN_MAX]; + +/* +** Optionally disable dynamic continuation prompt. +** Unless disabled, the continuation prompt shows open SQL lexemes if any, +** or open parentheses level if non-zero, or continuation prompt as set. +** This facility interacts with the scanner and process_input() where the +** below 5 macros are used. +*/ +#ifdef SQLITE_OMIT_DYNAPROMPT +# define CONTINUATION_PROMPT continuePrompt +# define CONTINUE_PROMPT_RESET +# define CONTINUE_PROMPT_AWAITS(p,s) +# define CONTINUE_PROMPT_AWAITC(p,c) +# define CONTINUE_PAREN_INCR(p,n) +# define CONTINUE_PROMPT_PSTATE 0 +typedef void *t_NoDynaPrompt; +# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt +#else +# define CONTINUATION_PROMPT dynamicContinuePrompt() +# define CONTINUE_PROMPT_RESET \ + do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) +# define CONTINUE_PROMPT_AWAITS(p,s) \ + if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) +# define CONTINUE_PROMPT_AWAITC(p,c) \ + if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) +# define CONTINUE_PAREN_INCR(p,n) \ + if(p && stdin_is_interactive) (trackParenLevel(p,n)) +# define CONTINUE_PROMPT_PSTATE (&dynPrompt) +typedef struct DynaPrompt *t_DynaPromptRef; +# define SCAN_TRACKER_REFTYPE t_DynaPromptRef + +static struct DynaPrompt { + char dynamicPrompt[PROMPT_LEN_MAX]; + char acAwait[2]; + int inParenLevel; + char *zScannerAwaits; +} dynPrompt = { {0}, {0}, 0, 0 }; + +/* Record parenthesis nesting level change, or force level to 0. */ +static void trackParenLevel(struct DynaPrompt *p, int ni){ + p->inParenLevel += ni; + if( ni==0 ) p->inParenLevel = 0; + p->zScannerAwaits = 0; +} + +/* Record that a lexeme is opened, or closed with args==0. */ +static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ + if( s!=0 || c==0 ){ + p->zScannerAwaits = s; + p->acAwait[0] = 0; + }else{ + p->acAwait[0] = c; + p->zScannerAwaits = p->acAwait; + } +} + +/* Upon demand, derive the continuation prompt to display. */ +static char *dynamicContinuePrompt(void){ + if( continuePrompt[0]==0 + || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ + return continuePrompt; + }else{ + if( dynPrompt.zScannerAwaits ){ + size_t ncp = strlen(continuePrompt); + size_t ndp = strlen(dynPrompt.zScannerAwaits); + if( ndp > ncp-3 ) return continuePrompt; + strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; + strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + }else{ + if( dynPrompt.inParenLevel>9 ){ + strncpy(dynPrompt.dynamicPrompt, "(..", 4); + }else if( dynPrompt.inParenLevel<0 ){ + strncpy(dynPrompt.dynamicPrompt, ")x!", 4); + }else{ + strncpy(dynPrompt.dynamicPrompt, "(x.", 4); + dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + } + strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); + } + } + return dynPrompt.dynamicPrompt; +} +#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ /* ** Render output like fprintf(). Except, if the output is going to the ** console and if this is running on a Windows machine, translate the ** output from UTF-8 into MBCS. @@ -727,11 +815,11 @@ char *zPrompt; char *zResult; if( in!=0 ){ zResult = local_getline(zPrior, in); }else{ - zPrompt = isContinuation ? continuePrompt : mainPrompt; + zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; #if SHELL_USE_LOCAL_GETLINE printf("%s", zPrompt); fflush(stdout); zResult = local_getline(zPrior, stdin); #else @@ -944,11 +1032,11 @@ ){ const char *zName; char *zFake; UNUSED_PARAMETER(nVal); zName = (const char*)sqlite3_value_text(apVal[0]); - zFake = zName ? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; + zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; if( zFake ){ sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), -1, sqlite3_free); free(zFake); } @@ -1043,10 +1131,17 @@ #endif INCLUDE ../ext/misc/memtrace.c INCLUDE ../ext/misc/shathree.c INCLUDE ../ext/misc/uint.c INCLUDE ../ext/misc/decimal.c +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +INCLUDE ../ext/misc/base64.c +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#define OMIT_BASE85_CHECKER +INCLUDE ../ext/misc/base85.c INCLUDE ../ext/misc/ieee754.c INCLUDE ../ext/misc/series.c INCLUDE ../ext/misc/regexp.c #ifndef SQLITE_SHELL_FIDDLE INCLUDE ../ext/misc/fileio.c @@ -1064,13 +1159,18 @@ #define SQLITE_SHELL_HAVE_RECOVER 1 #else #define SQLITE_SHELL_HAVE_RECOVER 0 #endif #if SQLITE_SHELL_HAVE_RECOVER -INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h +# ifndef SQLITE_HAVE_SQLITE3R +INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.c +# endif +#endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) #endif #if defined(SQLITE_ENABLE_SESSION) /* ** State information for a single open session @@ -1494,11 +1594,11 @@ if( p[i]=='\r' && p[i+1]=='\n' ) i++; p[j++] = p[i]; } sz = j; p[sz] = 0; - } + } sqlite3_result_text64(context, (const char*)p, sz, sqlite3_free, SQLITE_UTF8); } p = 0; @@ -2075,11 +2175,11 @@ } /* ** Display and reset the EXPLAIN QUERY PLAN data */ -static void eqp_render(ShellState *p){ +static void eqp_render(ShellState *p, i64 nCycle){ EQPGraphRow *pRow = p->sGraph.pRow; if( pRow ){ if( pRow->zText[0]=='-' ){ if( pRow->pNext==0 ){ eqp_reset(p); @@ -2086,10 +2186,12 @@ return; } utf8_printf(p->out, "%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); + }else if( nCycle>0 ){ + utf8_printf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); }else{ utf8_printf(p->out, "QUERY PLAN\n"); } p->sGraph.zPrefix[0] = 0; eqp_render_level(p, 0); @@ -2667,13 +2769,13 @@ } zCode = sqlite3_mprintf("%.*s", len, zSql); shell_check_oom(zCode); for(i=0; zCode[i]; i++){ if( IsSpace(zSql[i]) ) zCode[i] = ' '; } if( iOffset<25 ){ - zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode, iOffset, ""); + zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode,iOffset,""); }else{ - zMsg = sqlite3_mprintf("\n %z\n %*serror here ---^", zCode, iOffset-14, ""); + zMsg = sqlite3_mprintf("\n %z\n %*serror here ---^", zCode,iOffset-14,""); } return zMsg; } @@ -2857,11 +2959,11 @@ } } if( pArg->statsOn==3 ){ if( pArg->pStmt ){ - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); raw_printf(pArg->out, "VM-steps: %d\n", iCur); } return 0; } @@ -2938,12 +3040,14 @@ raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); raw_printf(pArg->out, "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); - iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); - iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); + iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, + bReset); + iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, + bReset); if( iHit || iMiss ){ raw_printf(pArg->out, "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); @@ -2963,10 +3067,39 @@ /* Do not remove this machine readable comment: extra-stats-output-here */ return 0; } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif + /* ** Display scan stats. */ static void display_scanstats( sqlite3 *db, /* Database to query */ @@ -2974,44 +3107,81 @@ ){ #ifndef SQLITE_ENABLE_STMT_SCANSTATUS UNUSED_PARAMETER(db); UNUSED_PARAMETER(pArg); #else - int i, k, n, mx; - raw_printf(pArg->out, "-------- scanstats --------\n"); - mx = 0; - for(k=0; k<=mx; k++){ - double rEstLoop = 1.0; - for(i=n=0; 1; i++){ - sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nLoop, nVisit; - double rEst; - int iSid; - const char *zExplain; - if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ - break; - } - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); - if( iSid>mx ) mx = iSid; - if( iSid!=k ) continue; - if( n==0 ){ - rEstLoop = (double)nLoop; - if( k>0 ) raw_printf(pArg->out, "-------- subquery %d -------\n", k); - } - n++; - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); - rEstLoop *= rEst; - raw_printf(pArg->out, - " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *p = pArg->pStmt; + int ii = 0; + i64 nTotal = 0; + int nWidth = 0; + eqp_reset(pArg); + + for(ii=0; 1; ii++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = strlen(z) + scanStatsHeight(p, ii)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 4; + + sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(ii=0; 1; ii++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *z = 0; + const char *zName = 0; + char *zText = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + zText = sqlite3_mprintf("%s", z); + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + char *z = 0; + if( nCycle>=0 && nTotal>0 ){ + z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, + nCycle, ((nCycle*100)+nTotal/2) / nTotal + ); + } + if( nLoop>=0 ){ + z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); + } + if( nRow>=0 ){ + z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); + } + + if( zName && pArg->scanstatsOn>1 ){ + double rpl = (double)nRow / (double)nLoop; + z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); + } + + zText = sqlite3_mprintf( + "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z ); } + + eqp_append(pArg, iId, iPid, zText); + sqlite3_free(zText); } - raw_printf(pArg->out, "---------------------------\n"); + + eqp_render(pArg, nTotal); #endif } /* ** Parameter azArray points to a zero-terminated array of strings. zStr @@ -3247,11 +3417,11 @@ /* Draw horizontal line N characters long using unicode box ** characters */ static void print_box_line(FILE *out, int N){ - const char zDash[] = + const char zDash[] = BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; const int nDash = sizeof(zDash) - 1; N *= 3; while( N>nDash ){ @@ -3376,11 +3546,11 @@ continue; } break; } zOut[j] = 0; - return (char*)zOut; + return (char*)zOut; } /* Extract the value of the i-th current column for pStmt as an SQL literal ** value. Memory is obtained from sqlite3_malloc64() and must be freed by ** the caller. @@ -3737,22 +3907,22 @@ ** code. In this case, (*pzErr) may be set to point to a buffer containing ** an English language error message. It is the responsibility of the ** caller to eventually free this buffer using sqlite3_free(). */ static int expertHandleSQL( - ShellState *pState, - const char *zSql, + ShellState *pState, + const char *zSql, char **pzErr ){ assert( pState->expert.pExpert ); assert( pzErr==0 || *pzErr==0 ); return sqlite3_expert_sql(pState->expert.pExpert, zSql, pzErr); } /* ** This function is called either to silently clean up the object -** created by the ".expert" command (if bCancel==1), or to generate a +** created by the ".expert" command (if bCancel==1), or to generate a ** report from it and then clean it up (if bCancel==0). ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error ** code. In this case, (*pzErr) may be set to point to a buffer containing ** an English language error message. It is the responsibility of the @@ -3843,11 +4013,12 @@ } if( rc==SQLITE_OK ){ pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); if( pState->expert.pExpert==0 ){ - raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory"); + raw_printf(stderr, "sqlite3_expert_new: %s\n", + zErr ? zErr : "out of memory"); rc = SQLITE_ERROR; }else{ sqlite3_expert_config( pState->expert.pExpert, EXPERT_CONFIG_SAMPLE, iSample ); @@ -3931,14 +4102,14 @@ while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); int iEqpId = sqlite3_column_int(pExplain, 0); int iParentId = sqlite3_column_int(pExplain, 1); if( zEQPLine==0 ) zEQPLine = ""; - if( zEQPLine[0]=='-' ) eqp_render(pArg); + if( zEQPLine[0]=='-' ) eqp_render(pArg, 0); eqp_append(pArg, iEqpId, iParentId, zEQPLine); } - eqp_render(pArg); + eqp_render(pArg, 0); } sqlite3_finalize(pExplain); sqlite3_free(zEQP); if( pArg->autoEQP>=AUTOEQP_full ){ /* Also do an EXPLAIN for ".eqp full" mode */ @@ -3983,11 +4154,11 @@ } bind_prepared_stmt(pArg, pStmt); exec_prepared_stmt(pArg, pStmt); explain_data_delete(pArg); - eqp_render(pArg); + eqp_render(pArg, 0); /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ display_stats(db, pArg, 0); } @@ -4523,11 +4694,11 @@ #endif #ifndef SQLITE_SHELL_FIDDLE ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save ?OPTIONS? FILE Write database to FILE (an alias for .backup ...)", #endif - ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", + ".scanstats on|off|est Turn sqlite3_stmt_scanstatus() metrics on or off", ".schema ?PATTERN? Show the CREATE statements matching PATTERN", " Options:", " --indent Try to pretty-print the schema", " --nosys Omit objects whose names start with \"sqlite_\"", ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", @@ -4804,11 +4975,11 @@ }else if( n==0 && dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ rc = SHELL_OPEN_ZIPFILE; } } fclose(f); - return rc; + return rc; } #ifndef SQLITE_OMIT_DESERIALIZE /* ** Reconstruct an in-memory database using the output from the "dbtotxt" @@ -4903,12 +5074,12 @@ ** must be a blob. The second a non-negative integer. This function ** reads and returns a 32-bit big-endian integer from byte ** offset (4*) of the blob. */ static void shellInt32( - sqlite3_context *context, - int argc, + sqlite3_context *context, + int argc, sqlite3_value **argv ){ const unsigned char *pBlob; int nBlob; int iInt; @@ -4931,12 +5102,12 @@ /* ** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, ** using "..." with internal double-quote characters doubled. */ static void shellIdQuote( - sqlite3_context *context, - int argc, + sqlite3_context *context, + int argc, sqlite3_value **argv ){ const char *zName = (const char*)sqlite3_value_text(argv[0]); UNUSED_PARAMETER(argc); if( zName ){ @@ -4947,12 +5118,12 @@ /* ** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. */ static void shellUSleepFunc( - sqlite3_context *context, - int argcUnused, + sqlite3_context *context, + int argcUnused, sqlite3_value **argv ){ int sleep = sqlite3_value_int(argv[0]); (void)argcUnused; sqlite3_sleep(sleep/1000); @@ -4960,11 +5131,11 @@ } /* ** Scalar function "shell_escape_crnl" used by the .recover command. ** The argument passed to this function is the output of built-in -** function quote(). If the first character of the input is "'", +** function quote(). If the first character of the input is "'", ** indicating that the value passed to quote() was a text value, ** then this function searches the input for "\n" and "\r" characters ** and adds a wrapper similar to the following: ** ** replace(replace(, '\n', char(10), '\r', char(13)); @@ -4971,12 +5142,12 @@ ** ** Or, if the first character of the input is not "'", then a copy ** of the input is returned. */ static void shellEscapeCrnl( - sqlite3_context *context, - int argc, + sqlite3_context *context, + int argc, sqlite3_value **argv ){ const char *zText = (const char*)sqlite3_value_text(argv[0]); UNUSED_PARAMETER(argc); if( zText && zText[0]=='\'' ){ @@ -5072,17 +5243,17 @@ const char *zDbFilename = p->pAuxDb->zDbFilename; if( p->openMode==SHELL_OPEN_UNSPEC ){ if( zDbFilename==0 || zDbFilename[0]==0 ){ p->openMode = SHELL_OPEN_NORMAL; }else{ - p->openMode = (u8)deduceDatabaseType(zDbFilename, + p->openMode = (u8)deduceDatabaseType(zDbFilename, (openFlags & OPEN_DB_ZIPFILE)!=0); } } switch( p->openMode ){ case SHELL_OPEN_APPENDVFS: { - sqlite3_open_v2(zDbFilename, &p->db, + sqlite3_open_v2(zDbFilename, &p->db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs"); break; } case SHELL_OPEN_HEXDB: case SHELL_OPEN_DESERIALIZE: { @@ -5113,16 +5284,19 @@ sqlite3_open(":memory:", &p->db); return; } exit(1); } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif sqlite3_shathree_init(p->db, 0, 0); sqlite3_uint_init(p->db, 0, 0); sqlite3_decimal_init(p->db, 0, 0); + sqlite3_base64_init(p->db, 0, 0); + sqlite3_base85_init(p->db, 0, 0); sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); #ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); @@ -5135,10 +5309,38 @@ if( !p->bSafeModePersist ){ sqlite3_zipfile_init(p->db, 0, 0); sqlite3_sqlar_init(p->db, 0, 0); } #endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * inititialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, @@ -5155,10 +5357,11 @@ sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename); shell_check_oom(zSql); sqlite3_exec(p->db, zSql, 0, 0, 0); @@ -5201,11 +5404,11 @@ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); if( rc ){ utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db)); - } + } } #if HAVE_READLINE || HAVE_EDITLINE /* ** Readline completion callbacks @@ -5231,10 +5434,12 @@ zRet = 0; } return zRet; } static char **readline_completion(const char *zText, int iStart, int iEnd){ + (void)iStart; + (void)iEnd; rl_attempted_completion_over = 1; return rl_completion_matches(zText, readline_completion_generator); } #elif HAVE_LINENOISE @@ -6442,20 +6647,20 @@ return SQLITE_ERROR; } #if !defined SQLITE_OMIT_VIRTUALTABLE static void shellPrepare( - sqlite3 *db, - int *pRc, - const char *zSql, + sqlite3 *db, + int *pRc, + const char *zSql, sqlite3_stmt **ppStmt ){ *ppStmt = 0; if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - raw_printf(stderr, "sql error: %s (%d)\n", + raw_printf(stderr, "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db) ); *pRc = rc; } } @@ -6467,14 +6672,14 @@ ** This routine is could be marked "static". But it is not always used, ** depending on compile-time options. By omitting the "static", we avoid ** nuisance compiler warnings about "defined but not used". */ void shellPreparePrintf( - sqlite3 *db, - int *pRc, + sqlite3 *db, + int *pRc, sqlite3_stmt **ppStmt, - const char *zFmt, + const char *zFmt, ... ){ *ppStmt = 0; if( *pRc==SQLITE_OK ){ va_list ap; @@ -6496,11 +6701,11 @@ ** This routine is could be marked "static". But it is not always used, ** depending on compile-time options. By omitting the "static", we avoid ** nuisance compiler warnings about "defined but not used". */ void shellFinalize( - int *pRc, + int *pRc, sqlite3_stmt *pStmt ){ if( pStmt ){ sqlite3 *db = sqlite3_db_handle(pStmt); int rc = sqlite3_finalize(pStmt); @@ -6518,11 +6723,11 @@ ** This routine is could be marked "static". But it is not always used, ** depending on compile-time options. By omitting the "static", we avoid ** nuisance compiler warnings about "defined but not used". */ void shellReset( - int *pRc, + int *pRc, sqlite3_stmt *pStmt ){ int rc = sqlite3_reset(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ @@ -6566,11 +6771,11 @@ showHelp(f,"archive"); return SQLITE_ERROR; } /* -** Print an error message for the .ar command to stderr and return +** Print an error message for the .ar command to stderr and return ** SQLITE_ERROR. */ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_list ap; char *z; @@ -6647,11 +6852,11 @@ } /* ** Parse the command line for an ".ar" command. The results are written into ** structure (*pAr). SQLITE_OK is returned if the command line is parsed -** successfully, otherwise an error message is written to stderr and +** successfully, otherwise an error message is written to stderr and ** SQLITE_ERROR returned. */ static int arParseCommand( char **azArg, /* Array of arguments passed to dot command */ int nArg, /* Number of entries in azArg[] */ @@ -6843,11 +7048,11 @@ ** The caller is responsible for eventually calling sqlite3_free() on ** any non-NULL (*pzWhere) value. Here, "match" means strict equality ** when pAr->bGlob is false and GLOB match when pAr->bGlob is true. */ static void arWhereClause( - int *pRc, + int *pRc, ArCommand *pAr, char **pzWhere /* OUT: New WHERE clause */ ){ char *zWhere = 0; const char *zSameOp = (pAr->bGlob)? "GLOB" : "="; @@ -6858,11 +7063,11 @@ int i; const char *zSep = ""; for(i=0; inArg; i++){ const char *z = pAr->azArg[i]; zWhere = sqlite3_mprintf( - "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'", + "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'", zWhere, zSep, zSameOp, z, strlen30(z)+1, zSameOp, z ); if( zWhere==0 ){ *pRc = SQLITE_NOMEM; break; @@ -6873,14 +7078,14 @@ } *pzWhere = zWhere; } /* -** Implementation of .ar "lisT" command. +** Implementation of .ar "lisT" command. */ static int arListCommand(ArCommand *pAr){ - const char *zSql = "SELECT %s FROM %s WHERE %s"; + const char *zSql = "SELECT %s FROM %s WHERE %s"; const char *azCols[] = { "name", "lsmode(mode), sz, datetime(mtime, 'unixepoch'), name" }; @@ -6898,11 +7103,11 @@ }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ utf8_printf(pAr->p->out, "%s % 10d %s %s\n", sqlite3_column_text(pSql, 0), - sqlite3_column_int(pSql, 1), + sqlite3_column_int(pSql, 1), sqlite3_column_text(pSql, 2), sqlite3_column_text(pSql, 3) ); }else{ utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); @@ -6955,21 +7160,21 @@ sqlite3_free(zSql); return rc; } /* -** Implementation of .ar "eXtract" command. +** Implementation of .ar "eXtract" command. */ static int arExtractCommand(ArCommand *pAr){ - const char *zSql1 = + const char *zSql1 = "SELECT " " ($dir || name)," " writefile(($dir || name), %s, mode, mtime) " "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" " AND name NOT GLOB '*..[/\\]*'"; - const char *azExtraArg[] = { + const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", "data" }; sqlite3_stmt *pSql = 0; @@ -6991,11 +7196,11 @@ zDir = sqlite3_mprintf(""); } if( zDir==0 ) rc = SQLITE_NOMEM; } - shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, + shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, azExtraArg[pAr->bZip], pAr->zSrcTable, zWhere ); if( rc==SQLITE_OK ){ j = sqlite3_bind_parameter_index(pSql, "$dir"); @@ -7069,11 +7274,11 @@ static int arCreateOrUpdateCommand( ArCommand *pAr, /* Command arguments and options */ int bUpdate, /* true for a --create. */ int bOnlyIfChanged /* Only update if file has changed */ ){ - const char *zCreate = + const char *zCreate = "CREATE TABLE IF NOT EXISTS sqlar(\n" " name TEXT PRIMARY KEY, -- name of the file\n" " mode INT, -- access permissions\n" " mtime INT, -- last modification time\n" " sz INT, -- original file size\n" @@ -7111,11 +7316,11 @@ char *zExists = 0; arExecSql(pAr, "PRAGMA page_size=512"); rc = arExecSql(pAr, "SAVEPOINT ar;"); if( rc!=SQLITE_OK ) return rc; - zTemp[0] = 0; + zTemp[0] = 0; if( pAr->bZip ){ /* Initialize the zipfile virtual table, if necessary */ if( pAr->zFile ){ sqlite3_uint64 r; sqlite3_randomness(sizeof(r),&r); @@ -7205,11 +7410,11 @@ } cmd.bZip = 1; }else if( cmd.zFile ){ int flags; if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS; - if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT + if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT || cmd.eCmd==AR_CMD_REMOVE || cmd.eCmd==AR_CMD_UPDATE ){ flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; }else{ flags = SQLITE_OPEN_READONLY; } @@ -7216,14 +7421,14 @@ cmd.db = 0; if( cmd.bDryRun ){ utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile, eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } - rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, + rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "cannot open file: %s (%s)\n", + utf8_printf(stderr, "cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db) ); goto end_ar_command; } sqlite3_fileio_init(cmd.db, 0, 0); @@ -7335,11 +7540,11 @@ }else if( n<=10 && memcmp("-no-rowids", z, n)==0 ){ bRowids = 0; } else{ - utf8_printf(stderr, "unexpected option: %s\n", azArg[i]); + utf8_printf(stderr, "unexpected option: %s\n", azArg[i]); showHelp(pState->out, azArg[0]); return 1; } } @@ -7698,11 +7903,11 @@ if( zDestFile==0 ){ raw_printf(stderr, "missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; - rc = sqlite3_open_v2(zDestFile, &pDest, + rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; @@ -7948,11 +8153,11 @@ if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]); utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n"); - } + } }else #if SQLITE_SHELL_HAVE_RECOVER if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbinfo", n)==0 ){ rc = shell_dbinfo_command(p, nArg, azArg); @@ -8015,11 +8220,11 @@ " name LIKE %Q ESCAPE '\\' AND" " sql LIKE 'CREATE VIRTUAL TABLE%%' AND" " substr(o.name, 1, length(name)+1) == (name||'_')" ")", azArg[i], azArg[i] ); - + if( zLike ){ zLike = sqlite3_mprintf("%z OR %z", zLike, zExpr); }else{ zLike = zExpr; } @@ -8148,11 +8353,11 @@ }else #ifndef SQLITE_OMIT_VIRTUALTABLE if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ - raw_printf(stderr, + raw_printf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", azArg[0]); rc = 1; }else{ open_db(p, 0); @@ -8167,11 +8372,11 @@ int ctrlCode; /* Integer code for that option */ const char *zUsage; /* Usage notes */ } aCtrl[] = { { "chunk_size", SQLITE_FCNTL_CHUNK_SIZE, "SIZE" }, { "data_version", SQLITE_FCNTL_DATA_VERSION, "" }, - { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, + { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, { "persist_wal", SQLITE_FCNTL_PERSIST_WAL, "[BOOLEAN]" }, /* { "pragma", SQLITE_FCNTL_PRAGMA, "NAME ARG" },*/ { "psow", SQLITE_FCNTL_POWERSAFE_OVERWRITE, "[BOOLEAN]" }, { "reserve_bytes", SQLITE_FCNTL_RESERVE_BYTES, "[N]" }, @@ -8188,11 +8393,11 @@ const char *zSchema = 0; open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; - if( zCmd[0]=='-' + if( zCmd[0]=='-' && (cli_strcmp(zCmd,"--schema")==0 || cli_strcmp(zCmd,"-schema")==0) && nArg>=4 ){ zSchema = azArg[2]; for(i=3; iscanstatsOn = (u8)booleanValue(azArg[1]); + if( cli_strcmp(azArg[1], "est")==0 ){ + p->scanstatsOn = 2; + }else{ + p->scanstatsOn = (u8)booleanValue(azArg[1]); + } #ifndef SQLITE_ENABLE_STMT_SCANSTATUS raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); #endif }else{ - raw_printf(stderr, "Usage: .scanstats on|off\n"); + raw_printf(stderr, "Usage: .scanstats on|off|est\n"); rc = 1; } }else if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ @@ -9562,11 +9771,12 @@ rc = 1; goto meta_command_exit; }else if( zName==0 ){ zName = azArg[ii]; }else{ - raw_printf(stderr, "Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n"); + raw_printf(stderr, + "Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; } } if( zName!=0 ){ @@ -9678,11 +9888,11 @@ }else if( (c=='s' && n==11 && cli_strncmp(azArg[0], "selecttrace", n)==0) || (c=='t' && n==9 && cli_strncmp(azArg[0], "treetrace", n)==0) ){ - unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + unsigned int x = nArg>=2? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x); }else #if defined(SQLITE_ENABLE_SESSION) if( c=='s' && cli_strncmp(azArg[0],"session",n)==0 && n>=3 ){ @@ -9863,11 +10073,12 @@ utf8_printf(stderr, "Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); + raw_printf(stderr, + "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ @@ -10077,16 +10288,16 @@ bSeparate = 1; if( sqlite3_strlike("sqlite\\_%", zLike, '\\')==0 ) bSchema = 1; } } if( bSchema ){ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " UNION ALL SELECT 'sqlite_schema'" " ORDER BY 1 collate nocase"; }else{ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " AND name NOT LIKE 'sqlite_%'" " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); @@ -10143,10 +10354,62 @@ if( bDebug ){ utf8_printf(p->out, "%s\n", zSql); }else{ shell_exec(p, zSql, 0); } +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && !defined(SQLITE_OMIT_VIRTUALTABLE) + { + int lrc; + char *zRevText = /* Query for reversible to-blob-to-text check */ + "SELECT lower(name) as tname FROM sqlite_schema\n" + "WHERE type='table' AND coalesce(rootpage,0)>1\n" + "AND name NOT LIKE 'sqlite_%%'%s\n" + "ORDER BY 1 collate nocase"; + zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with tabcols as materialized(\n" + "select tname, cname\n" + "from (" + " select ss.tname as tname, ti.name as cname\n" + " from (%z) ss\n inner join pragma_table_info(tname) ti))\n" + "select 'SELECT total(bad_text_count) AS bad_text_count\n" + "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n" + " from (select 'SELECT COUNT(*) AS bad_text_count\n" + "FROM '||tname||' WHERE '\n" + "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n" + "|| ' AND typeof('||cname||')=''text'' ',\n" + "' OR ') as query, tname from tabcols group by tname)" + , zRevText); + shell_check_oom(zRevText); + if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + assert(lrc==SQLITE_OK); + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); + lrc = SQLITE_ROW==sqlite3_step(pStmt); + if( lrc ){ + const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); + sqlite3_stmt *pCheckStmt; + lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); + if( SQLITE_OK==lrc ){ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int sz = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + sz, (sz>1)? "s": ""); + } + } + sqlite3_finalize(pCheckStmt); + } + sqlite3_finalize(pStmt); + } + sqlite3_free(zRevText); + } +#endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ sqlite3_free(zSql); }else #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) if( c=='s' @@ -10372,32 +10635,32 @@ const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ int unSafe; /* Not valid for --safe mode */ const char *zUsage; /* Usage notes */ } aCtrl[] = { - { "always", SQLITE_TESTCTRL_ALWAYS, 1, "BOOLEAN" }, - { "assert", SQLITE_TESTCTRL_ASSERT, 1, "BOOLEAN" }, - /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, "" },*/ - /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ - { "byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, - { "extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, - /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ - { "imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, - { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, - { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, - { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, - { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, + {"always", SQLITE_TESTCTRL_ALWAYS, 1, "BOOLEAN" }, + {"assert", SQLITE_TESTCTRL_ASSERT, 1, "BOOLEAN" }, + /*{"benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, "" },*/ + /*{"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ + {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, + {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, + /*{"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ + {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, + {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, + {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, + {"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, + {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, #ifdef YYCOVERAGE - { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE,0,"" }, + {"parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE,0,"" }, #endif - { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE,0, "OFFSET " }, - { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE,0, "" }, - { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, 0, "" }, - { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, 0, "SEED ?db?" }, - { "seek_count", SQLITE_TESTCTRL_SEEK_COUNT, 0, "" }, - { "sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, - { "tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, + {"pending_byte", SQLITE_TESTCTRL_PENDING_BYTE,0, "OFFSET " }, + {"prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE,0, "" }, + {"prng_save", SQLITE_TESTCTRL_PRNG_SAVE, 0, "" }, + {"prng_seed", SQLITE_TESTCTRL_PRNG_SEED, 0, "SEED ?db?" }, + {"seek_count", SQLITE_TESTCTRL_SEEK_COUNT, 0, "" }, + {"sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, + {"tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, }; int testctrl = -1; int iCtrl = -1; int rc2 = 0; /* 0: usage. 1: %d 2: %x 3: no-output */ int isOk = 0; @@ -10654,11 +10917,11 @@ rc = 1; goto meta_command_exit; } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); + p->traceOut = output_file_open(z, 0); } } if( p->traceOut==0 ){ sqlite3_trace_v2(p->db, 0, 0, 0); }else{ @@ -10818,11 +11081,11 @@ } } }else if( c=='w' && cli_strncmp(azArg[0], "wheretrace", n)==0 ){ - unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + unsigned int x = nArg>=2? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x); }else if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; @@ -10870,11 +11133,12 @@ /* ** Scan line for classification to guide shell's handling. ** The scan is resumable for subsequent lines when prior ** return values are passed as the 2nd argument. */ -static QuickScanState quickscan(char *zLine, QuickScanState qss){ +static QuickScanState quickscan(char *zLine, QuickScanState qss, + SCAN_TRACKER_REFTYPE pst){ char cin; char cWait = (char)qss; /* intentional narrowing loss */ if( cWait==0 ){ PlainScan: assert( cWait==0 ); @@ -10894,10 +11158,11 @@ continue; case '/': if( *zLine=='*' ){ ++zLine; cWait = '*'; + CONTINUE_PROMPT_AWAITS(pst, "/*"); qss = QSS_SETV(qss, cWait); goto TermScan; } break; case '[': @@ -10904,11 +11169,18 @@ cin = ']'; /* fall thru */ case '`': case '\'': case '"': cWait = cin; qss = QSS_HasDark | cWait; + CONTINUE_PROMPT_AWAITC(pst, cin); goto TermScan; + case '(': + CONTINUE_PAREN_INCR(pst, 1); + break; + case ')': + CONTINUE_PAREN_INCR(pst, -1); + break; default: break; } qss = (qss & ~QSS_EndingSemi) | QSS_HasDark; } @@ -10920,20 +11192,23 @@ case '*': if( *zLine != '/' ) continue; ++zLine; cWait = 0; + CONTINUE_PROMPT_AWAITC(pst, 0); qss = QSS_SETV(qss, 0); goto PlainScan; case '`': case '\'': case '"': if(*zLine==cWait){ + /* Swallow doubled end-delimiter.*/ ++zLine; continue; } /* fall thru */ case ']': cWait = 0; + CONTINUE_PROMPT_AWAITC(pst, 0); qss = QSS_SETV(qss, 0); goto PlainScan; default: assert(0); } } @@ -10953,21 +11228,19 @@ zLine += 1; /* Oracle */ else if ( ToLower(zLine[0])=='g' && ToLower(zLine[1])=='o' ) zLine += 2; /* SQL Server */ else return 0; - return quickscan(zLine, QSS_Start)==QSS_Start; + return quickscan(zLine, QSS_Start, 0)==QSS_Start; } /* -** We need a default sqlite3_complete() implementation to use in case -** the shell is compiled with SQLITE_OMIT_COMPLETE. The default assumes -** any arbitrary text is a complete SQL statement. This is not very -** user-friendly, but it does seem to work. +** The CLI needs a working sqlite3_complete() to work properly. So error +** out of the build if compiling with SQLITE_OMIT_COMPLETE. */ #ifdef SQLITE_OMIT_COMPLETE -#define sqlite3_complete(x) 1 +# error the CLI application is imcompatable with SQLITE_OMIT_COMPLETE. #endif /* ** Return true if zSql is a complete SQL statement. Return false if it ** ends in the middle of a string literal or C-style comment. @@ -11036,13 +11309,13 @@ if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo); } #ifdef SQLITE_SHELL_FIDDLE /* -** Alternate one_input_line() impl for wasm mode. This is not in the primary impl -** because we need the global shellState and cannot access it from that function -** without moving lots of code around (creating a larger/messier diff). +** Alternate one_input_line() impl for wasm mode. This is not in the primary +** impl because we need the global shellState and cannot access it from that +** function without moving lots of code around (creating a larger/messier diff). */ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ /* Parse the next line from shellState.wasm.zInput. */ const char *zBegin = shellState.wasm.zPos; const char *z = zBegin; @@ -11095,10 +11368,11 @@ " Check recursion.\n", MAX_INPUT_NESTING, p->lineno); return 1; } ++p->inputNesting; p->lineno = 0; + CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); zLine = one_input_line(p->in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ @@ -11113,18 +11387,19 @@ if( QSS_INPLAIN(qss) && line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){ memcpy(zLine,";",2); } - qss = quickscan(zLine, qss); + qss = quickscan(zLine, qss, CONTINUE_PROMPT_PSTATE); if( QSS_PLAINWHITE(qss) && nSql==0 ){ /* Just swallow single-line whitespace */ echo_group_input(p, zLine); qss = QSS_Start; continue; } if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ + CONTINUE_PROMPT_RESET; echo_group_input(p, zLine); if( zLine[0]=='.' ){ rc = do_meta_command(zLine, p); if( rc==2 ){ /* exit requested */ break; @@ -11156,10 +11431,11 @@ nSql += nLine; } if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); + CONTINUE_PROMPT_RESET; nSql = 0; if( p->outCount ){ output_reset(p); p->outCount = 0; }else{ @@ -11175,10 +11451,11 @@ } if( nSql ){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); + CONTINUE_PROMPT_RESET; } free(zSql); free(zLine); --p->inputNesting; return errCnt>0; @@ -11250,14 +11527,48 @@ home_dir = z; } return home_dir; } + +/* +** On non-Windows platforms, look for $XDG_CONFIG_HOME. +** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return +** the path to it, else return 0. The result is cached for +** subsequent calls. +*/ +static const char *find_xdg_config(void){ +#if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \ + || defined(__RTP__) || defined(_WRS_KERNEL) + return 0; +#else + static int alreadyTried = 0; + static char *zConfig = 0; + const char *zXdgHome; + + if( alreadyTried!=0 ){ + return zConfig; + } + alreadyTried = 1; + zXdgHome = getenv("XDG_CONFIG_HOME"); + if( zXdgHome==0 ){ + return 0; + } + zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); + shell_check_oom(zConfig); + if( access(zConfig,0)!=0 ){ + sqlite3_free(zConfig); + zConfig = 0; + } + return zConfig; +#endif +} /* ** Read input from the file given by sqliterc_override. Or if that -** parameter is NULL, take input from ~/.sqliterc +** parameter is NULL, take input from the first of find_xdg_config() +** or ~/.sqliterc which is found. ** ** Returns the number of errors. */ static void process_sqliterc( ShellState *p, /* Configuration data */ @@ -11267,11 +11578,14 @@ const char *sqliterc = sqliterc_override; char *zBuf = 0; FILE *inSaved = p->in; int savedLineno = p->lineno; - if (sqliterc == NULL) { + if( sqliterc == NULL ){ + sqliterc = find_xdg_config(); + } + if( sqliterc == NULL ){ home_dir = find_home_dir(0); if( home_dir==0 ){ raw_printf(stderr, "-- warning: cannot find home directory;" " cannot read ~/.sqliterc\n"); return; @@ -11816,16 +12130,16 @@ data.openMode = SHELL_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; }else if( cli_strcmp(z,"-ascii")==0 ){ data.mode = MODE_Ascii; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Unit); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Record); + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit); + sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Record); }else if( cli_strcmp(z,"-tabs")==0 ){ data.mode = MODE_List; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Tab); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row); + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Tab); + sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Row); }else if( cli_strcmp(z,"-separator")==0 ){ sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, "%s",cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-newline")==0 ){ sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -561,10 +561,11 @@ #define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) #define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) #define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) #define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ @@ -2182,11 +2183,11 @@ ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally ** rounded down to the next smaller multiple of 8. ^(The lookaside memory ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words ** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^ ** ** [[SQLITE_DBCONFIG_ENABLE_FKEY]] @@ -2332,12 +2333,16 @@ **
  • sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); **
  • [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); **
  • sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); ** ** Because resetting a database is destructive and irreversible, the -** process requires the use of this obscure API and multiple steps to help -** ensure that it does not happen by accident. +** process requires the use of this obscure API and multiple steps to +** help ensure that it does not happen by accident. Because this +** feature must be capable of resetting corrupt databases, and +** shutting down virtual tables may require access to that corrupt +** storage, the library must abandon any installed virtual tables +** without calling their xDestroy() methods. ** ** [[SQLITE_DBCONFIG_DEFENSIVE]]
    SQLITE_DBCONFIG_DEFENSIVE
    **
    The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the ** "defensive" flag for a database connection. When the defensive ** flag is enabled, language features that allow ordinary SQL to @@ -3281,27 +3286,27 @@ **
    ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement ** first begins running and possibly at other times during the ** execution of the prepared statement, such as at the start of each ** trigger subprogram. ^The P argument is a pointer to the ** [prepared statement]. ^The X argument is a pointer to a string which -** is the unexpanded SQL text of the prepared statement or an SQL comment +** is the unexpanded SQL text of the prepared statement or an SQL comment ** that indicates the invocation of a trigger. ^The callback can compute ** the same text that would have been returned by the legacy [sqlite3_trace()] ** interface by using the X argument when X begins with "--" and invoking ** [sqlite3_expanded_sql(P)] otherwise. ** ** [[SQLITE_TRACE_PROFILE]]
    SQLITE_TRACE_PROFILE
    **
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same ** information as is provided by the [sqlite3_profile()] callback. ** ^The P argument is a pointer to the [prepared statement] and the -** X argument points to a 64-bit integer which is the estimated of -** the number of nanosecond that the prepared statement took to run. +** X argument points to a 64-bit integer which is approximately +** the number of nanoseconds that the prepared statement took to run. ** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. ** ** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    **
    ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared -** statement generates a single row of result. +** statement generates a single row of result. ** ^The P argument is a pointer to the [prepared statement] and the ** X argument is unused. ** ** [[SQLITE_TRACE_CLOSE]]
    SQLITE_TRACE_CLOSE
    **
    ^An SQLITE_TRACE_CLOSE callback is invoked when a database @@ -5547,20 +5552,10 @@ ** such a conversion is possible without loss of information (in other ** words, if the value is a string that looks like a number) ** then the conversion is performed. Otherwise no conversion occurs. ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** -** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], -** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding -** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) -** returns something other than SQLITE_TEXT, then the return value from -** sqlite3_value_encoding(X) is meaningless. ^Calls to -** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X), -** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or -** sqlite3_value_bytes16(X) might change the encoding of the value X and -** thus change the return from subsequent calls to sqlite3_value_encoding(X). -** ** ^Within the [xUpdate] method of a [virtual table], the ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if ** and the prior [xColumn] method call that was invoked to extracted @@ -5621,10 +5616,31 @@ int sqlite3_value_bytes16(sqlite3_value*); int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); int sqlite3_value_nochange(sqlite3_value*); int sqlite3_value_frombind(sqlite3_value*); + +/* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ int sqlite3_value_encoding(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values ** METHOD: sqlite3_value @@ -7001,19 +7017,10 @@ ** ^This interface disables all automatic extensions previously ** registered using [sqlite3_auto_extension()]. */ void sqlite3_reset_auto_extension(void); -/* -** The interface to the virtual-table mechanism is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** Structures used by the virtual table interface */ typedef struct sqlite3_vtab sqlite3_vtab; typedef struct sqlite3_index_info sqlite3_index_info; @@ -7251,11 +7258,11 @@ ** ** The collating sequence to be used for comparison can be found using ** the [sqlite3_vtab_collation()] interface. For most real-world virtual ** tables, the collating sequence of constraints does not matter (for example ** because the constraints are numeric) and so the sqlite3_vtab_collation() -** interface is no commonly needed. +** interface is not commonly needed. */ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 #define SQLITE_INDEX_CONSTRAINT_LE 8 #define SQLITE_INDEX_CONSTRAINT_LT 16 @@ -7410,20 +7417,10 @@ ** purpose is to be a placeholder function that can be overloaded ** by a [virtual table]. */ int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); -/* -** The interface to the virtual-table mechanism defined above (back up -** to a comment remarkably similar to this one) is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** CAPI3REF: A Handle To An Open BLOB ** KEYWORDS: {BLOB handle} {BLOB handles} ** ** An instance of this object represents an open BLOB on which @@ -9623,11 +9620,11 @@ ** statement that was passed into [sqlite3_declare_vtab()], then the ** name of that alternative collating sequence is returned. **

  • Otherwise, "BINARY" is returned. ** */ -SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); +const char *sqlite3_vtab_collation(sqlite3_index_info*,int); /* ** CAPI3REF: Determine if a virtual table query is DISTINCT ** METHOD: sqlite3_index_info ** @@ -9892,10 +9889,14 @@ ** ** When the value returned to V is a string, space to hold that string is ** managed by the prepared statement S and will be automatically freed when ** S is finalized. ** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** **

    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    **
    ^The [sqlite3_int64] variable pointed to by the V parameter will be ** set to the total number of times that the X-th loop has run.
    ** @@ -9919,30 +9920,44 @@ ** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    **
    ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** -** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECTID
    **
    ^The "int" variable pointed to by the V parameter will be set to the -** "select-id" for the X-th loop. The select-id identifies which query or -** subquery the loop is part of. The main query has a select-id of zero. -** The select-id is the same value as is output in the first column -** of an [EXPLAIN QUERY PLAN] query. +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. **
    +** +** [[SQLITE_SCANSTAT_PARENTID]]
    SQLITE_SCANSTAT_PARENTID
    +**
    The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]]
    SQLITE_SCANSTAT_NCYCLE
    +**
    The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. */ #define SQLITE_SCANSTAT_NLOOP 0 #define SQLITE_SCANSTAT_NVISIT 1 #define SQLITE_SCANSTAT_EST 2 #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 #define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 /* ** CAPI3REF: Prepared Statement Scan Status ** METHOD: sqlite3_stmt ** -** This interface returns information about the predicted and measured +** These interfaces return information about the predicted and measured ** performance for pStmt. Advanced applications can use this ** interface to compare the predicted and the measured performance and ** issue warnings and/or rerun [ANALYZE] if discrepancies are found. ** ** Since this interface is expected to be rarely used, it is only @@ -9949,32 +9964,51 @@ ** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] ** compile-time option. ** ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior -** of this interface is undefined. -** ^The requested measurement is written into a variable pointed to by -** the "pOut" parameter. -** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. ^If idx is out of range - less than -** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned and the variable that pOut -** points to is unchanged. +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. ** -** ^Statistics might not be available for all loops in all statements. ^In cases -** where there exist loops with no available statistics, this function behaves -** as if the loop did not exist - it returns non-zero and leave the variable -** that pOut points to unchanged. +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ ); +int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ +#define SQLITE_SCANSTAT_COMPLEX 0x0001 /* ** CAPI3REF: Zero Scan-Status Counters ** METHOD: sqlite3_stmt ** Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -971,13 +971,13 @@ ** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the ** underlying malloc() implementation might return us 4-byte aligned ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) #else -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif /* ** Disable MMAP on platforms where it is known to not work */ @@ -1027,18 +1027,41 @@ #endif #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE) \ || defined(SQLITE_ENABLE_TREETRACE)) # define TREETRACE_ENABLED 1 -# define SELECTTRACE(K,P,S,X) \ +# define TREETRACE(K,P,S,X) \ if(sqlite3TreeTrace&(K)) \ sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else -# define SELECTTRACE(K,P,S,X) +# define TREETRACE(K,P,S,X) # define TREETRACE_ENABLED 0 #endif + +/* TREETRACE flag meanings: +** +** 0x00000001 Beginning and end of SELECT processing +** 0x00000002 WHERE clause processing +** 0x00000004 Query flattener +** 0x00000008 Result-set wildcard expansion +** 0x00000010 Query name resolution +** 0x00000020 Aggregate analysis +** 0x00000040 Window functions +** 0x00000080 Generated column names +** 0x00000100 Move HAVING terms into WHERE +** 0x00000200 Count-of-view optimization +** 0x00000400 Compound SELECT processing +** 0x00000800 Drop superfluous ORDER BY +** 0x00001000 LEFT JOIN simplifies to JOIN +** 0x00002000 Constant propagation +** 0x00004000 Push-down optimization +** 0x00008000 After all FROM-clause analysis +** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing +** 0x00020000 Transform DISTINCT into GROUP BY +** 0x00040000 SELECT tree dump after all code has been generated +*/ /* ** Macros for "wheretrace" */ extern u32 sqlite3WhereTrace; @@ -1047,10 +1070,40 @@ # define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X # define WHERETRACE_ENABLED 1 #else # define WHERETRACE(K,X) #endif + +/* +** Bits for the sqlite3WhereTrace mask: +** +** (---any--) Top-level block structure +** 0x-------F High-level debug messages +** 0x----FFF- More detail +** 0xFFFF---- Low-level debug messages +** +** 0x00000001 Code generation +** 0x00000002 Solver +** 0x00000004 Solver costs +** 0x00000008 WhereLoop inserts +** +** 0x00000010 Display sqlite3_index_info xBestIndex calls +** 0x00000020 Range an equality scan metrics +** 0x00000040 IN operator decisions +** 0x00000080 WhereLoop cost adjustements +** 0x00000100 +** 0x00000200 Covering index decisions +** 0x00000400 OR optimization +** 0x00000800 Index scanner +** 0x00001000 More details associated with code generation +** 0x00002000 +** 0x00004000 Show all WHERE terms at key points +** 0x00008000 Show the full SELECT statement at key places +** +** 0x00010000 Show more detail when printing WHERE terms +** 0x00020000 Show WHERE terms returned from whereScanNext() +*/ /* ** An instance of the following structure is used to store the busy-handler ** callback for a given sqlite handle. @@ -1789,10 +1842,11 @@ #define SQLITE_BalancedMerge 0x00200000 /* Balance multi-way merges */ #define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ #define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ +#define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ @@ -2183,10 +2237,11 @@ #define SQLITE_AFF_BLOB 0x41 /* 'A' */ #define SQLITE_AFF_TEXT 0x42 /* 'B' */ #define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ #define SQLITE_AFF_INTEGER 0x44 /* 'D' */ #define SQLITE_AFF_REAL 0x45 /* 'E' */ +#define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) /* ** The SQLITE_AFF_MASK values masks off the significant bits of an @@ -2714,20 +2769,19 @@ struct AggInfo { u8 directMode; /* Direct rendering mode means take data directly ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ int nAccumulator; /* Number of columns that show through to the output. @@ -2734,18 +2788,31 @@ ** Additional columns are used only as parameters to ** aggregate functions */ struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ +#ifdef SQLITE_DEBUG + Select *pSelect; /* SELECT statement that this AggInfo supports */ +#endif }; +/* +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. +*/ +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater ** than 32767 we have to make it 32-bit. 16-bit is preferred because ** it uses less memory in the Expr object, which is a big memory user @@ -3401,10 +3468,11 @@ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ #define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ +#define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ /* True if S exists and has SF_NestedFrom */ #define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) /* @@ -3509,11 +3577,11 @@ u8 eDest; /* How to dispose of the results. One of SRT_* above. */ int iSDParm; /* A parameter used by the eDest disposal method */ int iSDParm2; /* A second parameter for the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ - char *zAffSdst; /* Affinity used for SRT_Set, SRT_Table, and similar */ + char *zAffSdst; /* Affinity used for SRT_Set */ ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; /* ** During code generation of statements that do inserts into AUTOINCREMENT @@ -3568,14 +3636,14 @@ # define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) # define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) #else typedef unsigned int yDbMask; # define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) -# define DbMaskZero(M) (M)=0 -# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) -# define DbMaskAllZero(M) (M)==0 -# define DbMaskNonZero(M) (M)!=0 +# define DbMaskZero(M) ((M)=0) +# define DbMaskSet(M,I) ((M)|=(((yDbMask)1)<<(I))) +# define DbMaskAllZero(M) ((M)==0) +# define DbMaskNonZero(M) ((M)!=0) #endif /* ** For each index X that has as one of its arguments either an expression ** or the name of a virtual generated column, and if X is in scope such that @@ -3654,11 +3722,11 @@ ** of the base register during check-constraint eval */ int nLabel; /* The *negative* of the number of labels used */ int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ - IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ + IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ @@ -4614,11 +4682,11 @@ void sqlite3ColumnSetColl(sqlite3*,Column*,const char*zColl); const char *sqlite3ColumnColl(Column*); void sqlite3DeleteColumnNames(sqlite3*,Table*); void sqlite3GenerateColumnNames(Parse *pParse, Select *pSelect); int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); +void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char); Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); void sqlite3OpenSchemaTable(Parse *, int); Index *sqlite3PrimaryKeyIndex(Table*); i16 sqlite3TableColumnToIndex(Index*, i16); #ifdef SQLITE_OMIT_GENERATED_COLUMNS @@ -4985,10 +5053,11 @@ void sqlite3TableAffinity(Vdbe*, Table*, int); char sqlite3CompareAffinity(const Expr *pExpr, char aff2); int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); char sqlite3TableColumnAffinity(const Table*,int); char sqlite3ExprAffinity(const Expr *pExpr); +int sqlite3ExprDataType(const Expr *pExpr); int sqlite3Atoi64(const char*, i64*, int, u8); int sqlite3DecOrHexToI64(const char*, i64*); void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); void sqlite3Error(sqlite3*,int); void sqlite3ErrorClear(sqlite3*); @@ -5001,10 +5070,13 @@ const char *sqlite3ErrName(int); #endif #ifndef SQLITE_OMIT_DESERIALIZE int sqlite3MemdbInit(void); +int sqlite3IsMemdb(const sqlite3_vfs*); +#else +# define sqlite3IsMemdb(X) 0 #endif const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); @@ -5140,11 +5212,11 @@ void sqlite3OomClear(sqlite3*); int sqlite3ApiExit(sqlite3 *db, int); int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); -int sqlite3StrAccumEnlarge(StrAccum*, int); +int sqlite3StrAccumEnlarge(StrAccum*, i64); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumSetError(StrAccum*, u8); void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); @@ -5497,7 +5569,13 @@ #endif #if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) int sqlite3KvvfsInit(void); #endif + +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +sqlite3_uint64 sqlite3Hwtime(void); +#endif #endif /* SQLITEINT_H */ Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -2186,11 +2186,11 @@ return TCL_OK; } #ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** Usage: sqlite3_stmt_scanstatus STMT IDX +** Usage: sqlite3_stmt_scanstatus ?-flags FLAGS? STMT IDX */ static int SQLITE_TCLAPI test_stmt_scanstatus( void * clientData, Tcl_Interp *interp, int objc, @@ -2201,40 +2201,103 @@ const char *zName; const char *zExplain; sqlite3_int64 nLoop; sqlite3_int64 nVisit; + sqlite3_int64 nCycle; double rEst; int res; + int flags = 0; + int iSelectId = 0; + int iParentId = 0; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "STMT IDX"); + if( objc==5 ){ + struct Flag { + const char *zFlag; + int flag; + } aTbl[] = { + {"complex", SQLITE_SCANSTAT_COMPLEX}, + {0, 0} + }; + + Tcl_Obj **aFlag = 0; + int nFlag = 0; + int ii; + + if( Tcl_ListObjGetElements(interp, objv[2], &nFlag, &aFlag) ){ + return TCL_ERROR; + } + for(ii=0; iiflags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); if( ExprHasProperty(pExpr, EP_OuterON) ){ @@ -502,10 +502,13 @@ sqlite3_str_appendf(&x, " DDL"); } if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; } switch( pExpr->op ){ Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -261,11 +261,12 @@ sqlite3ExprDup(db, pChanges->a[i].pExpr, 0) ); } } pSelect = sqlite3SelectNew(pParse, pList, - pSrc, pWhere2, pGrp, 0, pOrderBy2, SF_UFSrcCheck|SF_IncludeHidden, pLimit2 + pSrc, pWhere2, pGrp, 0, pOrderBy2, + SF_UFSrcCheck|SF_IncludeHidden|SF_UpdateFrom, pLimit2 ); if( pSelect ) pSelect->selFlags |= SF_OrderByReqd; sqlite3SelectDestInit(&dest, eDest, iEph); dest.iSDParm2 = (pPk ? pPk->nKeyCol : -1); sqlite3Select(pParse, pSelect, &dest); Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -1711,5 +1711,14 @@ if( strncmp(z,zName,nName)==0 && z[nName]==0 ) return pIn[i]; i += pIn[i+1]; }while( i=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL - || affinity==SQLITE_AFF_NUMERIC ); + || affinity==SQLITE_AFF_NUMERIC || affinity==SQLITE_AFF_FLEXNUM ); if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags & (MEM_Real|MEM_IntReal))==0 ){ + if( (pRec->flags & MEM_Real)==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); - }else{ + }else if( affinity<=SQLITE_AFF_REAL ){ sqlite3VdbeIntegerAffinity(pRec); } } }else if( affinity==SQLITE_AFF_TEXT ){ /* Only attempt the conversion to TEXT if there is an integer or real @@ -615,21 +622,10 @@ # define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) #else # define REGISTER_TRACE(R,M) #endif - -#ifdef VDBE_PROFILE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - -#endif - #ifndef NDEBUG /* ** This function is only called from within an assert() expression. It ** checks that the sqlite3.nTransaction variable is correctly set to ** the number of non-transaction savepoints currently in the @@ -715,15 +711,14 @@ int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ Op *aOp = p->aOp; /* Copy of p->aOp */ Op *pOp = aOp; /* Current operation */ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#ifdef SQLITE_DEBUG Op *pOrigOp; /* Value of pOp at the top of the loop */ -#endif -#ifdef SQLITE_DEBUG int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ + u8 iCompareIsInit = 0; /* iCompare is initialized */ #endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ u8 encoding = ENC(db); /* The database encoding */ @@ -735,17 +730,19 @@ Mem *aMem = p->aMem; /* Copy of p->aMem */ Mem *pIn1 = 0; /* 1st input operand */ Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ -#ifdef VDBE_PROFILE - u64 start; /* CPU clock count at start of opcode */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 *pnCycle = 0; #endif /*** INSERT STACK UNION HERE ***/ assert( p->eVdbeState==VDBE_RUN_STATE ); /* sqlite3_step() verifies this */ - sqlite3VdbeEnter(p); + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeEnter(p); + } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; assert( 0 < db->nProgressOps ); nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); @@ -762,11 +759,10 @@ testcase( p->rc!=SQLITE_OK ); p->rc = SQLITE_OK; assert( p->bIsReader || p->readOnly!=0 ); p->iCurrentTime = 0; assert( p->explain==0 ); - p->pResultSet = 0; db->busyHandler.nBusy = 0; if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); @@ -799,16 +795,18 @@ /* Errors are detected by individual opcodes, with an immediate ** jumps to abort_due_to_error. */ assert( rc==SQLITE_OK ); assert( pOp>=aOp && pOp<&aOp[p->nOp]); -#ifdef VDBE_PROFILE - start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); -#endif nVmStep++; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec++; + pnCycle = &pOp->nCycle; +# ifdef VDBE_PROFILE + if( sqlite3NProfileCnt==0 ) +# endif + *pnCycle -= sqlite3Hwtime(); #endif /* Only allow tracing if SQLITE_DEBUG is defined. */ #ifdef SQLITE_DEBUG @@ -866,11 +864,11 @@ assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); memAboutToChange(p, &aMem[pOp->p3]); } } #endif -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#ifdef SQLITE_DEBUG pOrigOp = pOp; #endif switch( pOp->opcode ){ @@ -1591,14 +1589,14 @@ assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 || CORRUPT_DB ); assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); p->cacheCtr = (p->cacheCtr + 2)|1; - p->pResultSet = &aMem[pOp->p1]; + p->pResultRow = &aMem[pOp->p1]; #ifdef SQLITE_DEBUG { - Mem *pMem = p->pResultSet; + Mem *pMem = p->pResultRow; int i; for(i=0; ip2; i++){ assert( memIsValid(&pMem[i]) ); REGISTER_TRACE(pOp->p1+i, &pMem[i]); /* The registers in the result will not be used again when the @@ -2124,30 +2122,32 @@ pIn1 = &aMem[pOp->p1]; pIn3 = &aMem[pOp->p3]; flags1 = pIn1->flags; flags3 = pIn3->flags; if( (flags1 & flags3 & MEM_Int)!=0 ){ - assert( (pOp->p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_TEXT || CORRUPT_DB ); /* Common case of comparison of two integers */ if( pIn3->u.i > pIn1->u.i ){ if( sqlite3aGTb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } iCompare = +1; + VVA_ONLY( iCompareIsInit = 1; ) }else if( pIn3->u.i < pIn1->u.i ){ if( sqlite3aLTb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } iCompare = -1; + VVA_ONLY( iCompareIsInit = 1; ) }else{ if( sqlite3aEQb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } iCompare = 0; + VVA_ONLY( iCompareIsInit = 1; ) } VdbeBranchTaken(0, (pOp->p5 & SQLITE_NULLEQ)?2:3); break; } if( (flags1 | flags3)&MEM_Null ){ @@ -2175,10 +2175,11 @@ VdbeBranchTaken(2,3); if( pOp->p5 & SQLITE_JUMPIFNULL ){ goto jump_to_p2; } iCompare = 1; /* Operands are not equal */ + VVA_ONLY( iCompareIsInit = 1; ) break; } }else{ /* Neither operand is NULL and we couldn't do the special high-speed ** integer comparison case. So do a general-case comparison. */ @@ -2192,19 +2193,19 @@ } if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } - }else if( affinity==SQLITE_AFF_TEXT ){ + }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); - if( pIn1==pIn3 ) flags3 = flags1 | MEM_Str; + if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); testcase( pIn3->flags & MEM_IntReal ); @@ -2231,10 +2232,11 @@ res2 = sqlite3aEQb[pOp->opcode]; }else{ res2 = sqlite3aGTb[pOp->opcode]; } iCompare = res; + VVA_ONLY( iCompareIsInit = 1; ) /* Undo any changes made by applyAffinity() to the input registers. */ assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); pIn3->flags = flags3; assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); @@ -2269,10 +2271,11 @@ if( aOp[iAddr].opcode==OP_ReleaseReg ) continue; assert( aOp[iAddr].opcode==OP_Lt || aOp[iAddr].opcode==OP_Gt ); break; } #endif /* SQLITE_DEBUG */ + assert( iCompareIsInit ); VdbeBranchTaken(iCompare==0, 2); if( iCompare==0 ) goto jump_to_p2; break; } @@ -2363,10 +2366,11 @@ REGISTER_TRACE(p2+idx, &aMem[p2+idx]); assert( inKeyField ); pColl = pKeyInfo->aColl[i]; bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); + VVA_ONLY( iCompareIsInit = 1; ) if( iCompare ){ if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) ){ iCompare = -iCompare; @@ -2387,10 +2391,11 @@ ** ** This opcode must immediately follow an OP_Compare opcode. */ case OP_Jump: { /* jump */ assert( pOp>aOp && pOp[-1].opcode==OP_Compare ); + assert( iCompareIsInit ); if( iCompare<0 ){ VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ VdbeBranchTaken(1,4); pOp = &aOp[pOp->p2 - 1]; }else{ @@ -2786,11 +2791,11 @@ ** of large blobs is not loaded, thus saving CPU cycles. If the ** OPFLAG_TYPEOFARG bit is set then the result will only be used by the ** typeof() function or the IS NULL or IS NOT NULL operators or the ** equivalent. In this case, all content loading can be omitted. */ -case OP_Column: { +case OP_Column: { /* ncycle */ u32 p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The B-Tree cursor corresponding to pC */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int len; /* The length of the serialized data for the column */ @@ -4138,11 +4143,11 @@ ** This instruction works like OpenRead except that it opens the cursor ** in read/write mode. ** ** See also: OP_OpenRead, OP_ReopenIdx */ -case OP_ReopenIdx: { +case OP_ReopenIdx: { /* ncycle */ int nField; KeyInfo *pKeyInfo; u32 p2; int iDb; int wrFlag; @@ -4159,11 +4164,11 @@ sqlite3BtreeClearCursor(pCur->uc.pCursor); goto open_cursor_set_hints; } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -case OP_OpenRead: +case OP_OpenRead: /* ncycle */ case OP_OpenWrite: assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( p->bIsReader ); assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx @@ -4253,11 +4258,11 @@ ** cursor P2. The P2 cursor must have been opened by a prior OP_OpenEphemeral ** opcode. Only ephemeral cursors may be duplicated. ** ** Duplicate ephemeral cursors are used for self-joins of materialized views. */ -case OP_OpenDup: { +case OP_OpenDup: { /* ncycle */ VdbeCursor *pOrig; /* The original cursor to be duplicated */ VdbeCursor *pCx; /* The new cursor */ pOrig = p->apCsr[pOp->p2]; assert( pOrig ); @@ -4315,12 +4320,12 @@ ** This opcode works the same as OP_OpenEphemeral. It has a ** different name to distinguish its use. Tables created using ** by this opcode will be used for automatically created transient ** indices in joins. */ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { +case OP_OpenAutoindex: /* ncycle */ +case OP_OpenEphemeral: { /* ncycle */ VdbeCursor *pCx; KeyInfo *pKeyInfo; static const int vfsFlags = SQLITE_OPEN_READWRITE | @@ -4474,11 +4479,11 @@ /* Opcode: Close P1 * * * * ** ** Close a cursor previously opened as P1. If P1 is not ** currently open, this instruction is a no-op. */ -case OP_Close: { +case OP_Close: { /* ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); p->apCsr[pOp->p1] = 0; break; } @@ -4591,14 +4596,14 @@ ** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this ** is an equality search. ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group */ -case OP_SeekLE: /* jump, in3, group */ -case OP_SeekGE: /* jump, in3, group */ -case OP_SeekGT: { /* jump, in3, group */ +case OP_SeekLT: /* jump, in3, group, ncycle */ +case OP_SeekLE: /* jump, in3, group, ncycle */ +case OP_SeekGE: /* jump, in3, group, ncycle */ +case OP_SeekGT: { /* jump, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ UnpackedRecord r; /* The key to seek for */ int nField; /* Number of columns or fields in the key */ @@ -4860,11 +4865,11 @@ **
  • If the cursor ends up on a valid row that is past the target row ** (indicating that the target row does not exist in the btree) then ** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. ** */ -case OP_SeekScan: { +case OP_SeekScan: { /* ncycle */ VdbeCursor *pC; int res; int nStep; UnpackedRecord r; @@ -4982,11 +4987,11 @@ ** OP_IfNoHope opcode might run to see if the IN loop can be abandoned ** early, thus saving work. This is part of the IN-early-out optimization. ** ** P1 must be a valid b-tree cursor. */ -case OP_SeekHit: { +case OP_SeekHit: { /* ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pOp->p3>=pOp->p2 ); @@ -5114,11 +5119,11 @@ ** advanced in either direction. In other words, the Next and Prev ** opcodes do not work after this operation. ** ** See also: NotFound, Found, NotExists */ -case OP_IfNoHope: { /* jump, in3 */ +case OP_IfNoHope: { /* jump, in3, ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); #ifdef SQLITE_DEBUG @@ -5128,13 +5133,13 @@ #endif if( pC->seekHit>=pOp->p4.i ) break; /* Fall through into OP_NotFound */ /* no break */ deliberate_fall_through } -case OP_NoConflict: /* jump, in3 */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ +case OP_NoConflict: /* jump, in3, ncycle */ +case OP_NotFound: /* jump, in3, ncycle */ +case OP_Found: { /* jump, in3, ncycle */ int alreadyExists; int ii; VdbeCursor *pC; UnpackedRecord *pIdxKey; UnpackedRecord r; @@ -5260,11 +5265,11 @@ ** in either direction. In other words, the Next and Prev opcodes will ** not work following this opcode. ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3 */ +case OP_SeekRowid: { /* jump, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; u64 iKey; @@ -5285,11 +5290,11 @@ iKey = x.u.i; goto notExistsWithKey; } /* Fall through into OP_NotExists */ /* no break */ deliberate_fall_through -case OP_NotExists: /* jump, in3 */ +case OP_NotExists: /* jump, in3, ncycle */ pIn3 = &aMem[pOp->p3]; assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); iKey = pIn3->u.i; notExistsWithKey: @@ -5565,12 +5570,15 @@ } } if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + assert( (pOp->p5 & OPFLAG_LASTROWID)==0 || (pOp->p5 & OPFLAG_NCHANGE)!=0 ); + if( pOp->p5 & OPFLAG_NCHANGE ){ + p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + } assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); x.pData = pData->z; x.nData = pData->n; seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); if( pData->flags & MEM_Zero ){ @@ -5577,10 +5585,11 @@ x.nZero = pData->u.nZero; }else{ x.nZero = 0; } x.pKey = 0; + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), seekResult ); pC->deferredMoveto = 0; @@ -5908,11 +5917,11 @@ ** ** P1 can be either an ordinary table or a virtual table. There used to ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2 */ +case OP_Rowid: { /* out2, ncycle */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; const sqlite3_module *pModule; @@ -6007,12 +6016,12 @@ ** ** This opcode leaves the cursor configured to move in reverse order, ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. */ -case OP_SeekEnd: -case OP_Last: { /* jump */ +case OP_SeekEnd: /* ncycle */ +case OP_Last: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -6113,11 +6122,11 @@ ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump */ +case OP_Rewind: { /* jump, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -6207,11 +6216,11 @@ pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_Prev: /* jump */ +case OP_Prev: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; @@ -6222,11 +6231,11 @@ || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope || pC->seekOp==OP_NullRow); rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3); goto next_tail; -case OP_Next: /* jump */ +case OP_Next: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; @@ -6414,12 +6423,12 @@ ** the end of the index key pointed to by cursor P1. This integer should be ** the rowid of the table entry to which this index entry points. ** ** See also: Rowid, MakeRecord. */ -case OP_DeferredSeek: -case OP_IdxRowid: { /* out2 */ +case OP_DeferredSeek: /* ncycle */ +case OP_IdxRowid: { /* out2, ncycle */ VdbeCursor *pC; /* The P1 index cursor */ VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ i64 rowid; /* Rowid that P1 current points to */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -6477,12 +6486,12 @@ ** ** If cursor P1 was previously moved via OP_DeferredSeek, complete that ** seek operation now, without further delay. If the cursor seek has ** already occurred, this instruction is a no-op. */ -case OP_FinishSeek: { - VdbeCursor *pC; /* The P1 index cursor */ +case OP_FinishSeek: { /* ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; if( pC->deferredMoveto ){ rc = sqlite3VdbeFinishMoveto(pC); @@ -6533,14 +6542,14 @@ ** ROWID on the P1 index. ** ** If the P1 index entry is less than or equal to the key value then jump ** to P2. Otherwise fall through to the next instruction. */ -case OP_IdxLE: /* jump */ -case OP_IdxGT: /* jump */ -case OP_IdxLT: /* jump */ -case OP_IdxGE: { /* jump */ +case OP_IdxLE: /* jump, ncycle */ +case OP_IdxGT: /* jump, ncycle */ +case OP_IdxLT: /* jump, ncycle */ +case OP_IdxGE: { /* jump, ncycle */ VdbeCursor *pC; int res; UnpackedRecord r; assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -7157,13 +7166,10 @@ pFrame->apCsr = p->apCsr; pFrame->nCursor = p->nCursor; pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pFrame->anExec = p->anExec; -#endif #ifdef SQLITE_DEBUG pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; #endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; @@ -7196,13 +7202,10 @@ p->apCsr = (VdbeCursor **)&aMem[p->nMem]; pFrame->aOnce = (u8*)&p->apCsr[pProgram->nCsr]; memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = 0; -#endif #ifdef SQLITE_DEBUG /* Verify that second and subsequent executions of the same trigger do not ** try to reuse register values from the first use. */ { int i; @@ -7955,11 +7958,11 @@ ** ** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. ** P1 is a cursor number. This opcode opens a cursor to the virtual ** table and stores that cursor in P1. */ -case OP_VOpen: { +case OP_VOpen: { /* ncycle */ VdbeCursor *pCur; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; const sqlite3_module *pModule; @@ -8002,11 +8005,11 @@ ** can be used as the first argument to sqlite3_vtab_in_first() and ** sqlite3_vtab_in_next() to extract all of the values stored in the P1 ** cursor. Register P3 is used to hold the values returned by ** sqlite3_vtab_in_first() and sqlite3_vtab_in_next(). */ -case OP_VInitIn: { /* out2 */ +case OP_VInitIn: { /* out2, ncycle */ VdbeCursor *pC; /* The cursor containing the RHS values */ ValueList *pRhs; /* New ValueList object to put in reg[P2] */ pC = p->apCsr[pOp->p1]; pRhs = sqlite3_malloc64( sizeof(*pRhs) ); @@ -8039,11 +8042,11 @@ ** additional parameters which are passed to ** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter. ** ** A jump is made to P2 if the result set after filtering would be empty. */ -case OP_VFilter: { /* jump */ +case OP_VFilter: { /* jump, ncycle */ int nArg; int iQuery; const sqlite3_module *pModule; Mem *pQuery; Mem *pArgc; @@ -8099,11 +8102,11 @@ ** function to return true inside the xColumn method of the virtual ** table implementation. The P5 column might also contain other ** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are ** unused by OP_VColumn. */ -case OP_VColumn: { +case OP_VColumn: { /* ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; Mem *pDest; sqlite3_context sContext; @@ -8151,11 +8154,11 @@ ** ** Advance virtual table P1 to the next row in its result set and ** jump to instruction P2. Or, if the virtual table has reached ** the end of its result set, then fall through to the next instruction. */ -case OP_VNext: { /* jump */ +case OP_VNext: { /* jump, ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; int res; VdbeCursor *pCur; @@ -8734,16 +8737,16 @@ ** readability. From this point on down, the normal indentation rules are ** restored. *****************************************************************************/ } -#ifdef VDBE_PROFILE - { - u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - if( endTime>start ) pOrigOp->cycles += endTime - start; - pOrigOp->cnt++; - } +#if defined(VDBE_PROFILE) + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; #endif /* The following code adds nothing to the actual functionality ** of the program. It is only here for testing and debugging. ** On the other hand, it does burn CPU cycles every time through @@ -8815,10 +8818,22 @@ /* This is the only way out of this procedure. We have to ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: +#if defined(VDBE_PROFILE) + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } +#endif + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ nProgressLimit += db->nProgressOps; if( db->xProgress(db->pProgressArg) ){ nProgressLimit = LARGEST_UINT64; @@ -8826,11 +8841,13 @@ goto abort_due_to_error; } } #endif p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; - sqlite3VdbeLeave(p); + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeLeave(p); + } assert( rc!=SQLITE_OK || nExtraDelete==0 || sqlite3_strlike("DELETE%",p->zSql,0)!=0 ); return rc; Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -65,17 +65,17 @@ #endif } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS char *zComment; /* Comment to improve readability */ #endif -#ifdef VDBE_PROFILE - u32 cnt; /* Number of times this instruction was executed */ - u64 cycles; /* Total time spent executing this instruction */ -#endif #ifdef SQLITE_VDBE_COVERAGE u32 iSrcLine; /* Source-code line that generated this opcode ** with flags in the upper 8 bits */ +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 nExec; + u64 nCycle; #endif }; typedef struct VdbeOp VdbeOp; @@ -203,18 +203,24 @@ # define sqlite3VdbeVerifyAbortable(A,B) # define sqlite3VdbeNoJumpsOutsideSubrtn(A,B,C,D) #endif VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno); #ifndef SQLITE_OMIT_EXPLAIN - void sqlite3VdbeExplain(Parse*,u8,const char*,...); + int sqlite3VdbeExplain(Parse*,u8,const char*,...); void sqlite3VdbeExplainPop(Parse*); int sqlite3VdbeExplainParent(Parse*); # define ExplainQueryPlan(P) sqlite3VdbeExplain P +# ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define ExplainQueryPlan2(V,P) (V = sqlite3VdbeExplain P) +# else +# define ExplainQueryPlan2(V,P) ExplainQueryPlan(P) +# endif # define ExplainQueryPlanPop(P) sqlite3VdbeExplainPop(P) # define ExplainQueryPlanParent(P) sqlite3VdbeExplainParent(P) #else # define ExplainQueryPlan(P) +# define ExplainQueryPlan2(V,P) # define ExplainQueryPlanPop(P) # define ExplainQueryPlanParent(P) 0 # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ #endif #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) @@ -383,14 +389,18 @@ # define VDBE_OFFSET_LINENO(x) 0 #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +void sqlite3VdbeScanStatusRange(Vdbe*, int, int, int); +void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); #else -# define sqlite3VdbeScanStatus(a,b,c,d,e) +# define sqlite3VdbeScanStatus(a,b,c,d,e,f) +# define sqlite3VdbeScanStatusRange(a,b,c,d) +# define sqlite3VdbeScanStatusCounters(a,b,c,d) #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); #endif #endif /* SQLITE_VDBE_H */ Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -169,11 +169,10 @@ typedef struct VdbeFrame VdbeFrame; struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u8 *aOnce; /* Bitmask used by OP_Once */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ @@ -385,14 +384,23 @@ */ typedef unsigned bft; /* Bit Field Type */ /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. +** +** aAddrRange[]: +** This array is used by ScanStatus elements associated with EQP +** notes that make an SQLITE_SCANSTAT_NCYCLE value available. It is +** an array of up to 3 ranges of VM addresses for which the Vdbe.anCycle[] +** values should be summed to calculate the NCYCLE value. Each pair of +** integer addresses is a start and end address (both inclusive) for a range +** instructions. A start value of 0 indicates an empty range. */ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ + int aAddrRange[6]; int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ LogEst nEst; /* Estimated output rows per loop */ char *zName; /* Name of table or index */ @@ -444,11 +452,11 @@ Op *aOp; /* Space to hold the virtual machine's program */ int nOp; /* Number of instructions in the program */ int nOpAlloc; /* Slots allocated for aOp[] */ Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ + Mem *pResultRow; /* Current output row */ char *zErrMsg; /* Error message written here */ VList *pVList; /* Name of variables */ #ifndef SQLITE_OMIT_TRACE i64 startTime; /* Time when query started - used for profiling */ #endif @@ -481,11 +489,10 @@ int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - i64 *anExec; /* Number of times each op has been executed */ int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif }; Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -13,10 +13,11 @@ ** This file contains code use to implement APIs that are part of the ** VDBE. */ #include "sqliteInt.h" #include "vdbeInt.h" +#include "opcodes.h" #ifndef SQLITE_OMIT_DEPRECATED /* ** Return TRUE (non-zero) of the statement supplied as an argument needs ** to be recompiled. A statement needs to be recompiled whenever the @@ -503,11 +504,14 @@ void (*xDel)(void *), unsigned char enc ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + n &= ~(u64)1; + } if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); } @@ -518,29 +522,29 @@ const void *z, int n, void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16NATIVE, xDel); } void sqlite3_result_text16be( sqlite3_context *pCtx, const void *z, int n, void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16BE, xDel); } void sqlite3_result_text16le( sqlite3_context *pCtx, const void *z, int n, void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ Mem *pOut = pCtx->pOut; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); @@ -747,11 +751,11 @@ }else{ #ifndef SQLITE_OMIT_TRACE /* If the statement completed successfully, invoke the profile callback */ checkProfileCallback(db, p); #endif - + p->pResultRow = 0; if( rc==SQLITE_DONE && db->autoCommit ){ assert( p->rc==SQLITE_OK ); p->rc = doWalCallbacks(db); if( p->rc!=SQLITE_OK ){ rc = SQLITE_ERROR; @@ -1111,11 +1115,11 @@ ** Return the number of values available from the current row of the ** currently executing statement pStmt. */ int sqlite3_data_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - if( pVm==0 || pVm->pResultSet==0 ) return 0; + if( pVm==0 || pVm->pResultRow==0 ) return 0; return pVm->nResColumn; } /* ** Return a pointer to static memory containing an SQL NULL value. @@ -1166,12 +1170,12 @@ pVm = (Vdbe *)pStmt; if( pVm==0 ) return (Mem*)columnNullValue(); assert( pVm->db ); sqlite3_mutex_enter(pVm->db->mutex); - if( pVm->pResultSet!=0 && inResColumn && i>=0 ){ - pOut = &pVm->pResultSet[i]; + if( pVm->pResultRow!=0 && inResColumn && i>=0 ){ + pOut = &pVm->pResultRow[i]; }else{ sqlite3Error(pVm->db, SQLITE_RANGE); pOut = (Mem*)columnNullValue(); } return pOut; @@ -1601,22 +1605,25 @@ sqlite3_uint64 nData, void (*xDel)(void*), unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + nData &= ~(u16)1; + } return bindText(pStmt, i, zData, nData, xDel, enc); } #ifndef SQLITE_OMIT_UTF16 int sqlite3_bind_text16( sqlite3_stmt *pStmt, int i, const void *zData, - int nData, + int n, void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); + return bindText(pStmt, i, zData, n & ~(u64)1, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int rc; switch( sqlite3_value_type((sqlite3_value*)pValue) ){ @@ -2103,27 +2110,64 @@ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* ** Return status data for a single loop within query pStmt. */ -int sqlite3_stmt_scanstatus( +int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement being queried */ - int idx, /* Index of loop to report on */ + int iScan, /* Index of loop to report on */ int iScanStatusOp, /* Which metric to return */ + int flags, void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; ScanStatus *pScan; - if( idx<0 || idx>=p->nScan ) return 1; - pScan = &p->aScan[idx]; + int idx; + + if( iScan<0 ){ + int ii; + if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ + i64 res = 0; + for(ii=0; iinOp; ii++){ + res += p->aOp[ii].nCycle; + } + *(i64*)pOut = res; + return 0; + } + return 1; + } + if( flags & SQLITE_SCANSTAT_COMPLEX ){ + idx = iScan; + pScan = &p->aScan[idx]; + }else{ + /* If the COMPLEX flag is clear, then this function must ignore any + ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ + for(idx=0; idxnScan; idx++){ + pScan = &p->aScan[idx]; + if( pScan->zName ){ + iScan--; + if( iScan<0 ) break; + } + } + } + if( idx>=p->nScan ) return 1; + switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + if( pScan->addrLoop>0 ){ + *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_NVISIT: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + if( pScan->addrVisit>0 ){ + *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_EST: { double r = 1.0; LogEst x = pScan->nEst; @@ -2151,21 +2195,77 @@ *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; }else{ *(int*)pOut = -1; } break; + } + case SQLITE_SCANSTAT_PARENTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p2; + }else{ + *(int*)pOut = -1; + } + break; + } + case SQLITE_SCANSTAT_NCYCLE: { + i64 res = 0; + if( pScan->aAddrRange[0]==0 ){ + res = -1; + }else{ + int ii; + for(ii=0; iiaAddrRange); ii+=2){ + int iIns = pScan->aAddrRange[ii]; + int iEnd = pScan->aAddrRange[ii+1]; + if( iIns==0 ) break; + if( iIns>0 ){ + while( iIns<=iEnd ){ + res += p->aOp[iIns].nCycle; + iIns++; + } + }else{ + int iOp; + for(iOp=0; iOpnOp; iOp++){ + Op *pOp = &p->aOp[iOp]; + if( pOp->p1!=iEnd ) continue; + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ + continue; + } + res += p->aOp[iOp].nCycle; + } + } + } + } + *(i64*)pOut = res; + break; } default: { return 1; } } return 0; } + +/* +** Return status data for a single loop within query pStmt. +*/ +int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int iScan, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + return sqlite3_stmt_scanstatus_v2(pStmt, iScan, iScanStatusOp, 0, pOut); +} /* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - memset(p->anExec, 0, p->nOp * sizeof(i64)); + int ii; + for(ii=0; iinOp; ii++){ + Op *pOp = &p->aOp[ii]; + pOp->nExec = 0; + pOp->nCycle = 0; + } } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -210,10 +210,12 @@ ** sqlite3MisuseError(lineno) ** sqlite3CantopenError(lineno) */ static void test_addop_breakpoint(int pc, Op *pOp){ static int n = 0; + (void)pc; + (void)pOp; n++; } #endif /* @@ -259,21 +261,21 @@ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOp->zComment = 0; +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; #endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i, &p->aOp[i]); test_addop_breakpoint(i, &p->aOp[i]); } #endif -#ifdef VDBE_PROFILE - pOp->cycles = 0; - pOp->cnt = 0; -#endif #ifdef SQLITE_VDBE_COVERAGE pOp->iSrcLine = 0; #endif return i; } @@ -437,12 +439,13 @@ ** Add a new OP_Explain opcode. ** ** If the bPush flag is true, then make this opcode the parent for ** subsequent Explains until sqlite3VdbeExplainPop() is called. */ -void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ -#ifndef SQLITE_DEBUG +int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ + int addr = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ if( pParse->explain==2 ) #endif { @@ -453,17 +456,19 @@ va_start(ap, zFmt); zMsg = sqlite3VMPrintf(pParse->db, zFmt, ap); va_end(ap); v = pParse->pVdbe; iThis = v->nOp; - sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, + addr = sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); if( bPush){ pParse->addrExplain = iThis; } + sqlite3VdbeScanStatus(v, iThis, 0, 0, 0, 0); } + return addr; } /* ** Pop the EXPLAIN QUERY PLAN stack one level. */ @@ -1117,18 +1122,75 @@ sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); pNew->addrExplain = addrExplain; pNew->addrLoop = addrLoop; pNew->addrVisit = addrVisit; pNew->nEst = nEst; pNew->zName = sqlite3DbStrDup(p->db, zName); p->aScan = aNew; } } + +/* +** Add the range of instructions from addrStart to addrEnd (inclusive) to +** the set of those corresponding to the sqlite3_stmt_scanstatus() counters +** associated with the OP_Explain instruction at addrExplain. The +** sum of the sqlite3Hwtime() values for each of these instructions +** will be returned for SQLITE_SCANSTAT_NCYCLE requests. +*/ +void sqlite3VdbeScanStatusRange( + Vdbe *p, + int addrExplain, + int addrStart, + int addrEnd +){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } + } + } +} + +/* +** Set the addresses for the SQLITE_SCANSTAT_NLOOP and SQLITE_SCANSTAT_NROW +** counters for the query element associated with the OP_Explain at +** addrExplain. +*/ +void sqlite3VdbeScanStatusCounters( + Vdbe *p, + int addrExplain, + int addrLoop, + int addrVisit +){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrLoop = addrLoop; + pScan->addrVisit = addrVisit; + } +} #endif /* ** Change the value of the opcode, or P1, P2, P3, or P5 operands @@ -2254,11 +2316,10 @@ /* Even though this opcode does not use dynamic strings for ** the result, result columns may become dynamic if the user calls ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. */ releaseMemArray(pMem, 8); - p->pResultSet = 0; if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ sqlite3OomFault(db); @@ -2311,11 +2372,11 @@ sqlite3VdbeMemSetNull(pMem+7); #endif sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); p->nResColumn = 8; } - p->pResultSet = pMem; + p->pResultRow = pMem; if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; rc = SQLITE_ERROR; }else{ p->rc = SQLITE_OK; @@ -2422,11 +2483,11 @@ /* ** Rewind the VDBE back to the beginning in preparation for ** running it. */ void sqlite3VdbeRewind(Vdbe *p){ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#if defined(SQLITE_DEBUG) int i; #endif assert( p!=0 ); assert( p->eVdbeState==VDBE_INIT_STATE || p->eVdbeState==VDBE_READY_STATE @@ -2451,12 +2512,12 @@ p->minWriteFileFormat = 255; p->iStatement = 0; p->nFkConstraint = 0; #ifdef VDBE_PROFILE for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + p->aOp[i].nExec = 0; + p->aOp[i].nCycle = 0; } #endif } /* @@ -2561,24 +2622,18 @@ x.nNeeded = 0; p->aMem = allocSpace(&x, 0, nMem*sizeof(Mem)); p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); -#endif if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; if( !db->mallocFailed ){ p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); -#endif } } if( db->mallocFailed ){ p->nVar = 0; @@ -2589,13 +2644,10 @@ p->nVar = (ynVar)nVar; initMemArray(p->aVar, nVar, db, MEM_Null); p->nMem = nMem; initMemArray(p->aMem, nMem, db, MEM_Undefined); memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - memset(p->anExec, 0, p->nOp*sizeof(i64)); -#endif } sqlite3VdbeRewind(p); } /* @@ -2649,13 +2701,10 @@ ** control to the main program. */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - v->anExec = pFrame->anExec; -#endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; v->nMem = pFrame->nMem; v->apCsr = pFrame->apCsr; @@ -3455,11 +3504,11 @@ #endif if( p->zErrMsg ){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - p->pResultSet = 0; + p->pResultRow = 0; #ifdef SQLITE_DEBUG p->nWrite = 0; #endif /* Save profiling information from this VDBE run. @@ -3483,14 +3532,16 @@ } if( pc!='\n' ) fprintf(out, "\n"); } for(i=0; inOp; i++){ char zHdr[100]; + i64 cnt = p->aOp[i].nExec; + i64 cycles = p->aOp[i].nCycle; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", - p->aOp[i].cnt, - p->aOp[i].cycles, - p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + cnt, + cycles, + cnt>0 ? cycles/cnt : 0 ); fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); } fclose(out); Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -674,39 +674,36 @@ if( pMem->flags & MEM_Null ) return ifNull; return sqlite3VdbeRealValue(pMem)!=0.0; } /* -** The MEM structure is already a MEM_Real or MEM_IntReal. Try to -** make it a MEM_Int if we can. +** The MEM structure is already a MEM_Real. Try to also make it a +** MEM_Int if we can. */ void sqlite3VdbeIntegerAffinity(Mem *pMem){ + i64 ix; assert( pMem!=0 ); - assert( pMem->flags & (MEM_Real|MEM_IntReal) ); + assert( pMem->flags & MEM_Real ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - if( pMem->flags & MEM_IntReal ){ - MemSetTypeFlag(pMem, MEM_Int); - }else{ - i64 ix = doubleToInt64(pMem->u.r); - - /* Only mark the value as an integer if - ** - ** (1) the round-trip conversion real->int->real is a no-op, and - ** (2) The integer is neither the largest nor the smallest - ** possible integer (ticket #3922) - ** - ** The second and third terms in the following conditional enforces - ** the second condition under the assumption that addition overflow causes - ** values to wrap around. - */ - if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; - MemSetTypeFlag(pMem, MEM_Int); - } + ix = doubleToInt64(pMem->u.r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. + */ + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); } } /* ** Convert pMem to type integer. Invalidate any prior representations. Index: src/vdbevtab.c ================================================================== --- src/vdbevtab.c +++ src/vdbevtab.c @@ -81,10 +81,13 @@ "subprog TEXT," "stmt HIDDEN" ");" }; + (void)argc; + (void)argv; + (void)pzErr; rc = sqlite3_declare_vtab(db, azSchema[isTabUsed]); if( rc==SQLITE_OK ){ pNew = sqlite3_malloc( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; @@ -316,10 +319,11 @@ int argc, sqlite3_value **argv ){ bytecodevtab_cursor *pCur = (bytecodevtab_cursor *)pVtabCursor; bytecodevtab *pVTab = (bytecodevtab *)pVtabCursor->pVtab; int rc = SQLITE_OK; + (void)idxStr; bytecodevtabCursorClear(pCur); pCur->iRowid = 0; pCur->iAddr = 0; pCur->showSubprograms = idxNum==0; Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -703,11 +703,11 @@ ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; for(i=0; inConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", i, p->aConstraint[i].iColumn, @@ -723,11 +723,11 @@ p->aOrderBy[i].desc); } } static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, p->aConstraintUsage[i].argvIndex, p->aConstraintUsage[i].omit); @@ -810,10 +810,61 @@ } #endif #ifndef SQLITE_OMIT_AUTOMATIC_INDEX + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Argument pIdx represents an automatic index that the current statement +** will create and populate. Add an OP_Explain with text of the form: +** +** CREATE AUTOMATIC INDEX ON () [WHERE ] +** +** This is only required if sqlite3_stmt_scanstatus() is enabled, to +** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP +** values with. In order to avoid breaking legacy code and test cases, +** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. +*/ +static void explainAutomaticIndex( + Parse *pParse, + Index *pIdx, /* Automatic index to explain */ + int bPartial, /* True if pIdx is a partial index */ + int *pAddrExplain /* OUT: Address of OP_Explain */ +){ + if( pParse->explain!=2 ){ + Table *pTab = pIdx->pTable; + const char *zSep = ""; + char *zText = 0; + int ii = 0; + sqlite3_str *pStr = sqlite3_str_new(pParse->db); + sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName); + assert( pIdx->nColumn>1 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + for(ii=0; ii<(pIdx->nColumn-1); ii++){ + const char *zName = 0; + int iCol = pIdx->aiColumn[ii]; + + zName = pTab->aCol[iCol].zCnName; + sqlite3_str_appendf(pStr, "%s%s", zSep, zName); + zSep = ", "; + } + zText = sqlite3_str_finish(pStr); + if( zText==0 ){ + sqlite3OomFault(pParse->db); + }else{ + *pAddrExplain = sqlite3VdbeExplain( + pParse, 0, "%s)%s", zText, (bPartial ? " WHERE " : "") + ); + sqlite3_free(zText); + } + } +} +#else +# define explainAutomaticIndex(a,b,c,d) +#endif + /* ** Generate code to construct the Index object for an automatic index ** and to set up the WhereLevel object pLevel so that the code generator ** makes use of the automatic index. */ @@ -845,10 +896,13 @@ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ SrcItem *pTabItem; /* FROM clause term being indexed */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp = 0; /* Address of OP_Explain */ +#endif /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); @@ -968,10 +1022,11 @@ assert( n==nKeyCol ); pIdx->aiColumn[n] = XN_ROWID; pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ + explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); @@ -1003,10 +1058,11 @@ ); if( pLevel->regFilter ){ sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, regBase, pLoop->u.btree.nEq); } + sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pTabItem->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); @@ -1023,10 +1079,11 @@ sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); end_auto_index_create: sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ @@ -1740,11 +1797,11 @@ ** using the method described in the header comment for this function. */ if( nDiff!=1 || pUpper==0 || pLower==0 ){ int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); pLoop->nOut -= nAdjust; *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", nLower, nUpper, nAdjust*-1, pLoop->nOut)); } }else{ assert( *pbDone==0 ); @@ -1918,11 +1975,11 @@ nNew = 10; assert( 10==sqlite3LogEst(2) ); } if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n", pLoop->nOut, nOut)); } #endif pLoop->nOut = (LogEst)nOut; return rc; @@ -2016,11 +2073,11 @@ if( rc!=SQLITE_OK ) return rc; if( bOk==0 ) return SQLITE_NOTFOUND; pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + WHERETRACE(0x20,("equality scan regions %s(%d): %d\n", p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; return rc; } @@ -2066,11 +2123,11 @@ } if( rc==SQLITE_OK ){ if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; } #endif /* SQLITE_ENABLE_STAT4 */ @@ -2175,11 +2232,11 @@ sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ int i; for(i=0; inLTerm; i++){ sqlite3WhereTermPrint(p->aLTerm[i], i); } } @@ -3053,11 +3110,11 @@ ** to be true for half or more of the rows in the table. ** See tag-202002240-1 */ && pNew->nOut+10 > pProbe->aiRowLogEst[0] ){ #if WHERETRACE_ENABLED /* 0x01 */ - if( sqlite3WhereTrace & 0x01 ){ + if( sqlite3WhereTrace & 0x20 ){ sqlite3DebugPrintf( "STAT4 determines term has low selectivity:\n"); sqlite3WhereTermPrint(pTerm, 999); } #endif @@ -3090,13 +3147,21 @@ /* Set rCostIdx to the cost of visiting selected rows in index. Add ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ assert( pSrc->pTab->szTabRow>0 ); - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ + /* The pProbe->szIdxRow is low for an IPK table since the interior + ** pages are small. Thuse szIdxRow gives a good estimate of seek cost. + ** But the leaf pages are full-size, so pProbe->szIdxRow would badly + ** under-estimate the scanning cost. */ + rCostIdx = pNew->nOut + 16; + }else{ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + } pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); nOutUnadjusted = pNew->nOut; @@ -3244,21 +3309,44 @@ return 1; } } return 0; } + +/* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} /* ** Structure passed to the whereIsCoveringIndex Walker callback. */ +typedef struct CoveringIndexCheck CoveringIndexCheck; struct CoveringIndexCheck { Index *pIdx; /* The index */ int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ }; /* -** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. ** ** If the Expr node references the table with cursor pCk->iTabCur, then ** make sure that column is covered by the index pCk->pIdx. We know that ** all columns less than 63 (really BMS-1) are covered, so we don't need ** to check them. But we do need to check any column at 63 or greater. @@ -3266,75 +3354,107 @@ ** If the index does not cover the column, then set pWalk->eCode to ** non-zero and return WRC_Abort to stop the search. ** ** If this node does not disprove that the index can be a covering index, ** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. */ static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ - int i; /* Loop counter */ - const Index *pIdx; /* The index of interest */ - const i16 *aiColumn; /* Columns contained in the index */ - u16 nColumn; /* Number of columns in the index */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; - if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; - if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; - pIdx = pWalk->u.pCovIdxCk->pIdx; - aiColumn = pIdx->aiColumn; - nColumn = pIdx->nColumn; - for(i=0; iiColumn ) return WRC_Continue; - } - pWalk->eCode = 1; - return WRC_Abort; + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pCk->bUnidx = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; + return WRC_Prune; + } + return WRC_Continue; } /* ** pIdx is an index that covers all of the low-number columns used by -** pWInfo->pSelect (columns from 0 through 62). But there are columns -** in pWInfo->pSelect beyond 62. This routine tries to answer the question -** of whether pIdx covers *all* columns in the query. -** -** Return 0 if pIdx is a covering index. Return non-zero if pIdx is -** not a covering index or if we are unable to determine if pIdx is a -** covering index. -** -** This routine is an optimization. It is always safe to return non-zero. -** But returning zero when non-zero should have been returned can lead to -** incorrect bytecode and assertion faults. +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. +** +** The return value is one of these: +** +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. */ static SQLITE_NOINLINE u32 whereIsCoveringIndex( WhereInfo *pWInfo, /* The WHERE clause context */ Index *pIdx, /* Index that is being tested */ int iTabCur /* Cursor for the table being indexed */ ){ - int i; + int i, rc; struct CoveringIndexCheck ck; Walker w; if( pWInfo->pSelect==0 ){ /* We don't have access to the full query, so we cannot check to see ** if pIdx is covering. Assume it is not. */ - return 1; - } - for(i=0; inColumn; i++){ - if( pIdx->aiColumn[i]>=BMS-1 ) break; - } - if( i>=pIdx->nColumn ){ - /* pIdx does not index any columns greater than 62, but we know from - ** colMask that columns greater than 62 are used, so this is not a - ** covering index */ - return 1; + return 0; + } + if( pIdx->bHasExpr==0 ){ + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 0; + } } ck.pIdx = pIdx; ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; memset(&w, 0, sizeof(w)); w.xExprCallback = whereIsCoveringIndexWalkCallback; w.xSelectCallback = sqlite3SelectWalkNoop; w.u.pCovIdxCk = &ck; - w.eCode = 0; sqlite3WalkSelect(&w, pWInfo->pSelect); - return w.eCode; + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; } /* ** Add all WhereLoop objects for a single table of the join where the table ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be @@ -3415,11 +3535,11 @@ sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - sPk.szIdxRow = pTab->szTabRow; + sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */ sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->fg.notIndexed==0 ){ @@ -3546,18 +3666,42 @@ pNew->nOut = rSize; if( rc ) break; }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( m==TOPBIT ){ - m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0x200, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0x200, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0x200, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 ){ + WHERETRACE(0x200, + ("-> %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } /* Full scan via index */ if( b || !HasRowid(pTab) @@ -3726,11 +3870,11 @@ if( rc==SQLITE_CONSTRAINT ){ /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means ** that the particular combination of parameters provided is unusable. ** Make no entries in the loop table. */ - WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n")); + WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); return SQLITE_OK; } return rc; } @@ -3837,11 +3981,11 @@ rc = whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; } - WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", *pbIn, (sqlite3_uint64)mPrereq, (sqlite3_uint64)(pNew->prereq & ~mPrereq))); return rc; } @@ -4029,11 +4173,11 @@ return SQLITE_NOMEM_BKPT; } /* First call xBestIndex() with all constraints usable. */ WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); - WHERETRACE(0x40, (" VirtualOne: all usable\n")); + WHERETRACE(0x800, (" VirtualOne: all usable\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry ); if( bRetry ){ assert( rc==SQLITE_OK ); @@ -4054,11 +4198,11 @@ Bitmask mBestNoIn = 0; /* If the plan produced by the earlier call uses an IN(...) term, call ** xBestIndex again, this time with IN(...) terms disabled. */ if( bIn ){ - WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); assert( bIn==0 ); mBestNoIn = pNew->prereq & ~mPrereq; if( mBestNoIn==0 ){ @@ -4080,11 +4224,11 @@ if( mThis>mPrev && mThisprereq==mPrereq ){ seenZero = 1; @@ -4094,21 +4238,21 @@ /* If the calls to xBestIndex() in the above loop did not find a plan ** that requires no source tables at all (i.e. one guaranteed to be ** usable), make a call here with all source tables disabled */ if( rc==SQLITE_OK && seenZero==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); if( bIn==0 ) seenZeroNoIN = 1; } /* If the calls to xBestIndex() have so far failed to find a plan ** that requires no source tables at all and does not use an IN(...) ** operator, make a final call to obtain one here. */ if( rc==SQLITE_OK && seenZeroNoIN==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); } } @@ -4160,11 +4304,11 @@ int i, j; sSubBuild = *pBuilder; sSubBuild.pOrSet = &sCur; - WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; }else if( pOrTerm->leftCursor==iCur ){ tempWC.pWInfo = pWC->pWInfo; @@ -4177,13 +4321,13 @@ }else{ continue; } sCur.n = 0; #ifdef WHERETRACE_ENABLED - WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); - if( sqlite3WhereTrace & 0x400 ){ + if( sqlite3WhereTrace & 0x20000 ){ sqlite3WhereClausePrint(sSubBuild.pWC); } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ @@ -4241,11 +4385,11 @@ pNew->rRun = sSum.a[i].rRun + 1; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } - WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm)); } } return rc; } @@ -4589,12 +4733,12 @@ if( iColumn>=XN_ROWID ){ if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ - Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; - if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ + Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ continue; } } if( iColumn!=XN_ROWID ){ pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); @@ -4722,41 +4866,60 @@ ** Return the cost of sorting nRow rows, assuming that the keys have ** nOrderby columns and that the first nSorted columns are already in ** order. */ static LogEst whereSortingCost( - WhereInfo *pWInfo, - LogEst nRow, - int nOrderBy, - int nSorted + WhereInfo *pWInfo, /* Query planning context */ + LogEst nRow, /* Estimated number of rows to sort */ + int nOrderBy, /* Number of ORDER BY clause terms */ + int nSorted /* Number of initial ORDER BY terms naturally in order */ ){ - /* TUNING: Estimated cost of a full external sort, where N is + /* Estimated cost of a full external sort, where N is ** the number of rows to sort is: ** - ** cost = (3.0 * N * log(N)). + ** cost = (K * N * log(N)). ** ** Or, if the order-by clause has X terms but only the last Y ** terms are out of order, then block-sorting will reduce the ** sorting cost to: ** - ** cost = (3.0 * N * log(N)) * (Y/X) + ** cost = (K * N * log(N)) * (Y/X) + ** + ** The constant K is at least 2.0 but will be larger if there are a + ** large number of columns to be sorted, as the sorting time is + ** proportional to the amount of content to be sorted. The algorithm + ** does not currently distinguish between fat columns (BLOBs and TEXTs) + ** and skinny columns (INTs). It just uses the number of columns as + ** an approximation for the row width. ** - ** The (Y/X) term is implemented using stack variable rScale - ** below. + ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort + ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert. */ - LogEst rScale, rSortCost; - assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 16; + LogEst rSortCost, nCol; + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->pEList!=0 ); + /* TUNING: sorting cost proportional to the number of output columns: */ + nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30); + rSortCost = nRow + nCol; + if( nSorted>0 ){ + /* Scale the result by (Y/X) */ + rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + } /* Multiple by log(M) where M is the number of output rows. ** Use the LIMIT for M if it is smaller. Or if this sort is for ** a DISTINCT operator, M will be the number of distinct output ** rows, so fudge it downwards a bit. */ - if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ + rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */ + if( nSorted!=0 ){ + rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */ + } + if( pWInfo->iLimitiLimit; + } }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT ** reduces the number of output rows by a factor of 2 */ if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); } } @@ -4904,15 +5067,15 @@ if( aSortCost[isOrdered]==0 ){ aSortCost[isOrdered] = whereSortingCost( pWInfo, nRowEst, nOrderBy, isOrdered ); } - /* TUNING: Add a small extra penalty (5) to sorting as an + /* TUNING: Add a small extra penalty (3) to sorting as an ** extra encouragment to the query planner to select a plan ** where the rows emerge in the correct order without any sorting ** required. */ - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, rUnsorted, rCost)); @@ -5256,11 +5419,11 @@ if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS; #ifdef SQLITE_DEBUG pLoop->cId = '0'; #endif #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); } #endif return 1; } @@ -5386,11 +5549,11 @@ break; } } } if( pTerm drop loop %c not used\n", pLoop->cId)); + WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId)); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ pTerm->wtFlags |= TERM_CODED; } @@ -5446,11 +5609,11 @@ && (pTab->tabFlags & TF_HasStat1)!=0 ){ testcase( pItem->fg.jointype & JT_LEFT ); pLoop->wsFlags |= WHERE_BLOOMFILTER; pLoop->wsFlags &= ~WHERE_IDX_ONLY; - WHERETRACE(0xffff, ( + WHERETRACE(0xffffffff, ( "-> use Bloom-filter on loop %c because there are ~%.1e " "lookups into %s which has only ~%.1e rows\n", pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, (double)sqlite3LogEstToInt(pTab->nRowLogEst))); } @@ -5459,17 +5622,17 @@ } } /* ** This is an sqlite3ParserAddCleanup() callback that is invoked to -** free the Parse->pIdxExpr list when the Parse object is destroyed. +** free the Parse->pIdxEpr list when the Parse object is destroyed. */ static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ Parse *pParse = (Parse*)pObject; - while( pParse->pIdxExpr!=0 ){ - IndexedExpr *p = pParse->pIdxExpr; - pParse->pIdxExpr = p->pIENext; + while( pParse->pIdxEpr!=0 ){ + IndexedExpr *p = pParse->pIdxEpr; + pParse->pIdxEpr = p->pIENext; sqlite3ExprDelete(db, p->pExpr); sqlite3DbFreeNN(db, p); } } @@ -5477,17 +5640,17 @@ ** The index pIdx is used by a query and contains one or more expressions. ** In other words pIdx is an index on an expression. iIdxCur is the cursor ** number for the index and iDataCur is the cursor number for the corresponding ** table. ** -** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for ** each of the expressions in the index so that the expression code generator ** will know to replace occurrences of the indexed expression with ** references to the corresponding column of the index. */ static SQLITE_NOINLINE void whereAddIndexedExpr( - Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ Index *pIdx, /* The index-on-expression that contains the expressions */ int iIdxCur, /* Cursor number for pIdx */ SrcItem *pTabItem /* The FROM clause entry for the table */ ){ int i; @@ -5512,20 +5675,26 @@ continue; } if( sqlite3ExprIsConstant(pExpr) ) continue; p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; - p->pIENext = pParse->pIdxExpr; + p->pIENext = pParse->pIdxEpr; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr); + } +#endif p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; p->bMaybeNullRow = bMaybeNullRow; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS p->zIdxName = pIdx->zName; #endif - pParse->pIdxExpr = p; + pParse->pIdxEpr = p; if( p->pIENext==0 ){ sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); } } } @@ -5813,30 +5982,30 @@ } } /* Construct the WhereLoop objects */ #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0xffff ){ + if( sqlite3WhereTrace & 0xffffffff ){ sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); if( wctrlFlags & WHERE_USE_LIMIT ){ sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); - if( sqlite3WhereTrace & 0x100 ){ + if( sqlite3WhereTrace & 0x8000 ){ Select sSelect; memset(&sSelect, 0, sizeof(sSelect)); sSelect.selFlags = SF_WhereBegin; sSelect.pSrc = pTabList; sSelect.pWhere = pWhere; sSelect.pOrderBy = pOrderBy; sSelect.pEList = pResultSet; sqlite3TreeViewSelect(0, &sSelect, 0); } - } - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ - sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); - sqlite3WhereClausePrint(sWLB.pWC); + if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } } #endif if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); @@ -5848,11 +6017,11 @@ ** changed based on STAT4 information while computing subsequent loops, ** then we need to rerun the whole loop building process so that all ** loops will be built using the revised truthProb values. */ if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); - WHERETRACE(0xffff, + WHERETRACE(0xffffffff, ("**** Redo all loop computations due to" " TERM_HIGHTRUTH changes ****\n")); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; @@ -5934,15 +6103,15 @@ ){ whereCheckIfBloomFilterIsUseful(pWInfo); } #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */ sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); sqlite3WhereClausePrint(sWLB.pWC); } - WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n")); #endif pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. @@ -6472,13 +6641,20 @@ last = iEnd; }else{ last = pWInfo->iEndWhere; } if( pIdx->bHasExpr ){ - IndexedExpr *p = pParse->pIdxExpr; + IndexedExpr *p = pParse->pIdxEpr; while( p ){ if( p->iIdxCur==pLevel->iIdxCur ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n", + p->iIdxCur, p->iIdxCol); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr); + } +#endif p->iDataCur = -1; p->iIdxCur = -1; } p = p->pIENext; } Index: src/whereInt.h ================================================================== --- src/whereInt.h +++ src/whereInt.h @@ -632,7 +632,8 @@ #define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */ #define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ #define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -268,10 +268,12 @@ } sqlite3_str_append(&str, ")", 1); zMsg = sqlite3StrAccumFinish(&str); ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + + sqlite3VdbeScanStatus(v, sqlite3VdbeCurrentAddr(v)-1, 0, 0, 0, 0); return ret; } #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -290,18 +292,31 @@ WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ const char *zObj = 0; WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; + + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; } sqlite3VdbeScanStatus( v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } + } } #endif /* @@ -357,11 +372,11 @@ pTerm->wtFlags |= TERM_LIKECOND; }else{ pTerm->wtFlags |= TERM_CODED; } #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("DISABLE-"); sqlite3WhereTermPrint(pTerm, (int)(pTerm - (pTerm->pWC->a))); } #endif if( pTerm->iParent<0 ) break; @@ -1344,17 +1359,19 @@ pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x800 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); - sqlite3WhereLoopPrint(pLoop, pWC); + if( sqlite3WhereTrace & 0x1000 ){ + sqlite3WhereLoopPrint(pLoop, pWC); + } } - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ if( iLevel==0 ){ sqlite3DebugPrintf("WHERE clause being coded:\n"); sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); } sqlite3DebugPrintf("All WHERE-clause terms before coding:\n"); @@ -2274,11 +2291,11 @@ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + WHERETRACE(0xffffffff, ("Subplan for OR-clause:\n")); pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, 0, WHERE_OR_SUBCLAUSE, iCovCur); assert( pSubWInfo || pParse->nErr ); if( pSubWInfo ){ WhereLoop *pSubLoop; @@ -2511,16 +2528,16 @@ VdbeCoverageIf(v, (x&1)==1); VdbeCoverageIf(v, (x&1)==0); } #endif } -#ifdef WHERETRACE_ENABLED /* 0xffff */ +#ifdef WHERETRACE_ENABLED /* 0xffffffff */ if( sqlite3WhereTrace ){ VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", pWC->nTerm-j, pTerm, iLoop)); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("Coding auxiliary constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } #endif sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); @@ -2545,12 +2562,12 @@ if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; if( pTerm->leftCursor!=iCur ) continue; if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ) continue; pE = pTerm->pExpr; -#ifdef WHERETRACE_ENABLED /* 0x800 */ - if( sqlite3WhereTrace & 0x800 ){ +#ifdef WHERETRACE_ENABLED /* 0x4001 */ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("Coding transitive constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } #endif assert( !ExprHasProperty(pE, EP_OuterON) ); @@ -2661,17 +2678,17 @@ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); pTerm->wtFlags |= TERM_CODED; } } -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x20000 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", iLevel); sqlite3WhereClausePrint(pWC); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", iLevel, (u64)pLevel->notReady); } #endif return pLevel->notReady; Index: src/window.c ================================================================== --- src/window.c +++ src/window.c @@ -1067,11 +1067,11 @@ } pSub = sqlite3SelectNew( pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 ); - SELECTTRACE(1,pParse,pSub, + TREETRACE(0x40,pParse,pSub, ("New window-function subquery in FROM clause of (%u/%p)\n", p->selId, p)); p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside ** of sqlite3DbMallocRawNN() called from Index: test/affinity3.test ================================================================== --- test/affinity3.test +++ test/affinity3.test @@ -102,22 +102,22 @@ SELECT * FROM map_integer UNION SELECT * FROM map_text; CREATE TABLE mzed AS SELECT * FROM idmap; } -#do_execsql_test affinity3-210 { - #PRAGMA automatic_index=ON; - #SELECT * FROM data JOIN idmap USING(id); -#} {1 abc a 4 xyz e} +do_execsql_test affinity3-210 { + PRAGMA automatic_index=ON; + SELECT * FROM data JOIN idmap USING(id); +} {4 xyz e} do_execsql_test affinity3-220 { SELECT * FROM data JOIN mzed USING(id); -} {1 abc a 4 xyz e} +} {4 xyz e} do_execsql_test affinity3-250 { PRAGMA automatic_index=OFF; SELECT * FROM data JOIN idmap USING(id); -} {1 abc a 4 xyz e} +} {4 xyz e} do_execsql_test affinity3-260 { SELECT * FROM data JOIN mzed USING(id); -} {1 abc a 4 xyz e} +} {4 xyz e} finish_test ADDED test/basexx1.test Index: test/basexx1.test ================================================================== --- /dev/null +++ test/basexx1.test @@ -0,0 +1,155 @@ +# 2022 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix basexx + +if {[catch {load_static_extension db basexx} error]} { + puts "Skipping basexx tests, hit load error: $error" + finish_test; return +} + +# Empty blobs encode to empty strings. +do_execsql_test 100 { + SELECT base64(x'')||base85(x''); +} {{}} + +# Empty strings decode to empty blobs. +do_execsql_test 101 { + SELECT hex(x'01'||base64('')||base85('')||x'02'); +} {0102} + +# Basic base64 encoding +do_execsql_test 102 { + SELECT base64(x'000102030405'); + SELECT base64(x'0001020304'); + SELECT base64(x'00010203'); +} {{AAECAwQF +} {AAECAwQ= +} {AAECAw== +}} + +# Basic base64 decoding with pad chars +do_execsql_test 103 { + SELECT hex(base64('AAECAwQF')); + SELECT hex(base64('AAECAwQ=')); + SELECT hex(base64('AAECAw==')); +} {000102030405 0001020304 00010203} + +# Basic base64 decoding without pad chars and with whitespace +do_execsql_test 104 { + SELECT hex(base64(' AAECAwQF ')); + SELECT hex(base64(' AAECAwQ')); + SELECT hex(base64('AAECAw ')); +} {000102030405 0001020304 00010203} + +# Basic base85 encoding +do_execsql_test 105 { + SELECT base85(x'000102030405'); + SELECT base85(x'0001020304'); + SELECT base85(x'00010203'); +} {{##/2,#2/ +} {##/2,#* +} {##/2, +}} + +# Basic base85 decoding with and without whitespace +do_execsql_test 106 { + SELECT hex(base85('##/2,#2/')); + SELECT hex(base85('##/2,#*')); + SELECT hex(base85('##/2,')); + SELECT hex(base85(' ##/2,#2/ ')); + SELECT hex(base85(' ##/2,#*')); + SELECT hex(base85('##/2, ')); +} {000102030405 0001020304 00010203 000102030405 0001020304 00010203} + +# Round-trip some random blobs. +do_execsql_test 107 { + CREATE TEMP TABLE rb( len int, b blob ) STRICT; + INSERT INTO rb(len) VALUES (1),(2),(3),(4),(5),(150),(151),(152),(153),(1054); + UPDATE rb SET b = randomblob(len); + SELECT len, base64(base64(b))=b, base85(base85(b))=b + FROM rb ORDER BY len; +} {1 1 1 2 1 1 3 1 1 4 1 1 5 1 1 150 1 1 151 1 1 152 1 1 153 1 1 1054 1 1} + +# Same round-trip but with space or junk prepended and/or appended or not. +do_execsql_test 108 { + CREATE TEMP TABLE junk(j text, rank int); + INSERT INTO junk VALUES ('',0),(' ',1),('~',2); + SELECT len, base64(j.j||base64(b)||j.j)=b, base85(j.j||base85(b)||j.j)=b + FROM rb r, junk j WHERE j.rank=(r.len+r.len/25)%3 ORDER BY len; +} {1 1 1 2 1 1 3 1 1 4 1 1 5 1 1 150 1 1 151 1 1 152 1 1 153 1 1 1054 1 1} + +# Exercise the fail-on-too-large result feature. + +set inLimit [sqlite3_limit db SQLITE_LIMIT_LENGTH -1] +sqlite3_limit db SQLITE_LIMIT_LENGTH 1300 + +do_catchsql_test 109 { + SELECT len, base64(b) FROM rb WHERE len>200; +} {1 {blob expanded to base64 too big}} + +do_catchsql_test 110 { + SELECT len, base85(b) FROM rb WHERE len>200; +} {1 {blob expanded to base85 too big}} + +do_catchsql_test 111 { + SELECT length(base85(b))=1335 FROM rb WHERE len=1054; +} {1 {blob expanded to base85 too big}} + +sqlite3_limit db SQLITE_LIMIT_LENGTH $inLimit + +# Exercise is_base85(t) + +do_execsql_test 112 { + SELECT is_base85(' '||base85(x'123456')||char(10)), + is_base85('#$%&*+,-./0123456789:;<=>?@' + ||'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + ||'[\]^_`' + ||'abcdefghijklmnopqrstuvwxyz'), + is_base85('!'), is_base85('"'), is_base85(''''), is_base85('('), + is_base85(')'), is_base85(char(123)), is_base85('|'), is_base85(char(125)), + is_base85('~'), is_base85(char(127)); +} {1 1 0 0 0 0 0 0 0 0 0 0} + +do_execsql_test 113 { + SELECT is_base85(NULL) IS NULL; +} {1} + +do_catchsql_test 114 { + SELECT is_base85(1); +} {1 {is_base85 accepts only text or NULL}} + +do_catchsql_test 115 { + SELECT is_base85(1.1); +} {1 {is_base85 accepts only text or NULL}} + +do_catchsql_test 116 { + SELECT is_base85(x'00'); +} {1 {is_base85 accepts only text or NULL}} + +# Round-trip many bigger random blobs. + +do_execsql_test 117 { + CREATE TABLE bs(b blob, num); + INSERT INTO bs SELECT randomblob(4000 + n%3), n + FROM ( + WITH RECURSIVE seq(n) AS ( + VALUES(1) UNION ALL SELECT n+1 + FROM seq WHERE n<100 + ) SELECT n FROM seq); + SELECT num FROM bs WHERE base64(base64(b))!=b; + SELECT num FROM bs WHERE base85(base85(b))!=b; +} {} + +finish_test Index: test/cast.test ================================================================== --- test/cast.test +++ test/cast.test @@ -479,9 +479,80 @@ CREATE TABLE t0(c0); INSERT INTO t0(c0) VALUES (0); CREATE VIEW v1(c0, c1) AS SELECT CAST(0.0 AS NUMERIC), COUNT(*) OVER () FROM t0; SELECT v1.c0 FROM v1, t0 WHERE v1.c0=0; -} {0} +} {0.0} + +# Set the 2022-12-10 "reopen" of ticket [https://sqlite.org/src/tktview/57c47526c3] +# +do_execsql_test cast-9.1 { + CREATE TABLE dual(dummy TEXT); + INSERT INTO dual VALUES('X'); + SELECT CAST(4 AS NUMERIC); +} {4} +do_execsql_test cast-9.2 { + SELECT CAST(4.0 AS NUMERIC); +} {4.0} +do_execsql_test cast-9.3 { + SELECT CAST(4.5 AS NUMERIC); +} {4.5} +do_execsql_test cast-9.4 { + SELECT x, typeof(x) FROM (SELECT CAST(4 AS NUMERIC) AS x) JOIN dual; +} {4 integer} +do_execsql_test cast-9.5 { + SELECT x, typeof(x) FROM dual CROSS JOIN (SELECT CAST(4 AS NUMERIC) AS x); +} {4 integer} +do_execsql_test cast-9.10 { + SELECT x, typeof(x) FROM (SELECT CAST(4.0 AS NUMERIC) AS x) JOIN dual; +} {4.0 real} +do_execsql_test cast-9.11 { + SELECT x, typeof(x) FROM dual CROSS JOIN (SELECT CAST(4.0 AS NUMERIC) AS x); +} {4.0 real} +do_execsql_test cast-9.12 { + SELECT x, typeof(x) FROM (SELECT CAST(4.5 AS NUMERIC) AS x) JOIN dual; +} {4.5 real} +do_execsql_test cast-9.13 { + SELECT x, typeof(x) FROM dual CROSS JOIN (SELECT CAST(4.5 AS NUMERIC) AS x); +} {4.5 real} +# 2022-12-15 dbsqlfuzz c9ee6f9a0a8b8fefb02cf69de2a8b67ca39525c8 +# +# Added a new SQLITE_AFF_FLEXNUM that does not try to convert int to real or +# real to int. +# +do_execsql_test cast-10.1 { + VALUES(CAST(44 AS REAL)),(55); +} {44.0 55} +do_execsql_test cast-10.2 { + SELECT CAST(44 AS REAL) AS 'm' UNION ALL SELECT 55; +} {44.0 55} +do_execsql_test cast-10.3 { + SELECT * FROM (VALUES(CAST(44 AS REAL)),(55)); +} {44.0 55} +do_execsql_test cast-10.4 { + SELECT * FROM (SELECT CAST(44 AS REAL) AS 'm' UNION ALL SELECT 55); +} {44.0 55} +do_execsql_test cast-10.5 { + SELECT * FROM dual CROSS JOIN (VALUES(CAST(44 AS REAL)),(55)); +} {X 44.0 X 55} +do_execsql_test cast-10.6 { + SELECT * FROM dual CROSS JOIN (SELECT CAST(44 AS REAL) AS 'm' + UNION ALL SELECT 55); +} {X 44.0 X 55} +do_execsql_test cast-10.7 { + DROP VIEW v1; + CREATE VIEW v1 AS SELECT CAST(44 AS REAL) AS 'm' UNION ALL SELECT 55; + SELECT name, type FROM pragma_table_info('v1'); +} {m NUM} +do_execsql_test cast-10.8 { + CREATE VIEW v2 AS VALUES(CAST(44 AS REAL)),(55); + SELECT type FROM pragma_table_info('v2'); +} {NUM} +do_execsql_test cast-10.9 { + SELECT * FROM v1; +} {44.0 55} +do_execsql_test cast-10.10 { + SELECT * FROM v2; +} {44.0 55} finish_test Index: test/eqp.test ================================================================== --- test/eqp.test +++ test/eqp.test @@ -92,29 +92,29 @@ do_eqp_test 1.7.1 { SELECT * FROM t3 JOIN (SELECT 1) } { QUERY PLAN - |--MATERIALIZE (subquery-xxxxxx) + |--CO-ROUTINE (subquery-xxxxxx) | `--SCAN CONSTANT ROW |--SCAN (subquery-xxxxxx) `--SCAN t3 } do_eqp_test 1.7.2 { SELECT * FROM t3 JOIN (SELECT 1) AS v1 } { QUERY PLAN - |--MATERIALIZE v1 + |--CO-ROUTINE v1 | `--SCAN CONSTANT ROW |--SCAN v1 `--SCAN t3 } do_eqp_test 1.7.3 { SELECT * FROM t3 AS xx JOIN (SELECT 1) AS yy } { QUERY PLAN - |--MATERIALIZE yy + |--CO-ROUTINE yy | `--SCAN CONSTANT ROW |--SCAN yy `--SCAN xx } @@ -121,11 +121,11 @@ do_eqp_test 1.8 { SELECT * FROM t3 JOIN (SELECT 1 UNION SELECT 2) } { QUERY PLAN - |--MATERIALIZE (subquery-xxxxxx) + |--CO-ROUTINE (subquery-xxxxxx) | `--COMPOUND QUERY | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW | `--UNION USING TEMP B-TREE | `--SCAN CONSTANT ROW @@ -134,11 +134,11 @@ } do_eqp_test 1.9 { SELECT * FROM t3 JOIN (SELECT 1 EXCEPT SELECT a FROM t3 LIMIT 17) AS abc } { QUERY PLAN - |--MATERIALIZE abc + |--CO-ROUTINE abc | `--COMPOUND QUERY | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW | `--EXCEPT USING TEMP B-TREE | `--SCAN t3 @@ -147,11 +147,11 @@ } do_eqp_test 1.10 { SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17) AS abc } { QUERY PLAN - |--MATERIALIZE abc + |--CO-ROUTINE abc | `--COMPOUND QUERY | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW | `--INTERSECT USING TEMP B-TREE | `--SCAN t3 @@ -161,11 +161,11 @@ do_eqp_test 1.11 { SELECT * FROM t3 JOIN (SELECT 1 UNION ALL SELECT a FROM t3 LIMIT 17) abc } { QUERY PLAN - |--MATERIALIZE abc + |--CO-ROUTINE abc | `--COMPOUND QUERY | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW | `--UNION ALL | `--SCAN t3 @@ -293,11 +293,11 @@ (SELECT * FROM t1 ORDER BY x LIMIT 10) AS x1, (SELECT * FROM t2 ORDER BY x LIMIT 10) AS x2 ORDER BY x2.y LIMIT 5 } { QUERY PLAN - |--MATERIALIZE x1 + |--CO-ROUTINE x1 | |--SCAN t1 | `--USE TEMP B-TREE FOR ORDER BY |--MATERIALIZE x2 | `--SCAN t2 USING INDEX t2i1 |--SCAN x1 @@ -832,11 +832,11 @@ WHERE blob.rid=thread.last AND event.objid=thread.last ORDER BY 1; } { QUERY PLAN - |--MATERIALIZE thread + |--CO-ROUTINE thread | |--SCAN x USING INDEX forumthread | |--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR | |--CORRELATED SCALAR SUBQUERY xxxxxx | | |--SEARCH forumpost USING COVERING INDEX forumthread (froot=?) | | `--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR Index: test/fts3expr4.test ================================================================== --- test/fts3expr4.test +++ test/fts3expr4.test @@ -48,11 +48,20 @@ # In "col:word", if "col" is not the name of a column, the entire thing # is passed to the tokenizer. # do_icu_expr_test 1.7 {a:word} {PHRASE 0 0 word} -do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} +# do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word} +do_test 1.8 { + set res [ + db one {SELECT fts3_exprtest('icu en_US', 'd:word', 'a', 'b', 'c')} + ] + expr { + $res=="PHRASE 3 0 d:word" || + $res=="AND {AND {PHRASE 3 0 d} {PHRASE 3 0 :}} {PHRASE 3 0 word}" + } +} 1 set sqlite_fts3_enable_parentheses 0 do_icu_expr_test 2.1 { f (e NEAR/2 a) Index: test/fuzzdata8.db ================================================================== --- test/fuzzdata8.db +++ test/fuzzdata8.db cannot compute difference between binary files Index: test/fuzzinvariants.c ================================================================== --- test/fuzzinvariants.c +++ test/fuzzinvariants.c @@ -134,20 +134,15 @@ sqlite3_finalize(pTestStmt); return SQLITE_CORRUPT; } sqlite3_finalize(pCk); - if( sqlite3_strlike("%group%by%order%by%desc%",sqlite3_sql(pStmt),0)==0 ){ - /* dbsqlfuzz crash-647c162051c9b23ce091b7bbbe5125ce5f00e922 - ** Original statement is: - ** - ** SELECT a,c,d,b,'' FROM t1 GROUP BY 1 HAVING d<>345 ORDER BY a DESC; - ** - ** The values of c, d, and b are indeterminate and change when the - ** enclosed in the test query because the DESC is dropped. - ** - ** SELECT * FROM (...) WHERE "a"==0 + if( sqlite3_strlike("%group%by%",sqlite3_sql(pStmt),0)==0 ){ + /* + ** If there is a GROUP BY clause, it might not cover every term in the + ** output. And then non-covered terms can take on a value from any + ** row in the result set. This can cause differing answers. */ goto not_a_fault; } if( sqlite3_strlike("%limit%)%order%by%", sqlite3_sql(pTestStmt),0)==0 ){ @@ -234,11 +229,11 @@ */ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ const char *zIn; size_t nIn; const char *zAnd = "WHERE"; - int i; + int i, j; sqlite3_str *pTest; sqlite3_stmt *pBase = 0; sqlite3 *db = sqlite3_db_handle(pStmt); int rc; int nCol = sqlite3_column_count(pStmt); @@ -278,10 +273,18 @@ && (zSuffix[1]>'3' || isdigit(zSuffix[2])) ){ /* This is a randomized column name and so cannot be used in the ** WHERE clause. */ continue; + } + for(j=0; j1 && i+2!=iCnt ) continue; if( zColName==0 ) continue; if( sqlite3_column_type(pStmt, i)==SQLITE_NULL ){ Index: test/indexexpr1.test ================================================================== --- test/indexexpr1.test +++ test/indexexpr1.test @@ -418,11 +418,11 @@ } {1 3} # 2018-01-03 OSSFuzz discovers another test case for the same problem # above. # -do_execsql_test indexexpr-1510 { +do_execsql_test indexexpr1-1510 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(a PRIMARY KEY,b UNIQUE); REPLACE INTO t1 VALUES(2, 1); REPLACE INTO t1 SELECT 6,1; CREATE INDEX t1aa ON t1(a-a); @@ -432,30 +432,30 @@ # 2018-01-31 https://www.sqlite.org/src/tktview/343634942dd54ab57b702411 # When an index on an expression depends on the string representation of # a numeric table column, trouble can arise since there are multiple # string that can map to the same numeric value. (Ex: 123, 0123, 000123). # -do_execsql_test indexexpr-1600 { +do_execsql_test indexexpr1-1600 { DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a INTEGER, b); CREATE INDEX idx1 ON t1 (lower(a)); INSERT INTO t1 VALUES('0001234',3); PRAGMA integrity_check; } {ok} -do_execsql_test indexexpr-1610 { +do_execsql_test indexexpr1-1610 { INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1); SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b; } {0 1 2 3} -do_execsql_test indexexpr-1620 { +do_execsql_test indexexpr1-1620 { SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b; } {} # 2019-08-09 https://www.sqlite.org/src/info/9080b6227fabb466 # ExprImpliesExpr theorem prover bug: # "(NULL IS FALSE) IS FALSE" does not imply "NULL IS NULL" # -do_execsql_test indexexpr-1700 { +do_execsql_test indexexpr1-1700 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0); INSERT INTO t0(c0) VALUES (0); CREATE INDEX i0 ON t0(NULL > c0) WHERE (NULL NOT NULL); SELECT * FROM t0 WHERE ((NULL IS FALSE) IS FALSE); @@ -466,21 +466,21 @@ # table column of type REAL that is actually holding an MEM_IntReal # value, be sure to use the REAL value and not the INT value when # computing the expression. # ifcapable like_match_blobs { - do_execsql_test indexexpr-1800 { + do_execsql_test indexexpr1-1800 { DROP TABLE IF EXISTS t0; CREATE TABLE t0(c0 REAL, c1 TEXT); CREATE INDEX i0 ON t0(+c0, c0); INSERT INTO t0(c0) VALUES(0); SELECT CAST(+ t0.c0 AS BLOB) LIKE 0 FROM t0; } {0} - do_execsql_test indexexpr-1810 { + do_execsql_test indexexpr1-1810 { SELECT CAST(+ t0.c0 AS BLOB) LIKE '0.0' FROM t0; } {1} - do_execsql_test indexexpr-1820 { + do_execsql_test indexexpr1-1820 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(x REAL); CREATE INDEX t1x ON t1(x, +x); INSERT INTO t1(x) VALUES(2); SELECT +x FROM t1 WHERE x=2; @@ -489,21 +489,82 @@ # 2022-04-30 https://sqlite.org/forum/info/7efabf4b03328e57 # Assertion fault during a DELETE INDEXED BY. # reset_db -do_execsql_test indexexpr-1900 { +do_execsql_test indexexpr1-1900 { CREATE TABLE t1(x TEXT PRIMARY KEY, y TEXT, z INT); INSERT INTO t1(x,y,z) VALUES('alpha','ALPHA',1),('bravo','charlie',1); CREATE INDEX i1 ON t1(+y COLLATE NOCASE); SELECT * FROM t1; } {alpha ALPHA 1 bravo charlie 1} -do_execsql_test indexexpr-1910 { +do_execsql_test indexexpr1-1910 { DELETE FROM t1 INDEXED BY i1 WHERE x IS +y COLLATE NOCASE IN (SELECT z FROM t1) RETURNING *; } {alpha ALPHA 1} -do_execsql_test indexexpr-1920 { +do_execsql_test indexexpr1-1920 { SELECT * FROM t1; } {bravo charlie 1} + +# 2022-11-28 Ticket 695a1a53de +# Improved ability to recognize that an index on an expression is a +# covering index. +# +reset_db +do_execsql_test indexexpr1-2000 { + CREATE TABLE t1(a INT, b TEXT); + INSERT INTO t1(a,b) VALUES + (10, '{"one":5,"two":6}'), + (10, '{"one":50,"two":60}'), + (10, '{"three":99}'), + (11, '{"one":100,"two":200}'); + CREATE INDEX t1_one ON t1(a, b->>'one'); + CREATE INDEX t1_two ON t1(a, b->>'two'); +} +do_execsql_test indexexpr1-2010 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {/.* t1_one .*/} +do_execsql_test indexexpr1-2011 { + SELECT sum(b->>'one') FROM t1 WHERE a=10; /* Query AA */ +} {55} +do_execsql_test indexexpr1-2020 { + EXPLAIN QUERY PLAN + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {/.* t1_two .*/} +do_execsql_test indexexpr1-2021 { + SELECT sum(b->>'two') FROM t1 WHERE a=10; /* Query BB */ +} {66} +do_execsql_test indexexpr1-2030 { + DROP TABLE t1; + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} +do_execsql_test indexexpr1-2030 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {1 6 4 54 46} +do_execsql_test indexexpr1-2030 { + explain query plan + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1; +} {/.*SCAN t1 USING INDEX t1x.*/} + + finish_test Index: test/intreal.test ================================================================== --- test/intreal.test +++ test/intreal.test @@ -93,24 +93,6 @@ UPDATE OR REPLACE t0 SET c0 = 'a', c1 = ''; SELECT * FROM t0 ORDER BY t0.c1; PRAGMA integrity_check; } {a {} ok} - -reset_db -do_execsql_test 4.0 { - CREATE TABLE t1(a REAL, b AS ('expr') ); -} -do_execsql_test 4.1 { - INSERT INTO t1 VALUES( REPLACE(0, '', 'expr') ); -} -do_execsql_test 4.2 { - INSERT INTO t1 SELECT REPLACE(4, '', 'expr'); -} -do_execsql_test 4.3 { - SELECT typeof(a), a FROM t1; -} { - real 0.0 - real 4.0 -} - finish_test ADDED test/memdb2.test Index: test/memdb2.test ================================================================== --- /dev/null +++ test/memdb2.test @@ -0,0 +1,77 @@ +# 2022-12-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is the "memdb" VFS +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix memdb2 +do_not_use_codec + +ifcapable !deserialize { + finish_test + return +} + +db close + +#------------------------------------------------------------------------- +# Test that when using a memdb database, it is not possible to upgrade +# to an EXCLUSIVE lock if some other client is holding SHARED. +# +foreach {tn fname} { + 1 file:/test.db?vfs=memdb + 2 file:\\test.db?vfs=memdb +} { + if {$tn==2} breakpoint + sqlite3 db $fname -uri 1 + sqlite3 db2 $fname -uri 1 + + + do_execsql_test 1.$tn.1 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + } + + do_execsql_test -db db2 1.$tn.2 { + BEGIN; + SELECT * FROM t1; + } {1 2} + + do_execsql_test 1.$tn.3 { + BEGIN; + INSERT INTO t1 VALUES(3, 4); + } + + do_catchsql_test 1.$tn.4 { + COMMIT + } {1 {database is locked}} + + do_execsql_test -db db2 1.$tn.5 { + SELECT * FROM t1; + END; + } {1 2} + + do_execsql_test 1.$tn.6 { + COMMIT + } {} + + do_execsql_test -db db2 1.$tn.7 { + SELECT * FROM t1 + } {1 2 3 4} + + db close + db2 close +} + +finish_test + Index: test/misc1.test ================================================================== --- test/misc1.test +++ test/misc1.test @@ -603,32 +603,35 @@ do_execsql_test misc1-19.2 { CREATE TABLE t19b AS SELECT 4 AS '', 5 AS '', 6 AS ''; SELECT * FROM t19b; } {4 5 6} -# 2015-05-20: CREATE TABLE AS should not store INT value is a TEXT +# 2015-05-20: CREATE TABLE AS should not store INT value in a TEXT # column. +# +# 2022-12-14: Change: The column is not TEXT if the AS SELECT is +# a compound with different types on each arm. # do_execsql_test misc1-19.3 { CREATE TABLE t19c(x TEXT); CREATE TABLE t19d AS SELECT * FROM t19c UNION ALL SELECT 1234; SELECT x, typeof(x) FROM t19d; -} {1234 text} +} {1234 integer} # 2014-05-16: Tests for the SQLITE_TESTCTRL_FAULT_INSTALL feature. # unset -nocomplain fault_callbacks set fault_callbacks {} proc fault_callback {n} { lappend ::fault_callbacks $n return 0 } -do_test misc1-19.1 { +do_test misc1-19.11 { sqlite3_test_control_fault_install fault_callback set fault_callbacks } {0} -do_test misc1-19.2 { +do_test misc1-19.12 { sqlite3_test_control_fault_install set fault_callbacks } {0} # 2015-01-26: Valgrind-detected over-read. Index: test/multiplex3.test ================================================================== --- test/multiplex3.test +++ test/multiplex3.test @@ -80,10 +80,12 @@ do_test 1.0 { setup_and_save_db } {} do_faultsim_test 1 -prep { multiplex_restore_db sqlite3 db file:test.db?8_3_names=1 sqlite3_multiplex_control db main chunk_size [expr 256*1024] + execsql { PRAGMA journal_mode = truncate } + execsql { PRAGMA synchronous = off } } -body { execsql { UPDATE t1 SET a=randomblob(12), b=randomblob(1500) WHERE (rowid%32)=0 } } -test { Index: test/orderby1.test ================================================================== --- test/orderby1.test +++ test/orderby1.test @@ -41,10 +41,11 @@ (NULL, 2, 2, 'two-b'), (NULL, 3, 3, 'three-c'), (NULL, 1, 3, 'one-c'), (NULL, 2, 1, 'two-a'), (NULL, 3, 1, 'three-a'); + ANALYZE; COMMIT; } } {} do_test 1.1a { db eval { @@ -178,10 +179,11 @@ (20, 2, 'two-b'), (3, 3, 'three-c'), (1, 3, 'one-c'), (20, 1, 'two-a'), (3, 1, 'three-a'); + ANALYZE; COMMIT; } } {} do_test 2.1a { db eval { @@ -325,10 +327,11 @@ (NULL, 2, 2, 'two-b'), (NULL, 3, 3, 'three-c'), (NULL, 1, 3, 'one-c'), (NULL, 2, 1, 'two-a'), (NULL, 3, 1, 'three-a'); + ANALYZE; COMMIT; } } {} do_test 3.1a { db eval { Index: test/pushdown.test ================================================================== --- test/pushdown.test +++ test/pushdown.test @@ -84,8 +84,43 @@ ) AND f('three')=123 } set L } {three} - +# 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f +# Disable the push-down optimization for compound subqueries if any +# arm of the compound has an incompatible affinity. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t0(c0 INT); + INSERT INTO t0 VALUES(0); + CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1_a VALUES(1,'one'); + CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t1_b VALUES(2,'two'); + CREATE VIEW v0 AS SELECT CAST(t0.c0 AS INTEGER) AS c0 FROM t0; + CREATE VIEW v1(a,b) AS SELECT a, b FROM t1_a UNION ALL SELECT c, 0 FROM t1_b; + SELECT v1.a, quote(v1.b), t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0,v1; +} { + 1 'one' 0 + 2 0 0 +} +do_execsql_test 3.2 { + SELECT a, quote(b), cd FROM ( + SELECT v1.a, v1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0, v1 + ) WHERE a=2 AND b='0' AND cd=0; +} {} +do_execsql_test 3.3 { + SELECT a, quote(b), cd FROM ( + SELECT v1.a, v1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0, v1 + ) WHERE a=1 AND b='one' AND cd=0; +} {1 'one' 0} +do_execsql_test 3.4 { + SELECT a, quote(b), cd FROM ( + SELECT v1.a, v1.b, t0.c0 AS cd FROM t0 LEFT JOIN v0 ON v0.c0!=0, v1 + ) WHERE a=2 AND b=0 AND cd=0; +} { + 2 0 0 +} finish_test Index: test/scanstatus.test ================================================================== --- test/scanstatus.test +++ test/scanstatus.test @@ -34,11 +34,13 @@ set idx 0 set ret [list] while {1} { set r [sqlite3_stmt_scanstatus $stmt $idx] if {[llength $r]==0} break - lappend ret {*}$r + foreach v {nLoop nVisit nEst zName zExplain} { + lappend ret $v [dict get $r $v] + } incr idx } uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] } @@ -310,12 +312,12 @@ do_execsql_test 5.1.1 { SELECT count(*) FROM t1 WHERE a IN (SELECT b FROM t1 AS ii) } {2} do_scanstatus_test 5.1.2 { - nLoop 1 nVisit 10 nEst 10.0 zName t1bc - zExplain {SCAN ii USING COVERING INDEX t1bc} + nLoop 1 nVisit 10 nEst 10.0 zName t1 + zExplain {SCAN ii} nLoop 1 nVisit 2 nEst 8.0 zName sqlite_autoindex_t1_1 zExplain {SEARCH t1 USING COVERING INDEX sqlite_autoindex_t1_1 (a=?)} } do_execsql_test 5.2.1 { @@ -339,19 +341,19 @@ do_eqp_test 5.4.1 { SELECT count(*) FROM t1, t2 WHERE y = c; } { QUERY PLAN - |--SCAN t1 USING COVERING INDEX t1bc + |--SCAN t1 `--SEARCH t2 USING COVERING INDEX t2xy (ANY(x) AND y=?) } do_execsql_test 5.4.2 { SELECT count(*) FROM t1, t2 WHERE y = c; } {200} do_scanstatus_test 5.4.3 { - nLoop 1 nVisit 10 nEst 10.0 zName t1bc - zExplain {SCAN t1 USING COVERING INDEX t1bc} + nLoop 1 nVisit 10 nEst 10.0 zName t1 + zExplain {SCAN t1} nLoop 10 nVisit 200 nEst 56.0 zName t2xy zExplain {SEARCH t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} } do_eqp_test 5.5.1 { ADDED test/scanstatus2.test Index: test/scanstatus2.test ================================================================== --- /dev/null +++ test/scanstatus2.test @@ -0,0 +1,235 @@ +# 2022 December 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix scanstatus2 + +ifcapable !scanstatus { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t2 VALUES('e', 'f'); +} + +proc do_zexplain_test {v2 tn sql res} { + db eval $sql + set stmt [db version -last-stmt-ptr] + set idx 0 + set ret [list] + + set cmd sqlite3_stmt_scanstatus + set f [list] + if {$v2} { lappend f complex } + + while {1} { + set r [sqlite3_stmt_scanstatus -flags $f $stmt $idx] + if {[llength $r]==0} break + lappend ret [dict get $r zExplain] + incr idx + } + uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] +} + +proc get_cycles {stmt} { + set r [sqlite3_stmt_scanstatus $stmt -1] + dict get $r nCycle +} + +proc foreach_scan {varname stmt body} { + upvar $varname var + + for {set ii 0} {1} {incr ii} { + set r [sqlite3_stmt_scanstatus -flags complex $stmt $ii] + if {[llength $r]==0} break + array set var $r + uplevel $body + } +} + +proc get_eqp_graph {stmt iPar nIndent} { + set res "" + foreach_scan A $stmt { + if {$A(iParentId)==$iPar} { + set txt $A(zExplain) + if {$A(nCycle)>=0} { + append txt " (nCycle=$A(nCycle))" + } + append res "[string repeat - $nIndent]$txt\n" + append res [get_eqp_graph $stmt $A(iSelectId) [expr $nIndent+2]] + } + } + set res +} + +proc get_graph {stmt} { + set nCycle [get_cycles $stmt] + set res "QUERY (nCycle=$nCycle)\n" + append res [get_eqp_graph $stmt 0 2] +} + +proc do_graph_test {tn sql res} { + db eval $sql + set stmt [db version -last-stmt-ptr] + set graph [string trim [get_graph $stmt]] + + set graph [regsub -all {nCycle=[0-9]+} $graph nCycle=nnn] + uplevel [list do_test $tn [list set {} $graph] [string trim $res]] +} + +proc puts_graph {sql} { + db eval $sql + set stmt [db version -last-stmt-ptr] + puts [string trim [get_graph $stmt]] +} + + +do_zexplain_test 0 1.1 { + SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 +} { + {SCAN t2} + {SCAN t1} +} +do_zexplain_test 1 1.2 { + SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 +} { + {SCAN t2} + {CORRELATED SCALAR SUBQUERY 1} + {SCAN t1} +} + +do_graph_test 1.3 { + SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2 +} { +QUERY (nCycle=nnn) +--SCAN t2 (nCycle=nnn) +--CORRELATED SCALAR SUBQUERY 1 (nCycle=nnn) +----SCAN t1 (nCycle=nnn) +} + +do_graph_test 1.4 { + WITH v2(x,y) AS MATERIALIZED ( + SELECT x,y FROM t2 + ) + SELECT * FROM t1, v2 ORDER BY y; +} { +QUERY (nCycle=nnn) +--MATERIALIZE v2 (nCycle=nnn) +----SCAN t2 (nCycle=nnn) +--SCAN v2 (nCycle=nnn) +--SCAN t1 (nCycle=nnn) +--USE TEMP B-TREE FOR ORDER BY (nCycle=nnn) +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a); + INSERT INTO ft VALUES('abc'); + INSERT INTO ft VALUES('def'); + INSERT INTO ft VALUES('ghi'); +} + +do_graph_test 2.1 { + SELECT * FROM ft('def') +} { +QUERY (nCycle=nnn) +--SCAN ft VIRTUAL TABLE INDEX 0:M1 (nCycle=nnn) +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b); + CREATE TABLE x2(c, d); + + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000) + INSERT INTO x1 SELECT i, i FROM s; + INSERT INTO x2 SELECT a, b FROM x1; +} + +do_graph_test 2.1 { + SELECT * FROM x1, x2 WHERE c=+a; +} { +QUERY (nCycle=nnn) +--SCAN x1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON x2(c, d) (nCycle=nnn) +--SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE rt1 (id INTEGER PRIMARY KEY, x1, x2); + CREATE TABLE rt2 (id, x1, x2); +} + +do_graph_test 4.1 { + SELECT * FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=rt1.x1; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id, x2) (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.2 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=rt1.x1; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id) (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.3 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND (rt2.x1+1)=(rt1.x1+1); +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--SCAN rt2 (nCycle=nnn) +} + +do_graph_test 4.4 { + SELECT rt2.id FROM rt1, rt2 WHERE rt1.id%2 AND rt2.x1=(rt1.x1+1) AND rt2.id>5; +} { +QUERY (nCycle=nnn) +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON rt2(x1, id) WHERE (nCycle=nnn) +--SEARCH rt2 USING AUTOMATIC PARTIAL COVERING INDEX (x1=?) (nCycle=nnn) +} + +do_graph_test 4.5 { + SELECT v1.cnt FROM rt1, ( + SELECT count(*) AS cnt, rt2.x1 AS x1 FROM rt2 GROUP BY x1 + ) AS v1 WHERE rt1.x1=v1.x1 +} { +QUERY (nCycle=nnn) +--CO-ROUTINE v1 +----SCAN rt2 (nCycle=nnn) +----USE TEMP B-TREE FOR GROUP BY +--SCAN rt1 (nCycle=nnn) +--CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn) +--SEARCH v1 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) +} + +finish_test + + Index: test/sessionfuzz.c ================================================================== --- test/sessionfuzz.c +++ test/sessionfuzz.c @@ -697,11 +697,11 @@ #include #include #include #ifndef OMIT_ZLIB -#include "zlib.h" +#include #endif /* ** Implementation of the "sqlar_uncompress(X,SZ)" SQL function ** Index: test/shell1.test ================================================================== --- test/shell1.test +++ test/shell1.test @@ -1254,7 +1254,19 @@ select 2,1; select 3,4; } } {0 {1,2 2,1 3,4}} + +#---------------------------------------------------------------------------- +# Test cases shell1-10.*: Test that certain static extensions are there. +# +do_test shell1-10.1 { + catchcmd :memory: { +.mode list +.header off +select base64(base64(cast('digity-doo' as blob))), + base85(base85(cast('digity-doo' as blob))); +} +} {0 digity-doo|digity-doo} finish_test Index: test/shell4.test ================================================================== --- test/shell4.test +++ test/shell4.test @@ -122,10 +122,16 @@ catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;" } {0 {CREATE TABLE t1(x); SELECT * FROM t1;}} do_test shell4-2.5 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;" +} {0 {SELECT * FROM t1;}} +do_test shell4-2.6 { + catchcmd ":memory:" { +CREATE TABLE t1(x); +.trace --stmt stdout +SELECT * FROM t1;} } {0 {SELECT * FROM t1;}} } do_test shell4-3.1 { set fd [open t1.txt wb] Index: test/sort.test ================================================================== --- test/sort.test +++ test/sort.test @@ -593,6 +593,38 @@ reset_db do_execsql_test 17.1 { SELECT * FROM sqlite_master ORDER BY sql; } {} +# 2022-12-03 Ticket e8b674241947eb3b +# Improve estimates for the cost of sorting relative +# to the cost of doing an index lookup, so as to get +# a better query plan. See the ticket for a deetailed +# example. +# +reset_db +do_execsql_test 18.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<50) + -- increase to 5000 for actual test data ----^^ + INSERT INTO t1(a,b,c) SELECT x, random()%5000, random()%5000 FROM c; + CREATE TABLE t2(d,e,f); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<500) + -- increase to 50000 for actual test data -----^^^ + INSERT INTO t2(d,e,f) SELECT + NULLIF(0, random()%2), random()%5000, random()%5000 + FROM c; + ANALYZE; + UPDATE sqlite_stat1 SET stat='50000' WHERE tbl='t2'; + UPDATE sqlite_stat1 SET stat='5000' WHERE tbl='t1'; + ANALYZE sqlite_schema; +} {} +do_execsql_test 18.2 { + EXPLAIN QUERY PLAN + SELECT a FROM t1 JOIN t2 + WHERE a IN (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) + AND a=CASE WHEN d IS NOT NULL THEN e ELSE f END + ORDER BY a; +} {/.*SCAN t2.*SEARCH t1.*/} +# ^^^^^^^--^^^^^^^^^--- t2 should be the outer loop. + finish_test ADDED test/tkt-99378177930f87bd.test Index: test/tkt-99378177930f87bd.test ================================================================== --- /dev/null +++ test/tkt-99378177930f87bd.test @@ -0,0 +1,179 @@ +# 2022-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to verify that the enhancement +# request documented by ticket 99378177930f87bd is working. +# +# The enhancement is that if an aggregate query with a GROUP BY clause +# uses subexpressions in the arguments to aggregate functions that are +# also columns of an index, then the values are pulled from the index +# rather than being recomputed. This has the potential to make some +# indexed queries works as if the index were covering. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test tkt-99378-100 { + CREATE TABLE t1(a INT, b TEXT, c INT, d INT); + INSERT INTO t1(a,b,c,d) VALUES + (1, '{"x":1}', 12, 3), + (1, '{"x":2}', 4, 5), + (1, '{"x":1}', 6, 11), + (2, '{"x":1}', 22, 3), + (2, '{"x":2}', 4, 5), + (3, '{"x":1}', 6, 7); + CREATE INDEX t1x ON t1(d, a, b->>'x', c); +} {} +do_execsql_test tkt-99378-110 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} + +# The proof that the index on the expression is being used is in the +# fact that the byte code contains no "Function" opcodes. In other words, +# the ->> operator (which is implemented by a function) is never invoked. +# Instead, the b->>'x' value is pulled out of the index. +# +do_execsql_test tkt-99378-120 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + + +do_execsql_test tkt-99378-130 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-140 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY +a; +} {~/Function/} + +do_execsql_test tkt-99378-200 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(a, d, b->>'x', c); +} +do_execsql_test tkt-99378-210 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-220 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} +do_execsql_test tkt-99378-230 { + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} +do_execsql_test tkt-99378-240 { + EXPLAIN + SELECT a, + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} + +# 2022-12-20 dbsqlfuzz a644e70d7683a7ca59c71861a153c1dccf8850b9 +# +do_execsql_test tkt-99378-300 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INT); + CREATE INDEX i1 ON t1(a,a=a); + INSERT INTO t1 VALUES(1),(2),(3),(4); + SELECT * FROM t1 NATURAL JOIN t1 + WHERE a==1 + OR ( + (SELECT avg( + (SELECT sum((SELECT 1 FROM t1 NATURAL RIGHT JOIN t1 WHERE a=a)))) AS xyz + ) + AND a==2 + ); +} {1 2} +do_execsql_test tkt-99378-310 { + DROP INDEX i1; + SELECT * FROM t1 NATURAL JOIN t1 + WHERE a==1 + OR ( + (SELECT avg( + (SELECT sum((SELECT 1 FROM t1 NATURAL RIGHT JOIN t1 WHERE a=a)))) AS xyz + ) + AND a==2 + ); +} {1 2} + +finish_test Index: test/unionall.test ================================================================== --- test/unionall.test +++ test/unionall.test @@ -406,39 +406,41 @@ CREATE VIEW v0(c0) AS SELECT CAST(t0.c0 AS INTEGER) FROM t0; CREATE VIEW t1 AS SELECT a, b FROM t1_a UNION ALL SELECT c, c FROM t1_b UNION ALL SELECT e, f FROM t1_c; -} + SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1; +} {1 one 0 {} 4 four 0 {} 2 2 0 {} 5 5 0 {} 3 three 0 {} 6 six 0 {}} + optimization_control db all 1 do_execsql_test 8.2 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2; } {2 2 0 {}} do_execsql_test 8.3 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2.0; -} {} +} {2 2 0 {}} do_execsql_test 8.4 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b='2'; -} {2 2 0 {}} +} {} optimization_control db query-flattener,push-down 0 do_execsql_test 8.5 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2; } {2 2 0 {}} do_execsql_test 8.6 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2.0; -} {} +} {2 2 0 {}} do_execsql_test 8.7 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b='2'; -} {2 2 0 {}} +} {} optimization_control db all 0 do_execsql_test 8.8 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2; } {2 2 0 {}} do_execsql_test 8.9 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b=2.0; -} {} +} {2 2 0 {}} do_execsql_test 8.10 { SELECT * FROM (SELECT t1.a, t1.b, t0.c0 AS c, v0.c0 AS d FROM t0 LEFT JOIN v0 ON v0.c0>'0',t1) WHERE b='2'; -} {2 2 0 {}} +} {} finish_test Index: test/view.test ================================================================== --- test/view.test +++ test/view.test @@ -121,20 +121,20 @@ CREATE VIEW v9c(x) AS SELECT x FROM t9; CREATE VIEW v9d(x) AS SELECT * FROM t9; } {} do_execsql_test view-1.11 { PRAGMA table_info(v9a); -} {0 x INTEGER 0 {} 0} +} {0 x INT 0 {} 0} do_execsql_test view-1.12 { PRAGMA table_info(v9b); -} {0 x INTEGER 0 {} 0} +} {0 x INT 0 {} 0} do_execsql_test view-1.13 { PRAGMA table_info(v9c); -} {0 x INTEGER 0 {} 0} +} {0 x INT 0 {} 0} do_execsql_test view-1.14 { PRAGMA table_info(v9d); -} {0 x INTEGER 0 {} 0} +} {0 x INT 0 {} 0} do_test view-2.1 { execsql { CREATE VIEW v2 AS SELECT * FROM t1 WHERE a>5 }; # No semicolon @@ -773,7 +773,32 @@ do_catchsql_test view-29.1 { CREATE TABLE t2(c,d,e); SELECT name FROM sqlite_schema ORDER BY name; } {0 {t1 t2}} +#------------------------------------------------------------------------- +# 2022-12-11. https://sqlite.org/src/info/679ed6a2 +# +# 2022-12-14 change: If the AS SELECT of a VIEW is a compound where +# the datatypes on each arm of the compound are different, then the +# datatype of the overall column is BLOB (ANY). +# +reset_db +do_execsql_test view-30.0 { + CREATE TABLE t0(a INT, b TEXT); + + INSERT INTO t0 VALUES(1,'one'); + + CREATE VIEW t1 AS SELECT a, b FROM t0 UNION ALL SELECT 2, 2; + CREATE VIEW t2(a,b) AS SELECT a, b FROM t0 UNION ALL SELECT 2, 2; +} + +ifcapable schema_pragmas { + do_execsql_test view-30.1 { + PRAGMA table_info = t1; + } { 0 a INT 0 {} 0 1 b BLOB 0 {} 0 } + do_execsql_test view-30.2 { + PRAGMA table_info = t2; + } { 0 a INT 0 {} 0 1 b BLOB 0 {} 0 } +} finish_test Index: test/where.test ================================================================== --- test/where.test +++ test/where.test @@ -543,10 +543,11 @@ CREATE INDEX t3a ON t3(a); CREATE INDEX t3bc ON t3(b,c); CREATE INDEX t3acb ON t3(a,c,b); INSERT INTO t3 SELECT w, 101-w, y FROM t1; SELECT count(*), sum(a), sum(b), sum(c) FROM t3; + ANALYZE; } } {100 5050 5050 348550} do_test where-6.2 { cksort { SELECT * FROM t3 ORDER BY a LIMIT 3 @@ -1617,21 +1618,19 @@ 19 5 } # 2022-12-07 Yong Heng [https://sqlite.org/forum/forumpost/dfe8084751] # -ifcapable vtab { - do_execsql_test where-29.1 { - SELECT DISTINCT 'xyz' FROM pragma_cache_size - WHERE rowid OR abs(0) - ORDER BY - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1; - } {xyz} -} +do_execsql_test where-29.1 { + SELECT DISTINCT 'xyz' FROM pragma_cache_size + WHERE rowid OR abs(0) + ORDER BY + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1; +} {xyz} finish_test Index: test/with3.test ================================================================== --- test/with3.test +++ test/with3.test @@ -87,11 +87,11 @@ do_eqp_test 3.1.2 { WITH cnt(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM cnt LIMIT 1) SELECT * FROM cnt, y1 WHERE i=a } [string map {"\n " \n} { QUERY PLAN - |--MATERIALIZE cnt + |--CO-ROUTINE cnt | |--SETUP | | `--SCAN CONSTANT ROW | `--RECURSIVE STEP | `--SCAN cnt |--SCAN cnt @@ -101,11 +101,11 @@ do_eqp_test 3.1.3 { WITH cnt(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM cnt LIMIT 1000000) SELECT * FROM cnt, y1 WHERE i=a } [string map {"\n " \n} { QUERY PLAN - |--MATERIALIZE cnt + |--CO-ROUTINE cnt | |--SETUP | | `--SCAN CONSTANT ROW | `--RECURSIVE STEP | `--SCAN cnt |--SCAN y1 @@ -123,11 +123,11 @@ UNION ALL SELECT c.w + 1, x FROM w1, c LIMIT 1) SELECT * FROM c, w2, w1 WHERE c.id=w2.pk AND c.id=w1.pk; } { QUERY PLAN - |--MATERIALIZE c + |--CO-ROUTINE c | |--SETUP | | |--SCAN CONSTANT ROW | | `--SCALAR SUBQUERY xxxxxx | | `--SCAN w2 | `--RECURSIVE STEP Index: test/with6.test ================================================================== --- test/with6.test +++ test/with6.test @@ -85,11 +85,11 @@ FROM (SELECT x FROM c LIMIT 5) AS c1, (SELECT x FROM c LIMIT 5) AS c2, (SELECT x FROM c LIMIT 5) AS c3; } { QUERY PLAN - |--MATERIALIZE c1 + |--CO-ROUTINE c1 | |--CO-ROUTINE c | | `--SCAN 2 CONSTANT ROWS | `--SCAN c |--MATERIALIZE c2 | |--CO-ROUTINE c @@ -121,11 +121,11 @@ FROM (SELECT x FROM c LIMIT 5) AS c1, (SELECT x FROM c LIMIT 6) AS c2, (SELECT x FROM c LIMIT 7) AS c3; } { QUERY PLAN - |--MATERIALIZE c1 + |--CO-ROUTINE c1 | |--MATERIALIZE c | | `--SCAN 2 CONSTANT ROWS | `--SCAN c |--MATERIALIZE c2 | `--SCAN c @@ -149,11 +149,11 @@ FROM (SELECT x FROM c LIMIT 5) AS c1, (SELECT x FROM c LIMIT 6) AS c2, (SELECT x FROM c LIMIT 7) AS c3; } { QUERY PLAN - |--MATERIALIZE c1 + |--CO-ROUTINE c1 | |--MATERIALIZE c | | `--SCAN 2 CONSTANT ROWS | `--SCAN c |--MATERIALIZE c2 | `--SCAN c @@ -226,11 +226,11 @@ } {40404 40405 40406 40504 40505 40506 40604 40605 40606} do_eqp_test 211 { SELECT y FROM t2 ORDER BY y; } { QUERY PLAN - |--MATERIALIZE c1 + |--CO-ROUTINE c1 | |--CO-ROUTINE c | | `--SCAN 3 CONSTANT ROWS | `--SCAN c |--MATERIALIZE c2 | |--CO-ROUTINE c Index: test/zipfile.test ================================================================== --- test/zipfile.test +++ test/zipfile.test @@ -851,16 +851,8 @@ (9223372036854775806), (-9223372036854775807) ) SELECT DISTINCT typeof(zipfile(0,0,x,0)) FROM vlist; } {blob} - -# 2023-01-04 -# https://sqlite.org/forum/forumpost/d1c96a9032e564f8 -# Call to fopen() with a NULL filename. -# -do_catchsql_test 18.1 { - SELECT * FROM zipfile(NULL); -} {1 {cannot open file: }} - + finish_test Index: tool/mkopcodeh.tcl ================================================================== --- tool/mkopcodeh.tcl +++ tool/mkopcodeh.tcl @@ -84,10 +84,11 @@ set in1($name) 0 set in2($name) 0 set in3($name) 0 set out2($name) 0 set out3($name) 0 + set ncycle($name) 0 for {set i 3} {$i<[llength $line]-1} {incr i} { switch [string trim [lindex $line $i] ,] { same { incr i if {[lindex $line $i]=="as"} { @@ -105,10 +106,11 @@ in1 {set in1($name) 1} in2 {set in2($name) 1} in3 {set in3($name) 1} out2 {set out2($name) 1} out3 {set out3($name) 1} + ncycle {set ncycle($name) 1} } } if {$group($name)} { set newGroup 0 if {[info exists groups($nGroup)]} { @@ -138,10 +140,11 @@ set in1($name) 0 set in2($name) 0 set in3($name) 0 set out2($name) 0 set out3($name) 0 + set ncycle($name) 0 set op($name) -1 set order($nOp) $name incr nOp } @@ -283,10 +286,11 @@ if {$in1($name)} {incr x 2} if {$in2($name)} {incr x 4} if {$in3($name)} {incr x 8} if {$out2($name)} {incr x 16} if {$out3($name)} {incr x 32} + if {$ncycle($name)} {incr x 64} } set bv($i) $x } puts "" puts "/* Properties such as \"out2\" or \"jump\" that are specified in" @@ -297,10 +301,11 @@ puts "#define OPFLG_IN1 0x02 /* in1: P1 is an input */" puts "#define OPFLG_IN2 0x04 /* in2: P2 is an input */" puts "#define OPFLG_IN3 0x08 /* in3: P3 is an input */" puts "#define OPFLG_OUT2 0x10 /* out2: P2 is an output */" puts "#define OPFLG_OUT3 0x20 /* out3: P3 is an output */" +puts "#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */" puts "#define OPFLG_INITIALIZER \173\\" for {set i 0} {$i<=$max} {incr i} { if {$i%8==0} { puts -nonewline [format "/* %3d */" $i] } Index: tool/mkshellc.tcl ================================================================== --- tool/mkshellc.tcl +++ tool/mkshellc.tcl @@ -53,11 +53,11 @@ # puts $out "#line 1 \"$cfile\"" set in2 [open $topdir/src/$cfile] fconfigure $in2 -translation binary while {![eof $in2]} { set lx [omit_redundant_typedefs [gets $in2]] - if {[regexp {^#include "sqlite} $lx]} { + if {[regexp {^# *include "sqlite} $lx]} { set lx "/* $lx */" } if {[regexp {^# *include "test_windirent.h"} $lx]} { set lx "/* $lx */" } Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -38,15 +38,18 @@ # set addstatic 1 set linemacros 0 set useapicall 0 +set enable_recover 0 set srcdir tsrc for {set i 0} {$i<[llength $argv]} {incr i} { set x [lindex $argv $i] - if {[regexp {^-?-nostatic$} $x]} { + if {[regexp {^-?-enable-recover$} $x]} { + set enable_recover 1 + } elseif {[regexp {^-?-nostatic$} $x]} { set addstatic 0 } elseif {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { if {$ulm == ""} {set ulm 1} set linemacros $ulm } elseif {[regexp {^-?-useapicall$} $x]} { @@ -76,11 +79,13 @@ close $in # Open the output file and write a header comment at the beginning # of the file. # -set out [open sqlite3.c w] +set fname sqlite3.c +if {$enable_recover} { set fname sqlite3r.c } +set out [open $fname w] # Force the output to use unix line endings, even on Windows. fconfigure $out -translation lf set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] puts $out [subst \ {/****************************************************************************** @@ -160,10 +165,11 @@ vdbe.h vdbeInt.h vxworks.h wal.h whereInt.h + sqlite3recover.h } { set available_hdr($hdr) 1 } set available_hdr(sqliteInt.h) 0 set available_hdr(os_common.h) 0 @@ -323,11 +329,11 @@ # Process the source files. Process files containing commonly # used subroutines first in order to help the compiler find # inlining opportunities. # -foreach file { +set flist { sqliteInt.h os_common.h ctime.c global.c @@ -439,11 +445,15 @@ dbstat.c dbpage.c sqlite3session.c fts5.c stmt.c -} { +} +if {$enable_recover} { + lappend flist sqlite3recover.c dbdata.c +} +foreach file $flist { copy_file $srcdir/$file } puts $out \ "/* Return the source-id for this library */ Index: tool/mksqlite3h.tcl ================================================================== --- tool/mksqlite3h.tcl +++ tool/mksqlite3h.tcl @@ -37,13 +37,20 @@ set TOP [lindex $argv 0] # Enable use of SQLITE_APICALL macros at the right points? # set useapicall 0 + +# Include sqlite3recover.h? +# +set enable_recover 0 if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} { set useapicall 1 +} +if {[lsearch -regexp [lrange $argv 1 end] {^-+enable-recover}] != -1} { + set enable_recover 1 } # Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file. # set in [open $TOP/VERSION] @@ -82,10 +89,13 @@ $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h $TOP/ext/session/sqlite3session.h $TOP/ext/fts5/fts5.h }] +if {$enable_recover} { + lappend filelist "$TOP/ext/recover/sqlite3recover.h" +} # These are the functions that accept a variable number of arguments. They # always need to use the "cdecl" calling convention even when another calling # convention (e.g. "stcall") is being used for the rest of the library. set cdecllist { Index: tool/vdbe_profile.tcl ================================================================== --- tool/vdbe_profile.tcl +++ tool/vdbe_profile.tcl @@ -64,19 +64,25 @@ foreach stmt $allstmt { puts "********************************************************************" puts [string trim $sql($stmt)] puts "Execution count: $cnt($stmt)" + set tcx 0 + set ttx 0 for {set i 0} {[info exists stat($stmt,$i)]} {incr i} { foreach {cx tx detail} $stat($stmt,$i) break if {$cx==0} { set ax 0 } else { set ax [expr {$tx/$cx}] } puts [format {%8d %12d %12d %4d %s} $cx $tx $ax $i $detail] + incr tcx $cx + incr ttx $tx } + set tax [expr {$tcx>0?$ttx/$tcx:0}] + puts [format {%8d %12d %12d TOTAL} $tcx $ttx $tax] } puts "********************************************************************" puts "OPCODES:" foreach op [lsort [array names opcnt]] { set cx $opcnt($op)