Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.45.0 +3.45.3 Index: autoconf/tea/configure.ac ================================================================== --- autoconf/tea/configure.ac +++ autoconf/tea/configure.ac @@ -17,11 +17,11 @@ # so you can encode the package version directly into the source files. # This will also define a special symbol for Windows (BUILD_ # so that we create the export library with the dll. #----------------------------------------------------------------------- -AC_INIT([sqlite],[3.45.0]) +AC_INIT([sqlite],[3.45.3]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. # This will define a ${TEA_PLATFORM} variable == "unix" or "windows" # as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE. Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.45.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.45.3. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # @@ -724,12 +724,12 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.45.0' -PACKAGE_STRING='sqlite 3.45.0' +PACKAGE_VERSION='3.45.3' +PACKAGE_STRING='sqlite 3.45.3' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ @@ -1470,11 +1470,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.45.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.45.3 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1535,11 +1535,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.45.0:";; + short | recursive ) echo "Configuration of sqlite 3.45.3:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1666,11 +1666,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.45.0 +sqlite configure 3.45.3 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. @@ -2085,11 +2085,11 @@ } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.45.0, which was +It was created by sqlite $as_me 3.45.3, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF @@ -12479,11 +12479,11 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.45.0, which was +This file was extended by sqlite $as_me 3.45.3, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -12545,11 +12545,11 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.45.0 +sqlite config.status 3.45.3 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: ext/consio/console_io.c ================================================================== --- ext/consio/console_io.c +++ ext/consio/console_io.c @@ -27,10 +27,13 @@ # include "sqlite3.h" #endif #ifndef HAVE_CONSOLE_IO_H # include "console_io.h" #endif +#if defined(_MSC_VER) +# pragma warning(disable : 4204) +#endif #ifndef SQLITE_CIO_NO_TRANSLATE # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT # ifndef SHELL_NO_SYSINC # include @@ -124,10 +127,14 @@ ppst->pf = pf; ppst->reachesConsole = ( (short)isatty(fileno(pf)) ); return ppst->reachesConsole; # endif } + +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) +# endif # if CIO_WIN_WC_XLATE /* Define console modes for use with the Windows Console API. */ # define SHELL_CONI_MODE \ (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \ @@ -674,7 +681,11 @@ # if CIO_WIN_WC_XLATE } # endif } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ + +#if defined(_MSC_VER) +# pragma warning(default : 4204) +#endif #undef SHELL_INVALID_FILE_PTR Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -4004,42 +4004,33 @@ /* ** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual ** table. */ -static int fts3Integrity( +static int fts3IntegrityMethod( sqlite3_vtab *pVtab, /* The virtual table to be checked */ const char *zSchema, /* Name of schema in which pVtab lives */ const char *zTabname, /* Name of the pVTab table */ int isQuick, /* True if this is a quick_check */ char **pzErr /* Write error message here */ ){ Fts3Table *p = (Fts3Table*)pVtab; - char *zSql; int rc; - char *zErr = 0; + int bOk = 0; - assert( pzErr!=0 ); - assert( *pzErr==0 ); UNUSED_PARAMETER(isQuick); - zSql = sqlite3_mprintf( - "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');", - zSchema, zTabname, zTabname); - if( zSql==0 ){ - return SQLITE_NOMEM; - } - rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr); - sqlite3_free(zSql); - if( (rc&0xff)==SQLITE_CORRUPT ){ + rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( rc!=SQLITE_CORRUPT_VTAB || bOk==0 ); + if( rc!=SQLITE_OK && rc!=SQLITE_CORRUPT_VTAB ){ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS%d table %s.%s: %s", + p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); + }else if( bOk==0 ){ *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", p->bFts4 ? 4 : 3, zSchema, zTabname); - }else if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unable to validate the inverted index for" - " FTS%d table %s.%s: %s", - p->bFts4 ? 4 : 3, zSchema, zTabname, zErr); } - sqlite3_free(zErr); + sqlite3Fts3SegmentsClose(p); return SQLITE_OK; } @@ -4066,11 +4057,11 @@ /* xRename */ fts3RenameMethod, /* xSavepoint */ fts3SavepointMethod, /* xRelease */ fts3ReleaseMethod, /* xRollbackTo */ fts3RollbackToMethod, /* xShadowName */ fts3ShadowName, - /* xIntegrity */ fts3Integrity, + /* xIntegrity */ fts3IntegrityMethod, }; /* ** This function is registered as the module destructor (called when an ** FTS3 enabled database connection is closed). It frees the memory Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -650,8 +650,10 @@ int sqlite3FtsUnicodeIsalnum(int); int sqlite3FtsUnicodeIsdiacritic(int); #endif int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); + +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -5292,11 +5292,11 @@ ** to false before returning. ** ** If an error occurs (e.g. an OOM or IO error), return an SQLite error ** code. The final value of *pbOk is undefined in this case. */ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int rc = SQLITE_OK; /* Return code */ u64 cksum1 = 0; /* Checksum based on FTS index contents */ u64 cksum2 = 0; /* Checksum based on %_content contents */ sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ @@ -5370,11 +5370,11 @@ } sqlite3_finalize(pStmt); } - *pbOk = (cksum1==cksum2); + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); return rc; } /* ** Run the integrity-check. If no error occurs and the current contents of @@ -5410,11 +5410,11 @@ static int fts3DoIntegrityCheck( Fts3Table *p /* FTS3 table handle */ ){ int rc; int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } /* Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -6835,27 +6835,30 @@ ** a rowid of iFrom or greater. */ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ int ii; Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; for(ii=0; iinIter; ii++){ Fts5Iter *p = pT->apIter[ii]; if( p->base.bEof==0 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidpIndex, p, bFrom, iFrom); + fts5MultiIterNext(pIndex, p, bFrom, iFrom); while( bFrom && p->base.bEof==0 && p->base.iRowidpIndex->rc==SQLITE_OK + && pIndex->rc==SQLITE_OK ){ - fts5MultiIterNext(p->pIndex, p, 0, 0); + fts5MultiIterNext(pIndex, p, 0, 0); } } } - fts5IterSetOutputsTokendata(pIter); + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } } /* ** If the segment-iterator passed as the first argument is at EOF, then ** set pIter->term to a copy of buffer pTerm. Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -2969,31 +2969,25 @@ const char *zTabname, /* Name of the table itself */ int isQuick, /* True if this is a quick-check */ char **pzErr /* Write error message here */ ){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - Fts5Config *pConfig = pTab->p.pConfig; - char *zSql; - char *zErr = 0; int rc; + assert( pzErr!=0 && *pzErr==0 ); UNUSED_PARAM(isQuick); - zSql = sqlite3_mprintf( - "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');", - zSchema, zTabname, pConfig->zName); - if( zSql==0 ) return SQLITE_NOMEM; - rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr); - sqlite3_free(zSql); + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", zSchema, zTabname); }else if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" " FTS5 table %s.%s: %s", - zSchema, zTabname, zErr); + zSchema, zTabname, sqlite3_errstr(rc)); } - sqlite3_free(zErr); + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + return SQLITE_OK; } static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { Index: ext/fts5/fts5_tcl.c ================================================================== --- ext/fts5/fts5_tcl.c +++ ext/fts5/fts5_tcl.c @@ -1167,11 +1167,11 @@ /* ** Delete the OriginTextCtx object indicated by the only argument. */ static void f5tOrigintextTokenizerDelete(void *pCtx){ OriginTextCtx *p = (OriginTextCtx*)pCtx; - ckfree(p); + ckfree((char*)p); } static int f5tOrigintextCreate( void *pCtx, const char **azArg, Index: ext/fts5/test/fts5faultH.test ================================================================== --- ext/fts5/test/fts5faultH.test +++ ext/fts5/test/fts5faultH.test @@ -125,17 +125,26 @@ INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB'); INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB'); COMMIT; } -do_faultsim_test 3 -faults oom* -prep { +do_faultsim_test 3.1 -faults oom* -prep { } -body { execsql { SELECT rowid FROM t1('BBB AND AAA'); } } -test { faultsim_integrity_check faultsim_test_result {0 {10 35}} +} +do_faultsim_test 3.2 -faults oom* -prep { +} -body { + execsql { + SELECT count(*) FROM t1('BBB'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 27} } finish_test Index: ext/fts5/test/fts5integrity.test ================================================================== --- ext/fts5/test/fts5integrity.test +++ ext/fts5/test/fts5integrity.test @@ -352,7 +352,33 @@ } {ok} do_execsql_test 11.4 { DROP TABLE t1; PRAGMA integrity_check(t2); } {ok} + +#------------------------------------------------------------------- +reset_db + +do_execsql_test 12.1 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('one', 'two'); + INSERT INTO x1 VALUES('three', 'four'); + INSERT INTO x1 VALUES('five', 'six'); +} + +do_execsql_test 12.2 { + PRAGMA integrity_check +} {ok} + +db close +sqlite3 db test.db -readonly 1 + +explain_i { + PRAGMA integrity_check + } +do_execsql_test 12.3 { + PRAGMA integrity_check +} {ok} + + finish_test Index: ext/misc/noop.c ================================================================== --- ext/misc/noop.c +++ ext/misc/noop.c @@ -35,10 +35,28 @@ sqlite3_value **argv ){ assert( argc==1 ); sqlite3_result_value(context, argv[0]); } + +/* +** Implementation of the multitype_text() function. +** +** The function returns its argument. The result will always have a +** TEXT value. But if the original input is numeric, it will also +** have that numeric value. +*/ +static void multitypeTextFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + (void)argc; + (void)sqlite3_value_text(argv[0]); + sqlite3_result_value(context, argv[0]); +} #ifdef _WIN32 __declspec(dllexport) #endif int sqlite3_noop_init( @@ -62,7 +80,11 @@ 0, noopfunc, 0, 0); if( rc ) return rc; rc = sqlite3_create_function(db, "noop_nd", 1, SQLITE_UTF8, 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "multitype_text", 1, + SQLITE_UTF8, + 0, multitypeTextFunc, 0, 0); return rc; } Index: ext/recover/dbdata.c ================================================================== --- ext/recover/dbdata.c +++ ext/recover/dbdata.c @@ -492,10 +492,19 @@ } } } } +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; @@ -520,10 +529,13 @@ } assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } } if( pTab->bPtr ){ if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ pCsr->iCell = pCsr->nCell; @@ -564,23 +576,23 @@ } if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ + if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); + iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; } @@ -659,11 +671,13 @@ } }else{ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ + if( pCsr->pHdrPtr>=&pCsr->pRec[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ bNextPage = 1; }else{ int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); szField = dbdataValueBytes(iType); Index: ext/recover/recovercorrupt2.test ================================================================== --- ext/recover/recovercorrupt2.test +++ ext/recover/recovercorrupt2.test @@ -521,8 +521,36 @@ do_test 7.1 { set R [sqlite3_recover_init db main test.db2] catch { $R run } list [catch { $R finish } msg] $msg } {1 {file is not a database}} + +reset_db +breakpoint +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 8192 pagesize 4096 filename db.sqlite +| page 1 offset 0 +| 0: ac ae b3 76 74 65 20 66 6f 72 6d 61 74 20 33 00 ...vte format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 76 8a 0d ff ff ff 1e 0f cb 00 0f cb 00 00 ..v............. +| 4032: 00 00 00 00 00 00 00 00 00 00 00 33 01 06 17 19 ...........3.... +| 4048: 19 01 43 74 61 62 6c 65 54 61 62 6c 65 30 54 61 ..CtableTable0Ta +| 4064: 62 6c 65 30 02 43 52 45 41 54 45 20 54 41 42 4c ble0.CREATE TABL +| 4080: 45 20 54 61 62 6c 65 30 20 28 43 6f 6c 30 20 29 E Table0 (Col0 ) +| page 2 offset 4096 +| 0: 0d 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 ................ +| end db.sqlite +}]} {} + +do_test 8.1 { + set R [sqlite3_recover_init db main test.db2] + catch { $R run } + list [catch { $R finish } msg] $msg +} {0 {}} finish_test Index: ext/recover/sqlite3recover.c ================================================================== --- ext/recover/sqlite3recover.c +++ ext/recover/sqlite3recover.c @@ -1187,11 +1187,11 @@ if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zTbl, iRoot); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } }else if( rc!=SQLITE_ERROR ){ recoverDbError(p, p->dbOut); Index: ext/recover/test_recover.c ================================================================== --- ext/recover/test_recover.c +++ ext/recover/test_recover.c @@ -234,11 +234,11 @@ } if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; zDb = Tcl_GetString(objv[2]); if( zDb[0]=='\0' ) zDb = 0; - pNew = ckalloc(sizeof(TestRecover)); + pNew = (TestRecover*)ckalloc(sizeof(TestRecover)); if( bSql==0 ){ zUri = Tcl_GetString(objv[3]); pNew->p = sqlite3_recover_init(db, zDb, zUri); }else{ pNew->interp = interp; Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -692,15 +692,13 @@ /* ** Clear the Rtree.pNodeBlob object */ static void nodeBlobReset(Rtree *pRtree){ - if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ - sqlite3_blob *pBlob = pRtree->pNodeBlob; - pRtree->pNodeBlob = 0; - sqlite3_blob_close(pBlob); - } + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); } /* ** Obtain a reference to an r-tree node. */ @@ -740,11 +738,10 @@ rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, pRtree->zNodeName, "data", iNode, 0, &pRtree->pNodeBlob); } if( rc ){ - nodeBlobReset(pRtree); *ppNode = 0; /* If unable to open an sqlite3_blob on the desired row, that can only ** be because the shadow tables hold erroneous data. */ if( rc==SQLITE_ERROR ){ rc = SQLITE_CORRUPT_VTAB; @@ -800,10 +797,11 @@ rc = SQLITE_CORRUPT_VTAB; RTREE_IS_CORRUPT(pRtree); } *ppNode = pNode; }else{ + nodeBlobReset(pRtree); if( pNode ){ pRtree->nNodeRef--; sqlite3_free(pNode); } *ppNode = 0; @@ -944,10 +942,11 @@ RtreeNode *pNode, /* The node from which to extract a coordinate */ int iCell, /* The index of the cell within the node */ int iCoord, /* Which coordinate to extract */ RtreeCoord *pCoord /* OUT: Space to write result to */ ){ + assert( iCellzData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } /* ** Deserialize cell iCell of node pNode. Populate the structure pointed @@ -1133,11 +1132,13 @@ assert( pRtree->nCursor>0 ); resetCursor(pCsr); sqlite3_finalize(pCsr->pReadAux); sqlite3_free(pCsr); pRtree->nCursor--; - nodeBlobReset(pRtree); + if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){ + nodeBlobReset(pRtree); + } return SQLITE_OK; } /* ** Rtree virtual table module xEof method. @@ -1718,11 +1719,15 @@ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc==SQLITE_OK && ALWAYS(p) ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + if( p->iCell>=NCELL(pNode) ){ + rc = SQLITE_ABORT; + }else{ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } } return rc; } /* @@ -1736,10 +1741,11 @@ int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc ) return rc; if( NEVER(p==0) ) return SQLITE_OK; + if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else if( i<=pRtree->nDim2 ){ nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY @@ -3217,12 +3223,11 @@ /* ** Called when a transaction starts. */ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; - assert( pRtree->inWrTrans==0 ); - pRtree->inWrTrans++; + pRtree->inWrTrans = 1; return SQLITE_OK; } /* ** Called when a transaction completes (either by COMMIT or ROLLBACK). @@ -3231,10 +3236,13 @@ static int rtreeEndTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; pRtree->inWrTrans = 0; nodeBlobReset(pRtree); return SQLITE_OK; +} +static int rtreeRollback(sqlite3_vtab *pVtab){ + return rtreeEndTransaction(pVtab); } /* ** The xRename method for rtree module virtual tables. */ @@ -3350,11 +3358,11 @@ rtreeRowid, /* xRowid - read data */ rtreeUpdate, /* xUpdate - write data */ rtreeBeginTransaction, /* xBegin - begin transaction */ rtreeEndTransaction, /* xSync - sync transaction */ rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeRollback, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ ADDED ext/rtree/rtreeJ.test Index: ext/rtree/rtreeJ.test ================================================================== --- /dev/null +++ ext/rtree/rtreeJ.test @@ -0,0 +1,273 @@ +# 2024-02-03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# ROLLBACK in the middle of an RTREE query +# +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set testprefix rtreeJ +ifcapable !rtree { finish_test ; return } + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1, 1, 1), (2, 2, 2); +} {} + +do_execsql_test 1.1 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + +# If a ROLLBACK occurs that backs out changes to the RTREE, then +# all pending queries to the RTREE are aborted. +# +do_test 1.2 { + db eval { + BEGIN; + INSERT INTO t1 VALUES(3, 3, 3); + INSERT INTO t1 VALUES(4, 4, 4); + } + set rc [catch { + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { ROLLBACK } + } + lappend res $id $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} + +do_execsql_test 1.3 { + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0} + +# A COMMIT of changes to the RTREE does not affect pending queries +# +do_test 1.4 { + set res {} + db eval { + BEGIN; + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(6, 6, 6); + } + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { COMMIT } + } + lappend res $id $x1 $x2 + } + set res +} {1 1.0 1.0 2 2.0 2.0 5 5.0 5.0 6 6.0 6.0} + +do_execsql_test 1.5 { + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 5 5.0 5.0 6 6.0 6.0} + +do_execsql_test 1.6 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,1,1),(2,2,2),(3,3,3),(4,4,4); + CREATE TABLE t2(x); + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0 4 4.0 4.0} + +# A rollback that does not affect the rtree table because +# the rtree table has not been written to does not cause +# a query abort. +# +do_test 1.7 { + set res {} + db eval { + BEGIN; + INSERT INTO t2(x) VALUES(12345); + } + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { ROLLBACK } + } + lappend res $id $x1 $x2 + } + set res +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0 4 4.0 4.0} + +# ROLLBACK TO that affects the RTREE does cause a query abort. +# +do_test 1.8 { + db eval { + DELETE FROM t1 WHERE rowid>1; + BEGIN; + DELETE FROM t2; + INSERT INTO t2(x) VALUES(23456); + SAVEPOINT 'one'; + INSERT INTO t1 VALUES(2,2,2),(3,3,3); + } + set rc [catch { + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { ROLLBACK TO 'one'; } + } + lappend res $id $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} + +do_execsql_test 1.9 { + COMMIT; + SELECT * FROM t1; +} {1 1.0 1.0} + +# ROLLBACK TO that does not affect the RTREE does not cause a query abort. +# +do_execsql_test 1.10 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,1,1),(2,2,2),(3,3,3); + BEGIN; + DELETE FROM t2; + INSERT INTO t2(x) VALUES(34567); + SAVEPOINT 'one'; + INSERT INTO t2(x) VALUES('a string'); + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0} +do_test 1.11 { + set rc [catch { + set res {} + db eval { SELECT * FROM t1 } { + if {$id==2} { + # db eval { ROLLBACK TO 'one'; } + } + lappend res $id $x1 $x2 + } + set res + } msg] + list $rc $msg +} {0 {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0}} + +do_execsql_test 1.12 { + COMMIT; + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0} + +#---------------------------------------------------------------------- + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1, 1, 1), (2, 2, 2); + CREATE TABLE t2(x); +} {} + +do_test 2.1 { + db eval { + BEGIN; + INSERT INTO t1 VALUES(3, 3, 3); + PRAGMA writable_schema = RESET; + } + + set rc [catch { + db eval { SELECT x1, x2 FROM t1 } { + if {$x1==1} { + db eval { ROLLBACK } + } + lappend res $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} + +do_execsql_test 2.1 { + CREATE TABLE bak_node(nodeno, data); + CREATE TABLE bak_parent(nodeno, parentnode); + CREATE TABLE bak_rowid(rowid, nodeno); +} +proc save_t1 {} { + db eval { + DELETE FROM bak_node; + DELETE FROM bak_parent; + DELETE FROM bak_rowid; + INSERT INTO bak_node SELECT * FROM t1_node; + INSERT INTO bak_parent SELECT * FROM t1_parent; + INSERT INTO bak_rowid SELECT * FROM t1_rowid; + } +} +proc restore_t1 {} { + db eval { + DELETE FROM t1_node; + DELETE FROM t1_parent; + DELETE FROM t1_rowid; + INSERT INTO t1_node SELECT * FROM bak_node; + INSERT INTO t1_parent SELECT * FROM bak_parent; + INSERT INTO t1_rowid SELECT * FROM bak_rowid; + } +} + +do_test 2.3 { + save_t1 + db eval { + INSERT INTO t1 VALUES(3, 3, 3); + } + set rc [catch { + db eval { SELECT rowid, x1, x2 FROM t1 } { + if {$x1==1} { + restore_t1 + } + lappend res $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} +do_execsql_test 2.4 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + +do_test 2.5 { + save_t1 + db eval { + INSERT INTO t1 VALUES(3, 3, 3); + } + set rc [catch { + db eval { SELECT x1 FROM t1 } { + if {$x1==1} { + restore_t1 + } + lappend res $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} +do_execsql_test 2.6 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + +do_test 2.7 { + save_t1 + db eval { + INSERT INTO t1 VALUES(3, 3, 3); + } + set ::res [list] + set rc [catch { + db eval { SELECT 'abc' FROM t1 } { + if {$::res==[list]} { + restore_t1 + set ::bDone 1 + } + lappend res abc + } + } msg] + set res +} {abc abc abc} +do_execsql_test 2.6 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + + +finish_test Index: ext/wasm/GNUmakefile ================================================================== --- ext/wasm/GNUmakefile +++ ext/wasm/GNUmakefile @@ -650,19 +650,22 @@ } > $@ ######################################################################## # $(sqlite3-license-version.js) contains the license header and # in-comment build version info. +# +# Maintenance reminder: there are awk binaries out there which do not +# support -e SCRIPT. $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ $(MAKEFILE) @echo "Making $@..."; { \ cat $(sqlite3-license-version-header.js); \ echo '/*'; \ echo '** This code was built from sqlite3 version...'; \ echo "**"; \ - awk -e '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' \ - -e '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ + awk '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' $(sqlite3.h); \ + awk '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo "**"; \ echo "** Using the Emscripten SDK version $(emcc.version)."; \ echo '*/'; \ } > $@ Index: ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js ================================================================== --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -1269,11 +1269,11 @@ }/*extend sqlite3.oo1*/ thePool.log("VFS initialized."); return poolUtil; }).catch(async (e)=>{ await thePool.removeVfs().catch(()=>{}); - return e; + throw e; }); }).catch((err)=>{ //error("rejecting promise:",err); return initPromises[vfsName] = Promise.reject(err); }); Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -6278,11 +6278,14 @@ if( pCur->skipNext<0 ) return SQLITE_OK; } } pPage = pCur->pPage; - assert( pPage->isInit ); + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } if( !pPage->leaf ){ int idx = pCur->ix; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); if( rc ) return rc; rc = moveToRightmost(pCur); @@ -6951,11 +6954,14 @@ /* This is the common case where everything fits on the btree page ** and no overflow pages are required. */ n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; assert( nSrc<=nPayload ); testcase( nSrcpBt->nPreformatSize; - if( szNew<4 ) szNew = 4; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); if( info.nPayload!=info.nLocal ){ Pgno ovfl = get4byte(&newCell[szNew-4]); @@ -9459,11 +9468,11 @@ dropCell(pPage, idx, info.nSize, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } rc = insertCellFast(pPage, idx, newCell, szNew); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); @@ -9489,11 +9498,11 @@ ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() ** fails. Internal data structure corruption will result otherwise. ** Also, set the cursor state to invalid. This stops saveCursorPosition() Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -3004,13 +3004,16 @@ /* Legacy versions of SQLite allowed the use of the magic "rowid" column ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -62,10 +62,13 @@ #endif #ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN # if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), # endif +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", #endif #ifdef SQLITE_ALLOW_URI_AUTHORITY "ALLOW_URI_AUTHORITY", #endif #ifdef SQLITE_ATOMIC_INTRINSICS Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -216,13 +216,14 @@ if( ExprHasProperty(pExpr, EP_Unlikely) ){ assert( ExprUseXList(pExpr) ); assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } } return pExpr; } @@ -2737,13 +2738,16 @@ case TK_FLOAT: case TK_BLOB: return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -1099,17 +1099,17 @@ switch( sqlite3_value_type(pValue) ){ case SQLITE_FLOAT: { double r1, r2; const char *zVal; r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!.15g", r1); + sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); if( r1!=r2 ){ sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!.20e", r1); + sqlite3_str_appendf(pStr, "%!0.20e", r1); } } break; } case SQLITE_INTEGER: { @@ -1407,11 +1407,11 @@ || sqlite3_context_db_handle(context)->mallocFailed ); return; } if( zPattern[0]==0 ){ assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); - sqlite3_result_value(context, argv[0]); + sqlite3_result_text(context, (const char*)zStr, nStr, SQLITE_TRANSIENT); return; } nPattern = sqlite3_value_bytes(argv[1]); assert( zPattern==sqlite3_value_text(argv[1]) ); /* No encoding change */ zRep = sqlite3_value_text(argv[2]); @@ -1890,11 +1890,11 @@ p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ if( p->approx ){ if( p->ovrfl ){ sqlite3_result_error(context,"integer overflow",-1); - }else if( !sqlite3IsNaN(p->rErr) ){ + }else if( !sqlite3IsOverflow(p->rErr) ){ sqlite3_result_double(context, p->rSum+p->rErr); }else{ sqlite3_result_double(context, p->rSum); } }else{ @@ -1907,11 +1907,11 @@ p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ double r; if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } sqlite3_result_double(context, r/(double)p->cnt); } @@ -1921,11 +1921,11 @@ double r = 0.0; p = sqlite3_aggregate_context(context, 0); if( p ){ if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } } sqlite3_result_double(context, r); Index: src/global.c ================================================================== --- src/global.c +++ src/global.c @@ -287,10 +287,13 @@ SQLITE_MEMDB_DEFAULT_MAXSIZE, /* mxMemdbSize */ #endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ +#endif 0, /* bLocaltimeFault */ 0, /* xAltLocaltime */ 0x7ffffffe, /* iOnceResetThreshold */ SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ 0, /* iPrngSeed */ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -1084,11 +1084,11 @@ pNx->pUpsertSrc = pTabList; pNx->regData = regData; pNx->iDataCur = iDataCur; pNx->iIdxCur = iIdxCur; if( pNx->pUpsertTarget ){ - if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ goto insert_cleanup; } } pNx = pNx->pNextUpsert; }while( pNx!=0 ); @@ -2976,11 +2976,14 @@ ** further downstream. */ return 0; /* Corrupt schema - two indexes on the same btree */ } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif #ifndef SQLITE_OMIT_FOREIGN_KEY /* Disallow the transfer optimization if the destination table contains Index: src/json.c ================================================================== --- src/json.c +++ src/json.c @@ -586,19 +586,29 @@ jsonAppendCharExpand(p,c); }else{ p->zBuf[p->nUsed++] = c; } } + +/* Remove a single character from the end of the string +*/ +static void jsonStringTrimOneChar(JsonString *p){ + if( p->eErr==0 ){ + assert( p->nUsed>0 ); + p->nUsed--; + } +} + /* Make sure there is a zero terminator on p->zBuf[] ** ** Return true on success. Return false if an OOM prevents this ** from happening. */ static int jsonStringTerminate(JsonString *p){ jsonAppendChar(p, 0); - p->nUsed--; + jsonStringTrimOneChar(p); return p->eErr==0; } /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. @@ -1595,10 +1605,11 @@ return j+1; } case '[': { /* Parse array */ iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); iStart = pParse->nBlob; if( pParse->oom ) return -1; if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; @@ -1993,10 +2004,14 @@ */ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; memset(&px, 0, sizeof(px)); jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); (void)jsonTranslateTextToBlob(&px, 0); if( px.oom ){ @@ -2060,12 +2075,12 @@ } sz = (pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; n = 9; } - if( i+sz+n > pParse->nBlob - && i+sz+n > pParse->nBlob-pParse->delta + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta ){ sz = 0; n = 0; } *pSz = sz; @@ -2111,18 +2126,20 @@ jsonAppendRawNZ(pOut, "false", 5); return i+1; } case JSONB_INT: case JSONB_FLOAT: { + if( sz==0 ) goto malformed_jsonb; jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); break; } case JSONB_INT5: { /* Integer literal in hexadecimal notation */ u32 k = 2; sqlite3_uint64 u = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; int bOverflow = 0; + if( sz==0 ) goto malformed_jsonb; if( zIn[0]=='-' ){ jsonAppendChar(pOut, '-'); k++; }else if( zIn[0]=='+' ){ k++; @@ -2141,10 +2158,11 @@ break; } case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ u32 k = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( sz==0 ) goto malformed_jsonb; if( zIn[0]=='-' ){ jsonAppendChar(pOut, '-'); k++; } if( zIn[k]=='.' ){ @@ -2254,34 +2272,36 @@ } case JSONB_ARRAY: { jsonAppendChar(pOut, '['); j = i+n; iEnd = j+sz; - while( jeErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, ','); } - if( sz>0 ) pOut->nUsed--; + if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, ']'); break; } case JSONB_OBJECT: { int x = 0; jsonAppendChar(pOut, '{'); j = i+n; iEnd = j+sz; - while( jeErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } - if( x & 1 ) pOut->eErr |= JSTRING_MALFORMED; - if( sz>0 ) pOut->nUsed--; + if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, '}'); break; } default: { + malformed_jsonb: pOut->eErr |= JSTRING_MALFORMED; break; } } return i+n+sz; @@ -3203,10 +3223,42 @@ }else{ jsonBadPathError(ctx, zPath); } return; } + +/* +** If pArg is a blob that seems like a JSONB blob, then initialize +** p to point to that JSONB and return TRUE. If pArg does not seem like +** a JSONB blob, then return FALSE; +** +** This routine is only called if it is already known that pArg is a +** blob. The only open question is whether or not the blob appears +** to be a JSONB blob. +*/ +static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ + u32 n, sz = 0; + p->aBlob = (u8*)sqlite3_value_blob(pArg); + p->nBlob = (u32)sqlite3_value_bytes(pArg); + if( p->nBlob==0 ){ + p->aBlob = 0; + return 0; + } + if( NEVER(p->aBlob==0) ){ + return 0; + } + if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + && (n = jsonbPayloadSize(p, 0, &sz))>0 + && sz+n==p->nBlob + && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + ){ + return 1; + } + p->aBlob = 0; + p->nBlob = 0; + return 0; +} /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new ** JsonParse object. @@ -3260,38 +3312,34 @@ p->hasNonstd = pFromCache->hasNonstd; jsonParseFree(pFromCache); return p; } if( eType==SQLITE_BLOB ){ - u32 n, sz = 0; - p->aBlob = (u8*)sqlite3_value_blob(pArg); - p->nBlob = (u32)sqlite3_value_bytes(pArg); - if( p->nBlob==0 ){ - goto json_pfa_malformed; - } - if( NEVER(p->aBlob==0) ){ - goto json_pfa_oom; - } - if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){ - goto json_pfa_malformed; - } - n = jsonbPayloadSize(p, 0, &sz); - if( n==0 - || sz+n!=p->nBlob - || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0) - ){ - goto json_pfa_malformed; - } - if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ - goto json_pfa_oom; - } - return p; + if( jsonArgIsJsonb(pArg,p) ){ + if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ + goto json_pfa_oom; + } + return p; + } + /* If the blob is not valid JSONB, fall through into trying to cast + ** the blob into text which is then interpreted as JSON. (tag-20240123-a) + ** + ** This goes against all historical documentation about how the SQLite + ** JSON functions were suppose to work. From the beginning, blob was + ** reserved for expansion and a blob value should have raised an error. + ** But it did not, due to a bug. And many applications came to depend + ** upon this buggy behavior, espeically when using the CLI and reading + ** JSON text using readfile(), which returns a blob. For this reason + ** we will continue to support the bug moving forward. + ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d + */ } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; if( p->nJson==0 ) goto json_pfa_malformed; - if( NEVER(p->zJson==0) ) goto json_pfa_oom; + assert( p->zJson!=0 ); if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; return p; }else{ @@ -3453,14 +3501,14 @@ } if( showContent ){ if( sz==0 && x<=JSONB_FALSE ){ sqlite3_str_append(pOut, "\n", 1); }else{ - u32 i; + u32 j; sqlite3_str_appendall(pOut, ": \""); - for(i=iStart+n; iaBlob[i]; + for(j=iStart+n; jaBlob[j]; if( c<0x20 || c>=0x7f ) c = '.'; sqlite3_str_append(pOut, (char*)&c, 1); } sqlite3_str_append(pOut, "\"\n", 2); } @@ -4258,16 +4306,16 @@ sqlite3_result_int(ctx, 0); #endif return; } case SQLITE_BLOB: { - if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ + if( jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; - }else{ + }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; u32 iErr; memset(&px, 0, sizeof(px)); @@ -4274,12 +4322,15 @@ px.aBlob = (u8*)sqlite3_value_blob(argv[0]); px.nBlob = sqlite3_value_bytes(argv[0]); iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); res = iErr==0; } + break; } - break; + /* Fall through into interpreting the input as text. See note + ** above at tag-20240123-a. */ + /* no break */ deliberate_fall_through } default: { JsonParse px; if( (flags & 0x3)==0 ) break; memset(&px, 0, sizeof(px)); @@ -4400,21 +4451,21 @@ }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); } sqlite3_result_subtype(ctx, JSON_SUBTYPE); @@ -4520,21 +4571,21 @@ }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); } sqlite3_result_subtype(ctx, JSON_SUBTYPE); @@ -4861,10 +4912,13 @@ break; } case JEACH_VALUE: { u32 i = jsonSkipLabel(p); jsonReturnFromBlob(&p->sParse, i, ctx, 1); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } break; } case JEACH_TYPE: { u32 i = jsonSkipLabel(p); u8 eType = p->sParse.aBlob[i] & 0x0f; @@ -4907,13 +4961,13 @@ break; } case JEACH_JSON: { if( p->sParse.zJson==0 ){ sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); + SQLITE_TRANSIENT); }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); } break; } } return SQLITE_OK; @@ -5011,17 +5065,13 @@ jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; - if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); - }else{ - goto json_each_malformed_input; - } + if( jsonFuncArgMightBeBinary(argv[0]) ){ + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); if( p->sParse.zJson==0 ){ p->i = p->iEnd = 0; Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -762,10 +762,22 @@ case SQLITE_CONFIG_MEMDB_MAXSIZE: { sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64); break; } #endif /* SQLITE_OMIT_DESERIALIZE */ + + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } default: { rc = SQLITE_ERROR; break; } Index: src/memdb.c ================================================================== --- src/memdb.c +++ src/memdb.c @@ -797,10 +797,18 @@ rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ){ pOut = 0; }else{ sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } if( piSize ) *piSize = sz; if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ pOut = 0; }else{ pOut = sqlite3_malloc64( sz ); Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -5439,15 +5439,20 @@ #endif *pp = 0; #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } } #endif Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -4519,19 +4519,24 @@ OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", osGetCurrentProcessId(), fd, iOff, nAmt, pp)); #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ){ OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); return rc; } } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } } Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -7085,11 +7085,11 @@ /* ** Return the file handle for the journal file (if it exists). ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; #endif } Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1760,35 +1760,11 @@ int bStrict; /* True for a STRICT table */ int r2; /* Previous key for WITHOUT ROWID tables */ int mxCol; /* Maximum non-virtual column number */ if( pObjTab && pObjTab!=pTab ) continue; - if( !IsOrdinaryTable(pTab) ){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_vtab *pVTab; - int a1; - if( !IsVirtual(pTab) ) continue; - if( pTab->nCol<=0 ){ - const char *zMod = pTab->u.vtab.azArg[0]; - if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; - } - sqlite3ViewGetColumnNames(pParse, pTab); - if( pTab->u.vtab.p==0 ) continue; - pVTab = pTab->u.vtab.p->pVtab; - if( NEVER(pVTab==0) ) continue; - if( NEVER(pVTab->pModule==0) ) continue; - if( pVTab->pModule->iVersion<4 ) continue; - if( pVTab->pModule->xIntegrity==0 ) continue; - sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); - pTab->nTabRef++; - sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); - a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, a1); -#endif - continue; - } + if( !IsOrdinaryTable(pTab) ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; r2 = 0; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); @@ -1919,10 +1895,11 @@ /* OP_IsType does not detect NaN values in the database file ** which should be treated as a NULL. So if the header type ** is REAL, we have to load the actual data using OP_Column ** to reliably determine if the value is a NULL. */ sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); VdbeCoverage(v); } zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); @@ -2109,10 +2086,42 @@ if( pPk ){ sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } } } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( pObjTab && pObjTab!=pTab ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; + } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ @@ -2745,11 +2754,15 @@ return SQLITE_OK; } j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; + if( seen[1]==0 ){ + pIdxInfo->estimatedCost = (double)1000; + pIdxInfo->estimatedRows = 1000; + return SQLITE_OK; + } pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; j = seen[1]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 2; pIdxInfo->aConstraintUsage[j].omit = 1; Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -496,10 +496,11 @@ } #endif if( xtype==etFLOAT ){ iRound = -precision; }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; iRound = precision; }else{ iRound = precision+1; } sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -77,10 +77,12 @@ sqlite3 *db; /* The database connection */ assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( db->mallocFailed ){ sqlite3ExprDelete(db, pDup); pDup = 0; @@ -464,12 +466,41 @@ } break; } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && ALWAYS(pMatch!=0) + && ALWAYS(pMatch->pTab!=0) + && (pMatch->pTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; assert( ExprUseYTab(pExpr) ); @@ -591,17 +622,17 @@ /* ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; + cnt = cntTab; if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } /* @@ -962,10 +993,23 @@ ** If this optimization occurs, also restore the NameContext ref-counts ** to the state they where in before the "column" LHS expression was ** resolved. This prevents "column" from being counted as having been ** referenced, which might prevent a SELECT from being erroneously ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ case TK_NOTNULL: case TK_ISNULL: { int anRef[8]; NameContext *p; @@ -972,23 +1016,40 @@ int i; for(i=0, p=pNC; p && ipNext, i++){ anRef[i] = p->nRef; } sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ - testcase( ExprHasProperty(pExpr, EP_OuterON) ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->u.iValue = (pExpr->op==TK_NOTNULL); - pExpr->flags |= EP_IntValue; - pExpr->op = TK_INTEGER; - - for(i=0, p=pNC; p && ipNext, i++){ - p->nRef = anRef[i]; - } - sqlite3ExprDelete(pParse->db, pExpr->pLeft); - pExpr->pLeft = 0; - } + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; + } + + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ + } + } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; return WRC_Prune; } /* A column name: ID ** Or table name and column name: ID.ID @@ -1884,11 +1945,13 @@ sqlite3ErrorMsg(pParse, "HAVING clause on a non-aggregate query"); return WRC_Abort; } if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; if( pItem->fg.isTabFunc Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -1951,15 +1951,11 @@ /* The "table" is actually a sub-select or a view in the FROM clause ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ if( iColpEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif + && (!ViewCanHaveRowid || iCol>=0) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. */ @@ -5129,10 +5125,14 @@ ** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). ** ** (10) The inner query is not the right-hand table of a RIGHT JOIN. ** ** (11) The subquery is not a VALUES clause +** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. ** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ static int pushDownWhereTerms( @@ -5237,10 +5237,22 @@ } if( ExprHasProperty(pWhere,EP_OuterON) && pWhere->w.iJoin!=iCursor ){ return 0; /* restriction (5) */ + } +#endif + +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } } #endif if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ nChng++; @@ -5867,16 +5879,18 @@ pTab->zName = sqlite3MPrintf(pParse->db, "%!S", pFrom); } while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } @@ -6140,11 +6154,11 @@ if( pFrom->fg.isNestedFrom ){ assert( pFrom->pSelect!=0 ); pNestedFrom = pFrom->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); - assert( VisibleRowid(pTab)==0 ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; } pNestedFrom = 0; @@ -6172,11 +6186,12 @@ } }else{ pUsing = 0; } - nAdd = pTab->nCol + (VisibleRowid(pTab) && (selFlags&SF_NestedFrom)); + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; for(j=0; jnCol ){ @@ -6254,11 +6269,12 @@ break; /* OOM */ } pX = &pNew->a[pNew->nExpr-1]; assert( pX->zEName==0 ); if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - if( pNestedFrom ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ pX->zEName = sqlite3MPrintf(db, "%s.%s.%s", zSchemaName, zTabName, zName); Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -3762,10 +3762,11 @@ const char *zShowNull = p->nullValue; rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ) return; nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; nAlloc = nColumn*4; if( nAlloc<=0 ) nAlloc = 1; azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); shell_check_oom(azData); azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); @@ -3847,11 +3848,10 @@ n = strlenChar(z); j = i%nColumn; if( n>p->actualWidth[j] ) p->actualWidth[j] = n; } if( seenInterrupt ) goto columnar_end; - if( nColumn==0 ) goto columnar_end; switch( p->cMode ){ case MODE_Column: { colSep = " "; rowSep = "\n"; if( p->showHeader ){ @@ -8696,20 +8696,19 @@ }else #ifndef SQLITE_SHELL_FIDDLE if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* within this schema (may default to "main") */ + char *zSchema = 0; /* Schema of zTable */ char *zFile = 0; /* Name of file to extra content from */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ - int nByte; /* Number of bytes in an SQL string */ + i64 nByte; /* Number of bytes in an SQL string */ int i, j; /* Loop counters */ int needCommit; /* True to COMMIT or ROLLBACK at end */ int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql; /* An SQL statement */ - char *zFullTabName; /* Table name with schema if applicable */ + char *zSql = 0; /* An SQL statement */ ImportCtx sCtx; /* Reader context */ char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ int eVerbose = 0; /* Larger for more console output */ int nSkip = 0; /* Initial lines to skip */ int useOutputMode = 1; /* Use output mode to determine separators */ @@ -8839,28 +8838,18 @@ } /* Below, resources must be freed before exit. */ while( (nSkip--)>0 ){ while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} } - if( zSchema!=0 ){ - zFullTabName = sqlite3_mprintf("\"%w\".\"%w\"", zSchema, zTable); - }else{ - zFullTabName = sqlite3_mprintf("\"%w\"", zTable); - } - zSql = sqlite3_mprintf("SELECT * FROM %s", zFullTabName); - if( zSql==0 || zFullTabName==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - nByte = strlen30(zSql); - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) ){ + /* Table does not exist. Create it. */ sqlite3 *dbCols = 0; char *zRenames = 0; char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE %s", zFullTabName); + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); while( xRead(&sCtx) ){ zAutoColumn(sCtx.z, &dbCols, 0); if( sCtx.cTerm!=sCtx.cColSep ) break; } zColDefs = zAutoColumn(0, &dbCols, &zRenames); @@ -8871,47 +8860,68 @@ sqlite3_free(zRenames); } assert(dbCols==0); if( zColDefs==0 ){ eputf("%s: empty file\n", sCtx.zFile); - import_fail: - sqlite3_free(zCreate); - sqlite3_free(zSql); - sqlite3_free(zFullTabName); import_cleanup(&sCtx); rc = 1; goto meta_command_exit; } zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } if( eVerbose>=1 ){ oputf("%s\n", zCreate); } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + zCreate = 0; if( rc ){ eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zCreate); - zCreate = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + nByte = strlen(zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ if (pStmt) sqlite3_finalize(pStmt); eputf("Error: %s\n", sqlite3_errmsg(p->db)); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zSql); - nCol = sqlite3_column_count(pStmt); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zFullTabName); + if( zSchema ){ + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } j = strlen30(zSql); for(i=1; i=2 ){ oputf("Insert using: %s\n", zSql); } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ eputf("Error: %s\n", sqlite3_errmsg(p->db)); if (pStmt) sqlite3_finalize(pStmt); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zSql); - sqlite3_free(zFullTabName); needCommit = sqlite3_get_autocommit(p->db); if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); do{ int startLine = sCtx.nLine; for(i=0; i The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ int sqlite3_exec( sqlite3*, /* An open database */ const char *sql, /* SQL to be evaluated */ @@ -2139,10 +2141,26 @@ ** size can be adjusted up or down for individual databases using the ** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ #define SQLITE_CONFIG_SERIALIZED 3 /* nil */ @@ -2170,10 +2188,11 @@ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ #define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options ** ** These constants are the available integer configuration options that Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1121,10 +1121,11 @@ ** 0x00004000 Push-down optimization ** 0x00008000 After all FROM-clause analysis ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated +** 0x00080000 NOT NULL strength reduction */ /* ** Macros for "wheretrace" */ @@ -2527,10 +2528,19 @@ /* Does the table have a rowid */ #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** ** A foreign key is associated with two tables. The "from" table is ** the table that contains the REFERENCES clause that creates the foreign @@ -3446,10 +3456,11 @@ #define NC_HasWin 0x008000 /* One or more window functions seen */ #define NC_IsDDL 0x010000 /* Resolving names in a CREATE statement */ #define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ #define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ #define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */ #define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ /* ** An instance of the following object describes a single ON CONFLICT ** clause in an upsert. @@ -3469,10 +3480,11 @@ Expr *pUpsertTargetWhere; /* WHERE clause for partial index targets */ ExprList *pUpsertSet; /* The SET clause from an ON CONFLICT UPDATE */ Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + u8 isDup; /* True if 2nd or later with same pUpsertIdx */ /* Above this point is the parse tree for the ON CONFLICT clauses. ** The next group of fields stores intermediate data. */ void *pToFree; /* Free memory when deleting the Upsert object */ /* All fields above are owned by the Upsert object and must be freed ** when the Upsert is destroyed. The fields below are used to transfer @@ -4240,10 +4252,15 @@ sqlite3_int64 mxMemdbSize; /* Default max memdb size */ #endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ +#endif int bLocaltimeFault; /* True to fail localtime() calls */ int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ int iOnceResetThreshold; /* When to reset OP_Once counters */ u32 szSorterRef; /* Min size in bytes to use sorter-refs */ unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ @@ -4695,14 +4712,17 @@ #ifndef SQLITE_OMIT_FLOATING_POINT # define EXP754 (((u64)0x7ff)<<52) # define MAN754 ((((u64)1)<<52)-1) # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) int sqlite3IsNaN(double); + int sqlite3IsOverflow(double); #else -# define IsNaN(X) 0 -# define sqlite3IsNaN(X) 0 +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* ** An instance of the following structure holds information about SQL ** functions arguments that are the parameters to the printf() function. @@ -5544,11 +5564,11 @@ #endif #ifndef SQLITE_OMIT_UPSERT Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); void sqlite3UpsertDelete(sqlite3*,Upsert*); Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); - int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); + int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*); void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); int sqlite3UpsertNextIsIPK(Upsert*); #else #define sqlite3UpsertNew(u,v,w,x,y,z) ((Upsert*)0) Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -988,10 +988,43 @@ ){ sqlite3_int64 v = sqlite3_value_int64(argv[0]); sqlite3_result_int64(context, v); sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, context); } + +/* +** These SQL functions attempt to return a value (their first argument) +** that has been modified to have multiple datatypes. For example both +** TEXT and INTEGER. +*/ +static void addTextTypeFunction( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)sqlite3_value_text(argv[0]); + (void)argc; + sqlite3_result_value(context, argv[0]); +} +static void addIntTypeFunction( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)sqlite3_value_int64(argv[0]); + (void)argc; + sqlite3_result_value(context, argv[0]); +} +static void addRealTypeFunction( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)sqlite3_value_double(argv[0]); + (void)argc; + sqlite3_result_value(context, argv[0]); +} /* ** SQL function: strtod(X) ** ** Use the C-library strtod() function to convert string X into a double. @@ -1100,10 +1133,26 @@ */ if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "intreal", 1, SQLITE_UTF8, 0, intrealFunction, 0, 0); } + + /* The add_text_type(), add_int_type(), and add_real_type() functions + ** attempt to return a value that has multiple datatypes. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "add_text_type", 1, SQLITE_UTF8, + 0, addTextTypeFunction, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "add_int_type", 1, SQLITE_UTF8, + 0, addIntTypeFunction, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "add_real_type", 1, SQLITE_UTF8, + 0, addRealTypeFunction, 0, 0); + } /* Functions strtod() and dtostr() work as in the shell. These routines ** use the standard C library to convert between floating point and ** text. This is used to compare SQLite's internal conversion routines ** against the standard library conversion routines. Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -56,10 +56,18 @@ #ifdef SQLITE_32BIT_ROWID Tcl_SetVar2(interp, "sqlite_options", "rowid32", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "rowid32", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + Tcl_SetVar2( + interp, "sqlite_options", "allow_rowid_in_view", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2( + interp, "sqlite_options", "allow_rowid_in_view", "0", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_CASE_SENSITIVE_LIKE Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","1",TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","0",TCL_GLOBAL_ONLY); Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -919,10 +919,13 @@ }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } /* Populate the array of registers beginning at regNew with the new Index: src/upsert.c ================================================================== --- src/upsert.c +++ src/upsert.c @@ -88,11 +88,12 @@ ** is wrong. */ int sqlite3UpsertAnalyzeTarget( Parse *pParse, /* The parsing context */ SrcList *pTabList, /* Table into which we are inserting */ - Upsert *pUpsert /* The ON CONFLICT clauses */ + Upsert *pUpsert, /* The ON CONFLICT clauses */ + Upsert *pAll /* Complete list of all ON CONFLICT clauses */ ){ Table *pTab; /* That table into which we are inserting */ int rc; /* Result code */ int iCursor; /* Cursor used by pTab */ Index *pIdx; /* One of the indexes of pTab */ @@ -191,10 +192,18 @@ /* Column ii of the index did not match any term of the conflict target. ** Continue the search with the next index. */ continue; } pUpsert->pUpsertIdx = pIdx; + if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){ + /* Really this should be an error. The isDup ON CONFLICT clause will + ** never fire. But this problem was not discovered until three years + ** after multi-CONFLICT upsert was added, and so we silently ignore + ** the problem to prevent breaking applications that might actually + ** have redundant ON CONFLICT clauses. */ + pUpsert->isDup = 1; + } break; } if( pUpsert->pUpsertIdx==0 ){ char zWhich[16]; if( nClause==0 && pUpsert->pNextUpsert==0 ){ @@ -217,13 +226,17 @@ */ int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ Upsert *pNext; if( NEVER(pUpsert==0) ) return 0; pNext = pUpsert->pNextUpsert; - if( pNext==0 ) return 1; - if( pNext->pUpsertTarget==0 ) return 1; - if( pNext->pUpsertIdx==0 ) return 1; + while( 1 /*exit-by-return*/ ){ + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + if( !pNext->isDup ) return 0; + pNext = pNext->pNextUpsert; + } return 0; } /* ** Given the list of ON CONFLICT clauses described by pUpsert, and Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -66,10 +66,23 @@ testcase( rc ); return rc; } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is NaN or +Inf or -Inf. +*/ +int sqlite3IsOverflow(double x){ + int rc; /* The value return */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsOvfl(y); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. ** ** The value returned will never be negative. Nor will it ever be greater @@ -625,10 +638,13 @@ }else{ double rr[2]; u64 s2; rr[0] = (double)s; s2 = (u64)rr[0]; +#if defined(_MSC_VER) && _MSC_VER<1700 + if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } +#endif rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); if( e>0 ){ while( e>=100 ){ e -= 100; dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); @@ -1067,11 +1083,11 @@ assert( i>=0 && izBuf)-1 ); p->n = sizeof(p->zBuf) - 1 - i; assert( p->n>0 ); assert( p->nzBuf) ); p->iDP = p->n + exp; - if( iRound<0 ){ + if( iRound<=0 ){ iRound = p->iDP - iRound; if( iRound==0 && p->zBuf[i+1]>='5' ){ iRound = 1; p->zBuf[i--] = '0'; p->n++; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -2299,20 +2299,24 @@ if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ - if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags1 & MEM_Str)!=0 ){ + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags3 & MEM_Str)!=0 ){ + pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); testcase( pIn3->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn3, encoding, 1); testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -4057,10 +4057,27 @@ assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); swapMixedEndianFloat(x); memcpy(&pMem->u.r, &x, sizeof(x)); pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } +} +static int serialGet7( + const unsigned char *buf, /* Buffer to deserialize from */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + if( IsNaN(x) ){ + pMem->flags = MEM_Null; + return 1; + } + pMem->flags = MEM_Real; + return 0; } void sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ @@ -4737,11 +4754,11 @@ if( serial_type>=10 ){ rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + serialGet7(&aKey1[d1], &mem1); rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); i64 rhs = pRhs->u.i; if( lhsu.r ){ + if( serialGet7(&aKey1[d1], &mem1) ){ + rc = -1; /* mem1 is a NaN */ + }else if( mem1.u.ru.r ){ rc = -1; }else if( mem1.u.r>pRhs->u.r ){ rc = +1; + }else{ + assert( rc==0 ); } }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } } @@ -4839,11 +4860,18 @@ } /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0 && serial_type!=10); + if( serial_type==0 + || serial_type==10 + || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0) + ){ + assert( rc==0 ); + }else{ + rc = 1; + } } if( rc!=0 ){ int sortFlags = pPKey2->pKeyInfo->aSortFlags[i]; if( sortFlags ){ Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -5848,20 +5848,14 @@ assert( pIdx->bHasExpr ); pTab = pIdx->pTable; for(i=0; inColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } if( sqlite3ExprIsConstant(pExpr) ) continue; if( pExpr->op==TK_FUNCTION ){ @@ -5889,11 +5883,11 @@ #endif p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS p->zIdxName = pIdx->zName; @@ -6054,11 +6048,14 @@ db = pParse->db; memset(&sWLB, 0, sizeof(sWLB)); /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); - if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; + if( pOrderBy && pOrderBy->nExpr>=BMS ){ + pOrderBy = 0; + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ testcase( pTabList->nSrc==BMS ); Index: test/default.test ================================================================== --- test/default.test +++ test/default.test @@ -134,7 +134,11 @@ reset_db do_catchsql_test default-5.1 { CREATE TABLE t1 (a,b DEFAULT(random() NOTNULL IN (RAISE(IGNORE),2,3))); INSERT INTO t1(a) VALUES(1); } {1 {RAISE() may only be used within a trigger-program}} +do_catchsql_test default-5.2 { + CREATE TABLE Table0 (Col0 DEFAULT (RAISE(IGNORE) ) ) ; + INSERT INTO Table0 DEFAULT VALUES ; +} {1 {RAISE() may only be used within a trigger-program}} finish_test Index: test/fts4intck1.test ================================================================== --- test/fts4intck1.test +++ test/fts4intck1.test @@ -51,8 +51,25 @@ proc slang {in} {return $in} do_execsql_test 2.3 { PRAGMA integrity_check(t2); } {{malformed inverted index for FTS4 table main.t2}} + +#------------------------------------------------------------------------- +# Test that integrity-check works on a read-only database. +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts4(a, b); + INSERT INTO x1 VALUES('one', 'two'); + INSERT INTO x1 VALUES('three', 'four'); +} +db close +sqlite3 db test.db -readonly 1 + +do_execsql_test 3.1 { + PRAGMA integrity_check; +} {ok} + finish_test Index: test/func.test ================================================================== --- test/func.test +++ test/func.test @@ -783,10 +783,15 @@ sqlite3_finalize $::STMT execsql { SELECT quote(a), quote(b) FROM tbl2; } } {X'616263' NULL} + +# Test the quote function for +Inf and -Inf +do_execsql_test func-16.2 { + SELECT quote(4.2e+859), quote(-7.8e+904); +} {9.0e+999 -9.0e+999} # Correctly handle function error messages that include %. Ticket #1354 # do_test func-17.1 { proc testfunc1 args {error "Error %d with %s percents %p"} @@ -1040,10 +1045,13 @@ do_test func-21.8 { execsql { SELECT replace('aaaaaaa', 'a', '0123456789'); } } {0123456789012345678901234567890123456789012345678901234567890123456789} +do_execsql_test func-21.9 { + SELECT typeof(replace(1,'',0)); +} {text} ifcapable tclvar { do_test func-21.9 { # Attempt to exploit a buffer-overflow that at one time existed # in the REPLACE function. @@ -1550,7 +1558,28 @@ # do_execsql_test func-38.100 { WITH t1(x) AS (VALUES(9e+999)) SELECT sum(x), avg(x), total(x) FROM t1; WITH t1(x) AS (VALUES(-9e+999)) SELECT sum(x), avg(x), total(x) FROM t1; } {Inf Inf Inf -Inf -Inf -Inf} + +# 2024-03-21 https://sqlite.org/forum/forumpost/23b8688ef4 +# Another problem with Kahan-Babushka-Neumaier summation and +# infinities. +# +do_execsql_test func-39.101 { + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1) + SELECT sum(1.7976931348623157e308), + avg(1.7976931348623157e308), + total(1.7976931348623157e308) + FROM c; +} {1.79769313486232e+308 1.79769313486232e+308 1.79769313486232e+308} +for {set i 2} {$i<10} {incr i} { + do_execsql_test func-39.[expr {10*$i+100}] { + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<$i) + SELECT sum(1.7976931348623157e308), + avg(1.7976931348623157e308), + total(1.7976931348623157e308) + FROM c; + } {Inf Inf Inf} +} finish_test Index: test/fuzzcheck.c ================================================================== --- test/fuzzcheck.c +++ test/fuzzcheck.c @@ -159,12 +159,12 @@ } g; /* ** Include the external vt02.c and randomjson.c modules. */ -extern int sqlite3_vt02_init(sqlite3*,char***,void*); -extern int sqlite3_randomjson_init(sqlite3*,char***,void*); +extern int sqlite3_vt02_init(sqlite3*,char**,const sqlite3_api_routines*); +extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*); /* ** Print an error message and quit. */ Index: test/fuzzinvariants.c ================================================================== --- test/fuzzinvariants.c +++ test/fuzzinvariants.c @@ -294,10 +294,18 @@ ){ /* This is a randomized column name and so cannot be used in the ** WHERE clause. */ continue; } +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( sqlite3_strlike("%rowid%",zColName,0)==0 + || sqlite3_strlike("%oid%",zColName,0)==0 + ){ + /* ROWID values are unreliable if SQLITE_ALLOW_ROWID_IN_VIEW is used */ + continue; + } +#endif for(j=0; jjx.atom AND type NOT IN ('array','object'); } {} + +# 2024-02-16 https://sqlite.org/forum/forumpost/ecb94cd210 +# Regression in json_tree()/json_each(). The value column +# should have the "J" subtype if the value is an array or +# object. +# +do_execsql_test json101-5.10 { + SELECT json_insert('{}','$.a',value) FROM json_tree('[1,2,3]') WHERE atom IS NULL; +} {{{"a":[1,2,3]}}} +# ^^^^^^^--- In double-quotes, a string literal, prior to bug fix + +do_execsql_test json101-5.11 { + SELECT json_insert('{}','$.a',value) FROM json_tree('"[1,2,3]"'); +} {{{"a":"[1,2,3]"}}} do_execsql_test json101-6.1 { SELECT json_valid('{"a":55,"b":72,}'); } {0} do_execsql_test json101-6.2 { ADDED test/json107.test Index: test/json107.test ================================================================== --- /dev/null +++ test/json107.test @@ -0,0 +1,86 @@ +# 2024-01-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Legacy JSON bug: If the input is a BLOB that when cast into TEXT looks +# like valid JSON, then treat it as valid JSON. +# +# The original intent of the JSON functions was to raise an error on any +# BLOB input. That intent was clearly documented, but the code failed to +# to implement it. Subsequently, many applications began to depend on the +# incorrect behavior, especially apps that used readfile() to read JSON +# content, since readfile() returns a BLOB. So we need to support the +# bug moving forward. +# +# The tests in this fail verify that the original buggy behavior is +# preserved. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json107 + +if {[db one {PRAGMA encoding}]!="UTF-8"} { + # These tests only work for a UTF-8 encoding. + finish_test + return +} + +do_execsql_test 1.1 { + SELECT json_valid( CAST('{"a":1}' AS BLOB) ); +} 1 +do_execsql_test 1.1.1 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 1); +} 1 +do_execsql_test 1.1.2 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 2); +} 1 +do_execsql_test 1.1.4 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 4); +} 0 +do_execsql_test 1.1.8 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 8); +} 0 + +do_execsql_test 1.2.1 { + SELECT CAST('{"a":123}' AS blob) -> 'a'; +} 123 +do_execsql_test 1.2.2 { + SELECT CAST('{"a":123}' AS blob) ->> 'a'; +} 123 +do_execsql_test 1.2.3 { + SELECT json_extract(CAST('{"a":123}' AS blob), '$.a'); +} 123 +do_execsql_test 1.3 { + SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456); +} {{{"a":123,"b":456}}} +do_execsql_test 1.4 { + SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a'); +} {{{"b":456}}} +do_execsql_test 1.5 { + SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789); +} {{{"a":789,"b":456}}} +do_execsql_test 1.6 { + SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789); +} {{{"a":789,"b":456}}} +do_execsql_test 1.7 { + SELECT json_type(CAST('{"a":123,"b":456}' AS blob)); +} object +do_execsql_test 1.8 { + SELECT json(CAST('{"a":123,"b":456}' AS blob)); +} {{{"a":123,"b":456}}} + +ifcapable vtab { + do_execsql_test 2.1 { + SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) ) + WHERE atom; + } {a 123 b 456} +} +finish_test Index: test/jsonb01.test ================================================================== --- test/jsonb01.test +++ test/jsonb01.test @@ -43,7 +43,11 @@ } $res do_execsql_test jsonb01-1.2.$id.2 { SELECT json_remove(x,$path) FROM t1; } $res } + +do_catchsql_test jsonb01-2.0 { + SELECT x'8ce6ffffffff171333' -> '$'; +} {1 {malformed JSON}} finish_test Index: test/memdb1.test ================================================================== --- test/memdb1.test +++ test/memdb1.test @@ -82,11 +82,10 @@ } {1 2} do_test 152 { catchsql {INSERT INTO t1 VALUES(3,4);} } {1 {attempt to write a readonly database}} -breakpoint do_test 160 { db deserialize -maxsize 32768 $db1 db eval {SELECT * FROM t1} } {1 2} do_test 161 { @@ -246,10 +245,11 @@ db close set fd [open test.db] fconfigure $fd -translation binary -encoding binary set data [read $fd [expr 20*1024]] + close $fd sqlite3 db "" db deserialize $data do_execsql_test 810 { @@ -265,6 +265,19 @@ do_catchsql_test 830 { PRAGMA wal_checkpoint; } {1 {database disk image is malformed}} } +# 2024-01-20 +# https://sqlite.org/forum/forumpost/498777780e16880a +# +# Make sure a database is initialized before serializing it. +# +reset_db +sqlite3 dbempty :memory: +do_test 900 { + set len [string length [dbempty serialize]] + expr {$len>0} +} 1 +dbempty close + finish_test Index: test/misc2.test ================================================================== --- test/misc2.test +++ test/misc2.test @@ -52,23 +52,38 @@ CREATE TABLE t2(a,b,c); INSERT INTO t2 VALUES(7,8,9); } } {} ifcapable subquery { - do_catchsql_test misc2-2.2 { - SELECT rowid, * FROM (SELECT * FROM t1, t2); - } {1 {no such column: rowid}} + ifcapable allow_rowid_in_view { + do_catchsql_test misc2-2.2 { + SELECT rowid, * FROM (SELECT * FROM t1, t2); + } {0 {{} 1 2 3 7 8 9}} + } else { + do_catchsql_test misc2-2.2 { + SELECT rowid, * FROM (SELECT * FROM t1, t2); + } {1 {no such column: rowid}} + } do_catchsql_test misc2-2.2b { SELECT 'rowid', * FROM (SELECT * FROM t1, t2); } {0 {rowid 1 2 3 7 8 9}} } ifcapable view { - do_catchsql_test misc2-2.3 { - CREATE VIEW v1 AS SELECT * FROM t1, t2; - SELECT rowid, * FROM v1; - } {1 {no such column: rowid}} + ifcapable allow_rowid_in_view { + do_catchsql_test misc2-2.3 { + CREATE VIEW v1 AS SELECT * FROM t1, t2; + SELECT rowid, * FROM v1; + } {0 {{} 1 2 3 7 8 9}} + } else { + do_catchsql_test misc2-2.3 { + CREATE VIEW v1 AS SELECT * FROM t1, t2; + SELECT rowid, * FROM v1; + } {1 {no such column: rowid}} + } + + do_catchsql_test misc2-2.3b { SELECT 'rowid', * FROM v1; } {0 {rowid 1 2 3 7 8 9}} } ;# ifcapable view Index: test/misc8.test ================================================================== --- test/misc8.test +++ test/misc8.test @@ -98,10 +98,15 @@ 0 10 {} 10 {} {} } # 2016-02-26: An assertion fault found by the libFuzzer project # +ifcapable allow_rowid_in_view { + set nosuch "1 {ambiguous column name: rowid}" +} else { + set nosuch "1 {no such column: rowid}" +} do_catchsql_test misc8-3.0 { SELECT * FROM ( (SELECT 0 AS i) AS x1, @@ -108,11 +113,11 @@ (SELECT 1) AS x2 ) AS x3, (SELECT 6 AS j UNION ALL SELECT 7) AS x4 WHERE i0 AND 0%y; -} {1 {no such column: rowid}} +} {1 {ambiguous column name: rowid}} } reset_db do_execsql_test 6.0 { CREATE TABLE t1(a,b); Index: test/upsert5.test ================================================================== --- test/upsert5.test +++ test/upsert5.test @@ -405,7 +405,49 @@ INSERT INTO t2(a,b,c,e,d) VALUES(1,2,3,4,5) ON CONFLICT(c) DO UPDATE SET b='' ON CONFLICT((SELECT t2 FROM nosuchtable)) DO NOTHING; } {1 {no such table: nosuchtable}} + +# 2024-03-08 https://sqlite.org/forum/forumpost/919c6579c8 +# A redundant ON CONFLICT clause in an upsert can lead to +# index corruption. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(aa INTEGER PRIMARY KEY, bb INT); + INSERT INTO t1 VALUES(11,22); + CREATE UNIQUE INDEX t1bb ON t1(bb); + REPLACE INTO t1 VALUES(11,33) + ON CONFLICT(bb) DO UPDATE SET aa = 44 + ON CONFLICT(bb) DO UPDATE SET aa = 44; + PRAGMA integrity_check; +} {ok} +do_execsql_test 3.1 { + SELECT * FROM t1 NOT INDEXED; +} {11 33} +do_execsql_test 3.2 { + SELECT * FROM t1 INDEXED BY t1bb; +} {11 33} +do_execsql_test 3.3 { + DROP TABLE t1; + CREATE TABLE t1(aa INTEGER PRIMARY KEY, bb INT, cc INT); + INSERT INTO t1 VALUES(10,21,32),(11,22,33),(12,23,34); + CREATE UNIQUE INDEX t1bb ON t1(bb); + CREATE UNIQUE INDEX t1cc ON t1(cc); + REPLACE INTO t1 VALUES(11,44,55) + ON CONFLICT(bb) DO UPDATE SET aa = 99 + ON CONFLICT(cc) DO UPDATE SET aa = 99 + ON CONFLICT(bb) DO UPDATE SET aa = 99; + PRAGMA integrity_check; +} {ok} +do_execsql_test 3.4 { + SELECT * FROM t1 NOT INDEXED ORDER BY +aa; +} {10 21 32 11 44 55 12 23 34} +do_execsql_test 3.5 { + SELECT * FROM t1 INDEXED BY t1bb ORDER BY +aa; +} {10 21 32 11 44 55 12 23 34} +do_execsql_test 3.6 { + SELECT * FROM t1 INDEXED BY t1cc ORDER BY +aa; +} {10 21 32 11 44 55 12 23 34} finish_test Index: test/vacuum-into.test ================================================================== --- test/vacuum-into.test +++ test/vacuum-into.test @@ -24,17 +24,40 @@ return } forcedelete out.db do_execsql_test vacuum-into-100 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b ANY, + c INT AS (b+1), --- See "2024-04-09" block + CHECK( typeof(b)!='integer' OR b>a-5 ) --- comment below + ); WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c; CREATE INDEX t1b ON t1(b); DELETE FROM t1 WHERE a%2; SELECT count(*), sum(a), sum(length(b)) FROM t1; } {50 2550 30000} + +# Update 2024-04-09 for forum post eec177d68fe7fa2c. +# +# VACUUM INTO is sensitive to tables holding both generated columns +# and CHECK constraints. +# +# CHECK constraints are ignored for read-only databases in order to save +# memory (see check-in 34ddf02d3d21151b on 2014-05-21). But the xfer +# optimization normally only works if CHECK constraints match between the +# source and destination tables. So the xfer optimization was not +# working for VACUUM INTO when the source was a read-only database and the +# table held CHECK constraints. But if the table has generated columns, +# then the xfer optimization is required or else VACUUM will raise an +# error. +# +# Fix this by ignoring CHECK constraints when determining whether or not +# the xfer optimization can run while doing VACUUM. + do_execsql_test vacuum-into-110 { VACUUM main INTO 'out.db'; } {} sqlite3 db2 out.db do_test vacuum-into-120 { @@ -86,15 +109,25 @@ VACUUM INTO target2() } {1 {no such function: target2}} # The ability to VACUUM INTO a read-only database db close +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 1 +} else { + file attributes test.db -permissions 292 ;# 292 == 0444 +} sqlite3 db test.db -readonly 1 forcedelete test.db2 do_execsql_test vacuum-into-500 { VACUUM INTO 'test.db2'; } +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 0 +} else { + file attributes test.db -permissions 420 ;# 420 = 0644 +} sqlite3 db2 test.db2 do_test vacuum-into-510 { db2 eval {SELECT name FROM sqlite_master ORDER BY 1} } {t1 t1b t2} db2 close Index: tool/mkctimec.tcl ================================================================== --- tool/mkctimec.tcl +++ tool/mkctimec.tcl @@ -103,10 +103,11 @@ # All Boolean compile time options which default to 0 or empty. # set boolean_defnil_options { SQLITE_32BIT_ROWID SQLITE_4_BYTE_ALIGNED_MALLOC + SQLITE_ALLOW_ROWID_IN_VIEW SQLITE_ALLOW_URI_AUTHORITY SQLITE_BUG_COMPATIBLE_20160819 SQLITE_CASE_SENSITIVE_LIKE SQLITE_CHECK_PAGES SQLITE_COVERAGE_TEST