Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -591,11 +591,12 @@ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db \ $(TOP)/test/fuzzdata4.db \ $(TOP)/test/fuzzdata5.db \ - $(TOP)/test/fuzzdata6.db + $(TOP)/test/fuzzdata6.db \ + $(TOP)/test/fuzzdata7.db # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt @@ -608,10 +609,11 @@ SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 @@ -675,10 +677,26 @@ $(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS) dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS) +DBFUZZ2_OPTS = \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_DEBUG \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_EANBLE_FTS5 + +dbfuzz2: $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h + clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined -o dbfuzz2 \ + $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c + mkdir -p dbfuzz2-dir + cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir + mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \ $(TLIBS) -rpath "$(libdir)" MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1621,20 +1621,22 @@ $(TOP)\test\fuzzdata1.db \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db \ $(TOP)\test\fuzzdata4.db \ $(TOP)\test\fuzzdata5.db \ - $(TOP)\test\fuzzdata6.db + $(TOP)\test\fuzzdata6.db \ + $(TOP)\test\fuzzdata7.db # <> # Additional compiler options for the shell. These are only effective # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF # <> # Extra compiler options for various test tools. # Index: ext/session/session1.test ================================================================== --- ext/session/session1.test +++ ext/session/session1.test @@ -652,11 +652,18 @@ execsql BEGIN catch { sqlite3changeset_apply_v2 -nosavepoint db $C xConflict } msg execsql { SELECT * FROM x1 } } {1 one 2 two 3 iii} execsql ROLLBACK + +do_test $tn.14.1 { sqlite3session_config strm_size -1 } 64 +do_test $tn.14.2 { sqlite3session_config strm_size 65536 } 65536 +do_test $tn.14.3 { sqlite3session_config strm_size 64 } 64 +do_test $tn.14.4 { + list [catch {sqlite3session_config invalid 123} msg] $msg +} {1 SQLITE_MISUSE} }] } finish_test Index: ext/session/sqlite3session.c ================================================================== --- ext/session/sqlite3session.c +++ ext/session/sqlite3session.c @@ -23,10 +23,12 @@ # else # define SESSIONS_STRM_CHUNK_SIZE 1024 # endif #endif +static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; + typedef struct SessionHook SessionHook; struct SessionHook { void *pCtx; int (*xOld)(void*,int,sqlite3_value**); int (*xNew)(void*,int,sqlite3_value**); @@ -242,10 +244,46 @@ ** and fields associated with modified columns contain the new column values. ** ** The records associated with INSERT changes are in the same format as for ** changesets. It is not possible for a record associated with an INSERT ** change to contain a field set to "undefined". +** +** REBASE BLOB FORMAT: +** +** A rebase blob may be output by sqlite3changeset_apply_v2() and its +** streaming equivalent for use with the sqlite3_rebaser APIs to rebase +** existing changesets. A rebase blob contains one entry for each conflict +** resolved using either the OMIT or REPLACE strategies within the apply_v2() +** call. +** +** The format used for a rebase blob is very similar to that used for +** changesets. All entries related to a single table are grouped together. +** +** Each group of entries begins with a table header in changeset format: +** +** 1 byte: Constant 0x54 (capital 'T') +** Varint: Number of columns in the table. +** nCol bytes: 0x01 for PK columns, 0x00 otherwise. +** N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +** +** Followed by one or more entries associated with the table. +** +** 1 byte: Either SQLITE_INSERT (0x12), DELETE (0x09). +** 1 byte: Flag. 0x01 for REPLACE, 0x00 for OMIT. +** record: (in the record format defined above). +** +** In a rebase blob, the first field is set to SQLITE_INSERT if the change +** that caused the conflict was an INSERT or UPDATE, or to SQLITE_DELETE if +** it was a DELETE. The second field is set to 0x01 if the conflict +** resolution strategy was REPLACE, or 0x00 if it was OMIT. +** +** If the change that caused the conflict was a DELETE, then the single +** record is a copy of the old.* record from the original changeset. If it +** was an INSERT, then the single record is a copy of the new.* record. If +** the conflicting change was an UPDATE, then the single record is a copy +** of the new.* record with the PK fields filled in based on the original +** old.* record. */ /* ** For each row modified during a session, there exists a single instance of ** this structure stored in a SessionTable.aChange[] hash table. @@ -2395,16 +2433,16 @@ } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pSel); } - /* If the buffer is now larger than SESSIONS_STRM_CHUNK_SIZE, pass + /* If the buffer is now larger than sessions_strm_chunk_size, pass ** its contents to the xOutput() callback. */ if( xOutput && rc==SQLITE_OK && buf.nBuf>nNoop - && buf.nBuf>SESSIONS_STRM_CHUNK_SIZE + && buf.nBuf>sessions_strm_chunk_size ){ rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf); nNoop = -1; buf.nBuf = 0; } @@ -2612,11 +2650,11 @@ /* ** If the SessionInput object passed as the only argument is a streaming ** object and the buffer is full, discard some data to free up space. */ static void sessionDiscardData(SessionInput *pIn){ - if( pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){ + if( pIn->xInput && pIn->iNext>=sessions_strm_chunk_size ){ int nMove = pIn->buf.nBuf - pIn->iNext; assert( nMove>=0 ); if( nMove>0 ){ memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove); } @@ -2635,11 +2673,11 @@ */ static int sessionInputBuffer(SessionInput *pIn, int nByte){ int rc = SQLITE_OK; if( pIn->xInput ){ while( !pIn->bEof && (pIn->iNext+nByte)>=pIn->nData && rc==SQLITE_OK ){ - int nNew = SESSIONS_STRM_CHUNK_SIZE; + int nNew = sessions_strm_chunk_size; if( pIn->bNoDiscard==0 ) sessionDiscardData(pIn); if( SQLITE_OK==sessionBufferGrow(&pIn->buf, nNew, &rc) ){ rc = pIn->xInput(pIn->pIn, &pIn->buf.aBuf[pIn->buf.nBuf], &nNew); if( nNew==0 ){ @@ -3360,11 +3398,11 @@ rc = SQLITE_CORRUPT_BKPT; goto finished_invert; } assert( rc==SQLITE_OK ); - if( xOutput && sOut.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){ + if( xOutput && sOut.nBuf>=sessions_strm_chunk_size ){ rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); sOut.nBuf = 0; if( rc!=SQLITE_OK ) goto finished_invert; } } @@ -3439,11 +3477,12 @@ u8 *abPK; /* Boolean array - true if column is in PK */ int bStat1; /* True if table is sqlite_stat1 */ int bDeferConstraints; /* True to defer constraints */ SessionBuffer constraints; /* Deferred constraints are stored here */ SessionBuffer rebase; /* Rebase information (if any) here */ - int bRebaseStarted; /* If table header is already in rebase */ + u8 bRebaseStarted; /* If table header is already in rebase */ + u8 bRebase; /* True to collect rebase information */ }; /* ** Formulate a statement to DELETE a row from database db. Assuming a table ** structure like this: @@ -3836,39 +3875,40 @@ SessionApplyCtx *p, /* Apply context */ int eType, /* Conflict resolution (OMIT or REPLACE) */ sqlite3_changeset_iter *pIter /* Iterator pointing at current change */ ){ int rc = SQLITE_OK; - int i; - int eOp = pIter->op; - if( p->bRebaseStarted==0 ){ - /* Append a table-header to the rebase buffer */ - const char *zTab = pIter->zTab; - sessionAppendByte(&p->rebase, 'T', &rc); - sessionAppendVarint(&p->rebase, p->nCol, &rc); - sessionAppendBlob(&p->rebase, p->abPK, p->nCol, &rc); - sessionAppendBlob(&p->rebase, (u8*)zTab, (int)strlen(zTab)+1, &rc); - p->bRebaseStarted = 1; - } - - assert( eType==SQLITE_CHANGESET_REPLACE||eType==SQLITE_CHANGESET_OMIT ); - assert( eOp==SQLITE_DELETE || eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE ); - - sessionAppendByte(&p->rebase, - (eOp==SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT), &rc - ); - sessionAppendByte(&p->rebase, (eType==SQLITE_CHANGESET_REPLACE), &rc); - for(i=0; inCol; i++){ - sqlite3_value *pVal = 0; - if( eOp==SQLITE_DELETE || (eOp==SQLITE_UPDATE && p->abPK[i]) ){ - sqlite3changeset_old(pIter, i, &pVal); - }else{ - sqlite3changeset_new(pIter, i, &pVal); - } - sessionAppendValue(&p->rebase, pVal, &rc); - } - + if( p->bRebase ){ + int i; + int eOp = pIter->op; + if( p->bRebaseStarted==0 ){ + /* Append a table-header to the rebase buffer */ + const char *zTab = pIter->zTab; + sessionAppendByte(&p->rebase, 'T', &rc); + sessionAppendVarint(&p->rebase, p->nCol, &rc); + sessionAppendBlob(&p->rebase, p->abPK, p->nCol, &rc); + sessionAppendBlob(&p->rebase, (u8*)zTab, (int)strlen(zTab)+1, &rc); + p->bRebaseStarted = 1; + } + + assert( eType==SQLITE_CHANGESET_REPLACE||eType==SQLITE_CHANGESET_OMIT ); + assert( eOp==SQLITE_DELETE || eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE ); + + sessionAppendByte(&p->rebase, + (eOp==SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT), &rc + ); + sessionAppendByte(&p->rebase, (eType==SQLITE_CHANGESET_REPLACE), &rc); + for(i=0; inCol; i++){ + sqlite3_value *pVal = 0; + if( eOp==SQLITE_DELETE || (eOp==SQLITE_UPDATE && p->abPK[i]) ){ + sqlite3changeset_old(pIter, i, &pVal); + }else{ + sqlite3changeset_new(pIter, i, &pVal); + } + sessionAppendValue(&p->rebase, pVal, &rc); + } + } return rc; } /* ** Invoke the conflict handler for the change that the changeset iterator @@ -4273,10 +4313,11 @@ assert( xConflict!=0 ); pIter->in.bNoDiscard = 1; memset(&sApply, 0, sizeof(sApply)); + sApply.bRebase = (ppRebase && pnRebase); sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); } if( rc==SQLITE_OK ){ @@ -4423,11 +4464,12 @@ sqlite3_exec(db, "ROLLBACK TO changeset_apply", 0, 0, 0); sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0); } } - if( rc==SQLITE_OK && bPatchset==0 && ppRebase && pnRebase ){ + assert( sApply.bRebase || sApply.rebase.nBuf==0 ); + if( rc==SQLITE_OK && bPatchset==0 && sApply.bRebase ){ *ppRebase = (void*)sApply.rebase.aBuf; *pnRebase = sApply.rebase.nBuf; sApply.rebase.aBuf = 0; } sqlite3_finalize(sApply.pInsert); @@ -4893,11 +4935,11 @@ SessionChange *p; for(p=pTab->apChange[i]; p; p=p->pNext){ sessionAppendByte(&buf, p->op, &rc); sessionAppendByte(&buf, p->bIndirect, &rc); sessionAppendBlob(&buf, p->aRecord, p->nRecord, &rc); - if( rc==SQLITE_OK && xOutput && buf.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){ + if( rc==SQLITE_OK && xOutput && buf.nBuf>=sessions_strm_chunk_size ){ rc = xOutput(pOut, buf.aBuf, buf.nBuf); buf.nBuf = 0; } } } @@ -5289,11 +5331,11 @@ if( bDone==0 ){ sessionAppendByte(&sOut, pIter->op, &rc); sessionAppendByte(&sOut, pIter->bIndirect, &rc); sessionAppendBlob(&sOut, aRec, nRec, &rc); } - if( rc==SQLITE_OK && xOutput && sOut.nBuf>SESSIONS_STRM_CHUNK_SIZE ){ + if( rc==SQLITE_OK && xOutput && sOut.nBuf>sessions_strm_chunk_size ){ rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); sOut.nBuf = 0; } if( rc ) break; } @@ -5399,7 +5441,28 @@ if( p ){ sessionDeleteTable(p->grp.pList); sqlite3_free(p); } } + +/* +** Global configuration +*/ +int sqlite3session_config(int op, void *pArg){ + int rc = SQLITE_OK; + switch( op ){ + case SQLITE_SESSION_CONFIG_STRMSIZE: { + int *pInt = (int*)pArg; + if( *pInt>0 ){ + sessions_strm_chunk_size = *pInt; + } + *pInt = sessions_strm_chunk_size; + break; + } + default: + rc = SQLITE_MISUSE; + break; + } + return rc; +} #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ Index: ext/session/sqlite3session.h ================================================================== --- ext/session/sqlite3session.h +++ ext/session/sqlite3session.h @@ -1608,14 +1608,53 @@ void *pIn, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ); +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +**
+**
SQLITE_SESSION_CONFIG_STRMSIZE
+** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +**
+** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 /* ** Make sure we can call this stuff from C++. */ #ifdef __cplusplus } #endif #endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ Index: ext/session/test_session.c ================================================================== --- ext/session/test_session.c +++ ext/session/test_session.c @@ -1236,10 +1236,49 @@ (ClientData)pNew, test_rebaser_del ); Tcl_SetObjResult(interp, objv[1]); return TCL_OK; } + +/* +** tclcmd: sqlite3rebaser_configure OP VALUE +*/ +static int SQLITE_TCLAPI test_sqlite3session_config( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct ConfigOpt { + const char *zSub; + int op; + } aSub[] = { + { "strm_size", SQLITE_SESSION_CONFIG_STRMSIZE }, + { "invalid", 0 }, + { 0 } + }; + int rc; + int iSub; + int iVal; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "OP VALUE"); + return SQLITE_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR; + + rc = sqlite3session_config(aSub[iSub].op, (void*)&iVal); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); + return TCL_OK; +} int TestSession_Init(Tcl_Interp *interp){ struct Cmd { const char *zCmd; Tcl_ObjCmdProc *xProc; @@ -1252,10 +1291,11 @@ { "sqlite3changeset_apply_v2", test_sqlite3changeset_apply_v2 }, { "sqlite3changeset_apply_replace_all", test_sqlite3changeset_apply_replace_all }, { "sql_exec_changeset", test_sql_exec_changeset }, { "sqlite3rebaser_create", test_sqlite3rebaser_create }, + { "sqlite3session_config", test_sqlite3session_config }, }; int i; for(i=0; imutex) ); assert( pDbPage->pBt==pBt ); + if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -2420,11 +2420,11 @@ ** erasing iTable (this can happen with an auto-vacuum database). */ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); - assert( iTable>1 ); + if( iTable<2 ) sqlite3ErrorMsg(pParse, "corrupt schema"); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM /* OP_Destroy stores an in integer r1. If this integer ** is non-zero, then it is the root page number of a table moved to Index: src/dbstat.c ================================================================== --- src/dbstat.c +++ src/dbstat.c @@ -252,18 +252,24 @@ *ppCursor = (sqlite3_vtab_cursor *)pCsr; return SQLITE_OK; } -static void statClearPage(StatPage *p){ +static void statClearCells(StatPage *p){ int i; if( p->aCell ){ for(i=0; inCell; i++){ sqlite3_free(p->aCell[i].aOvfl); } sqlite3_free(p->aCell); } + p->nCell = 0; + p->aCell = 0; +} + +static void statClearPage(StatPage *p){ + statClearCells(p); sqlite3PagerUnref(p->pPg); sqlite3_free(p->zPath); memset(p, 0, sizeof(StatPage)); } @@ -322,26 +328,37 @@ u8 *aData = sqlite3PagerGetData(p->pPg); u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0]; p->flags = aHdr[0]; + if( p->flags==0x0A || p->flags==0x0D ){ + isLeaf = 1; + nHdr = 8; + }else if( p->flags==0x05 || p->flags==0x02 ){ + isLeaf = 0; + nHdr = 12; + }else{ + goto statPageIsCorrupt; + } + if( p->iPgno==1 ) nHdr += 100; p->nCell = get2byte(&aHdr[3]); p->nMxPayload = 0; - - isLeaf = (p->flags==0x0A || p->flags==0x0D); - nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100; + szPage = sqlite3BtreeGetPageSize(pBt); nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell; nUnused += (int)aHdr[7]; iOff = get2byte(&aHdr[1]); while( iOff ){ + int iNext; + if( iOff>=szPage ) goto statPageIsCorrupt; nUnused += get2byte(&aData[iOff+2]); - iOff = get2byte(&aData[iOff]); + iNext = get2byte(&aData[iOff]); + if( iNext0 ) goto statPageIsCorrupt; + iOff = iNext; } p->nUnused = nUnused; p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]); - szPage = sqlite3BtreeGetPageSize(pBt); if( p->nCell ){ int i; /* Used to iterate through cells */ int nUsable; /* Usable bytes per page */ @@ -354,10 +371,11 @@ for(i=0; inCell; i++){ StatCell *pCell = &p->aCell[i]; iOff = get2byte(&aData[nHdr+i*2]); + if( iOff=szPage ) goto statPageIsCorrupt; if( !isLeaf ){ pCell->iChildPg = sqlite3Get4byte(&aData[iOff]); iOff += 4; } if( p->flags==0x05 ){ @@ -370,12 +388,12 @@ u64 dummy; iOff += sqlite3GetVarint(&aData[iOff], &dummy); } if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; getLocalPayload(nUsable, p->flags, nPayload, &nLocal); + if( nLocal<0 ) goto statPageIsCorrupt; pCell->nLocal = nLocal; - assert( nLocal>=0 ); assert( nPayload>=(u32)nLocal ); assert( nLocal<=(nUsable-35) ); if( nPayload>(u32)nLocal ){ int j; int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); @@ -400,10 +418,15 @@ } } } return SQLITE_OK; + +statPageIsCorrupt: + p->flags = 0; + statClearCells(p); + return SQLITE_OK; } /* ** Populate the pCsr->iOffset and pCsr->szPage member variables. Based on ** the current value of pCsr->iPageno. Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1035,11 +1035,11 @@ case PragTyp_FLAG: { if( zRight==0 ){ setPragmaResultColumnNames(v, pPragma); returnSingleInt(v, (db->flags & pPragma->iArg)!=0 ); }else{ - int mask = pPragma->iArg; /* Mask of bits to set or clear. */ + u64 mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ /* Foreign key support may not be enabled or disabled while not ** in auto-commit mode. */ mask &= ~(SQLITE_ForeignKeys); } Index: src/pragma.h ================================================================== --- src/pragma.h +++ src/pragma.h @@ -125,11 +125,11 @@ const char *const zName; /* Name of pragma */ u8 ePragTyp; /* PragTyp_XXX value */ u8 mPragFlg; /* Zero or more PragFlg_XXX values */ u8 iPragCName; /* Start of column names in pragCName[] */ u8 nPragCName; /* Num of col names. 0 means use pragma name */ - u32 iArg; /* Extra argument */ + u64 iArg; /* Extra argument */ } PragmaName; static const PragmaName aPragmaName[] = { #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) {/* zName: */ "activate_extensions", /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, @@ -661,9 +661,9 @@ #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) {/* zName: */ "writable_schema", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_WriteSchema }, + /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; /* Number of pragmas: 62 on by default, 81 total. */ Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -328,12 +328,12 @@ } if( db->mallocFailed ){ rc = SQLITE_NOMEM_BKPT; sqlite3ResetAllSchemasOfConnection(db); } - if( rc==SQLITE_OK || (db->flags&SQLITE_WriteSchema)){ - /* Black magic: If the SQLITE_WriteSchema flag is set, then consider + if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){ + /* Black magic: If the SQLITE_NoSchemaError flag is set, then consider ** the schema loaded, even if errors occurred. In this situation the ** current sqlite3_prepare() operation will fail, but the following one ** will attempt to compile the supplied statement against whatever subset ** of the schema was loaded before the error occurred. The primary ** purpose of this is to allow access to the sqlite_master table Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -1057,15 +1057,16 @@ #define AUTOEQP_trigger 2 /* On and also show plans for triggers */ #define AUTOEQP_full 3 /* Show full EXPLAIN */ /* Allowed values for ShellState.openMode */ -#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ -#define SHELL_OPEN_NORMAL 1 /* Normal database file */ -#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ -#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ -#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ +#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ +#define SHELL_OPEN_NORMAL 1 /* Normal database file */ +#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ +#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ +#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ /* ** These are the allowed shellFlgs values */ #define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ @@ -3430,14 +3431,17 @@ " Other options:", " -e Invoke system text editor", " -x Open in a spreadsheet", ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE", " Options:", - " --append Use appendvfs to append database to the end of FILE", - " --new Initialize FILE to an empty database", - " --readonly Open FILE readonly", - " --zip FILE is a ZIP archive", + " --append Use appendvfs to append database to the end of FILE", +#ifdef SQLITE_ENABLE_DESERIALIZE + " --deserialize Load into memory useing sqlite3_deserialize()", +#endif + " --new Initialize FILE to an empty database", + " --readonly Open FILE readonly", + " --zip FILE is a ZIP archive", ".output ?FILE? Send output to FILE or stdout if FILE is omitted", " If FILE begins with '|' then open it as a pipe.", ".print STRING... Print literal STRING", ".prompt MAIN CONTINUE Replace the standard prompts", ".quit Exit this program", @@ -3671,10 +3675,15 @@ return SHELL_OPEN_ZIPFILE; }else{ return SHELL_OPEN_NORMAL; } } + n = fread(zBuf, 16, 1, f); + if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){ + fclose(f); + return SHELL_OPEN_NORMAL; + } fseek(f, -25, SEEK_END); n = fread(zBuf, 25, 1, f); if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){ rc = SHELL_OPEN_APPENDVFS; }else{ @@ -3721,10 +3730,14 @@ switch( p->openMode ){ case SHELL_OPEN_APPENDVFS: { sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs"); break; + } + case SHELL_OPEN_DESERIALIZE: { + sqlite3_open(0, &p->db); + break; } case SHELL_OPEN_ZIPFILE: { sqlite3_open(":memory:", &p->db); break; } @@ -3771,10 +3784,22 @@ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename); sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_free(zSql); } +#ifdef SQLITE_ENABLE_DESERIALIZE + else if( p->openMode==SHELL_OPEN_DESERIALIZE ){ + int nData = 0; + unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData); + int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, + SQLITE_DESERIALIZE_RESIZEABLE | + SQLITE_DESERIALIZE_FREEONCLOSE); + if( rc ){ + utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); + } + } +#endif } } /* ** Attempt to close the databaes connection. Report errors. @@ -6651,10 +6676,14 @@ #endif }else if( optionMatch(z, "append") ){ p->openMode = SHELL_OPEN_APPENDVFS; }else if( optionMatch(z, "readonly") ){ p->openMode = SHELL_OPEN_READONLY; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( optionMatch(z, "deserialize") ){ + p->openMode = SHELL_OPEN_DESERIALIZE; +#endif }else if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; goto meta_command_exit; } @@ -8619,10 +8648,14 @@ }else if( strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif }else if( strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( strcmp(z,"-deserialize")==0 ){ + data.openMode = SHELL_OPEN_DESERIALIZE; +#endif }else if( strcmp(z,"-readonly")==0 ){ data.openMode = SHELL_OPEN_READONLY; #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( strncmp(z, "-A",2)==0 ){ /* All remaining command-line arguments are passed to the ".archive" @@ -8714,10 +8747,14 @@ }else if( strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif }else if( strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; +#ifdef SQLITE_ENABLE_DESERIALIZE + }else if( strcmp(z,"-deserialize")==0 ){ + data.openMode = SHELL_OPEN_DESERIALIZE; +#endif }else if( strcmp(z,"-readonly")==0 ){ data.openMode = SHELL_OPEN_READONLY; }else if( strcmp(z,"-ascii")==0 ){ data.mode = MODE_Ascii; sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1385,11 +1385,11 @@ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ sqlite3_mutex *mutex; /* Connection mutex */ Db *aDb; /* All backends */ int nDb; /* Number of backends currently in use */ u32 mDbFlags; /* flags recording internal state */ - u32 flags; /* flags settable by pragmas. See below */ + u64 flags; /* flags settable by pragmas. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ i64 szMmap; /* Default mmap_size setting */ u32 nSchemaLock; /* Do not reset the schema when non-zero */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ @@ -1551,18 +1551,20 @@ #define SQLITE_Fts3Tokenizer 0x00400000 /* Enable fts3_tokenizer(2) */ #define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee*/ #define SQLITE_TriggerEQP 0x01000000 /* Show trigger EXPLAIN QUERY PLAN */ #define SQLITE_ResetDatabase 0x02000000 /* Reset the database */ #define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ +#define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ /* Flags used only if debugging */ +#define HI(X) ((u64)(X)<<32) #ifdef SQLITE_DEBUG -#define SQLITE_SqlTrace 0x08000000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x10000000 /* Debug listings of VDBE programs */ -#define SQLITE_VdbeTrace 0x20000000 /* True to trace VDBE execution */ -#define SQLITE_VdbeAddopTrace 0x40000000 /* Trace sqlite3VdbeAddOp() calls */ -#define SQLITE_VdbeEQP 0x80000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_SqlTrace HI(0x0001) /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing HI(0x0002) /* Debug listings of VDBE progs */ +#define SQLITE_VdbeTrace HI(0x0004) /* True to trace VDBE execution */ +#define SQLITE_VdbeAddopTrace HI(0x0008) /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_VdbeEQP HI(0x0010) /* Debug EXPLAIN QUERY PLAN */ #endif /* ** Allowed values for sqlite3.mDbFlags */ Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -1571,10 +1571,11 @@ pTab = pItem->pTab; assert( pTab!=0 ); pArgs = pItem->u1.pFuncArg; if( pArgs==0 ) return; for(j=k=0; jnExpr; j++){ + Expr *pRhs; while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} if( k>=pTab->nCol ){ sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", pTab->zName, j); return; @@ -1582,10 +1583,11 @@ pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0); if( pColRef==0 ) return; pColRef->iTable = pItem->iCursor; pColRef->iColumn = k++; pColRef->y.pTab = pTab; - pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, - sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0)); + pRhs = sqlite3PExpr(pParse, TK_UPLUS, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); } } Index: test/bestindex4.test ================================================================== --- test/bestindex4.test +++ test/bestindex4.test @@ -114,7 +114,60 @@ } } } } + +#------------------------------------------------------------------------- +# Test that a parameter passed to a table-valued function cannot be +# used to drive an index. i.e. that in the following: +# +# SELECT * FROM tbl, vtab(tbl.x); +# +# The implicit constraint "tbl.x = vtab.hidden" is not optimized using +# an index on tbl.x. +# +reset_db +register_tcl_module db +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c, d HIDDEN)" + } + + xBestIndex { + set clist [lindex $args 0] + if {[llength $clist]!=1} { error "unexpected constraint list" } + catch { array unset C } + array set C [lindex $clist 0] + if {$C(usable)} { + return [list omit 0 idxnum 555 rows 10 cost 100] + } + return [list cost 100000000] + } + + } + + return {} +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + CREATE TABLE t1 (x INT PRIMARY KEY); +} {} + +do_execsql_test 2.1 { + EXPLAIN QUERY PLAN SELECT * FROM t1, x1 WHERE x1.d=t1.x; +} { + 3 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 0:} + 7 0 0 {SEARCH TABLE t1 USING COVERING INDEX sqlite_autoindex_t1_1 (x=?)} +} + +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN SELECT * FROM t1, x1(t1.x) +} { + 3 0 0 {SCAN TABLE t1} + 5 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:} +} + finish_test ADDED test/dbfuzz2-seed1.db Index: test/dbfuzz2-seed1.db ================================================================== --- test/dbfuzz2-seed1.db +++ test/dbfuzz2-seed1.db cannot compute difference between binary files ADDED test/dbfuzz2.c Index: test/dbfuzz2.c ================================================================== --- test/dbfuzz2.c +++ test/dbfuzz2.c @@ -0,0 +1,138 @@ +/* +** 2018-10-26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This program is designed for fuzz-testing SQLite database files using +** the -fsanitize=fuzzer option of clang. +** +** The -fsanitize=fuzzer option causes a main() to be inserted automatically. +** That main() invokes LLVMFuzzerTestOneInput(D,S) to be invoked repeatedly. +** Each D is a fuzzed database file. The code in this file runs various +** SQL statements against that database, trying to provoke a failure. +** +** For best results the seed database files should have these tables: +** +** Table "t1" with columns "a" and "b" +** Tables "t2" and "t3 with the same number of compatible columns +** "t3" should have a column names "x" +** Table "t4" with a column "x" that is compatible with t3.x. +** +** Any of these tables can be virtual tables, for example FTS or RTree tables. +** +** To run this test: +** +** mkdir dir +** cp dbfuzz2-seed*.db dir +** clang-6.0 -I. -g -O1 -fsanitize=fuzzer \ +** -DTHREADSAFE=0 -DSQLITE_ENABLE_DESERIALIZE \ +** -DSQLITE_ENABLE_DBSTAT_VTAB dbfuzz2.c sqlite3.c -ldl +** ./a.out dir +*/ +#include +#include +#include +#include +#include +#include +#include +#include "sqlite3.h" + +/* +** This is the is the SQL that is run against the database. +*/ +static const char *azSql[] = { + "PRAGMA integrity_check;", + "SELECT * FROM sqlite_master;", + "SELECT sum(length(name)) FROM dbstat;", + "UPDATE t1 SET b=a, a=b WHERE a=1 ){ + printf("************** nByte=%d ***************\n", (int)nByte); + fflush(stdout); + } + rc = sqlite3_open(0, &db); + if( rc ) return 1; + a = sqlite3_malloc64(nByte+1); + if( a==0 ) return 1; + memcpy(a, aData, nByte); + sqlite3_deserialize(db, "main", a, nByte, nByte, + SQLITE_DESERIALIZE_RESIZEABLE | + SQLITE_DESERIALIZE_FREEONCLOSE); + for(i=0; i=1 ){ + printf("%s\n", azSql[i]); + fflush(stdout); + } + sqlite3_exec(db, azSql[i], 0, 0, 0); + } + rc = sqlite3_close(db); + if( rc!=SQLITE_OK ){ + fprintf(stdout, "sqlite3_close() returns %d\n", rc); + } + if( sqlite3_memory_used()!=0 ){ + int nAlloc = 0; + int nNotUsed = 0; + sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &nAlloc, &nNotUsed, 0); + fprintf(stderr,"Memory leak: %lld bytes in %d allocations\n", + sqlite3_memory_used(), nAlloc); + exit(1); + } + return 0; +} + +/* libFuzzer invokes this routine once when the executable starts, to +** process the command-line arguments. +*/ +int LLVMFuzzerInitialize(int *pArgc, char ***pArgv){ + int i, j; + int argc = *pArgc; + char **newArgv; + char **argv = *pArgv; + newArgv = malloc( sizeof(char*)*(argc+1) ); + if( newArgv==0 ) return 0; + newArgv[0] = argv[0]; + for(i=j=1; i