/ Check-in [4b370c74]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge fixes from trunk, especially rebustness against corrupt database files.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: 4b370c74ae0f25157c161592588eb3357513d139c5a0d2fee3e0f988add1d9fc
User & Date: drh 2018-10-31 01:26:24
Context
2018-11-05
13:53
Merge recent trunk enhancements. check-in: 62acf7a0 user: drh tags: apple-osx
2018-10-31
01:26
Merge fixes from trunk, especially rebustness against corrupt database files. check-in: 4b370c74 user: drh tags: apple-osx
01:12
Merge fuzz test cases computed by dbfuzz2. check-in: e0d30c18 user: drh tags: trunk
2018-10-25
16:58
Merge bug fixes from trunk. check-in: 1e973f65 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   589    589   #
   590    590   FUZZDATA = \
   591    591     $(TOP)/test/fuzzdata1.db \
   592    592     $(TOP)/test/fuzzdata2.db \
   593    593     $(TOP)/test/fuzzdata3.db \
   594    594     $(TOP)/test/fuzzdata4.db \
   595    595     $(TOP)/test/fuzzdata5.db \
   596         -  $(TOP)/test/fuzzdata6.db
          596  +  $(TOP)/test/fuzzdata6.db \
          597  +  $(TOP)/test/fuzzdata7.db
   597    598   
   598    599   # Standard options to testfixture
   599    600   #
   600    601   TESTOPTS = --verbose=file --output=test-out.txt
   601    602   
   602    603   # Extra compiler options for various shell tools
   603    604   #
................................................................................
   606    607   SHELL_OPT += -DSQLITE_ENABLE_RTREE
   607    608   SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
   608    609   SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   609    610   SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB
   610    611   SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
   611    612   SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
   612    613   SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
          614  +SHELL_OPT += -DSQLITE_ENABLE_DESERIALIZE
   613    615   SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS
   614    616   FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
   615    617   FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
   616    618   FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
   617    619   FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
   618    620   FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
   619    621   DBFUZZ_OPT = 
................................................................................
   673    675   
   674    676   sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
   675    677   	$(CC) $(CFLAGS) -I. -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)
   676    678   
   677    679   dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
   678    680   	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)
   679    681   
          682  +DBFUZZ2_OPTS = \
          683  +  -DSQLITE_THREADSAFE=0 \
          684  +  -DSQLITE_OMIT_LOAD_EXTENSION \
          685  +  -DSQLITE_ENABLE_DESERIALIZE \
          686  +  -DSQLITE_DEBUG \
          687  +  -DSQLITE_ENABLE_DBSTAT_VTAB \
          688  +  -DSQLITE_ENABLE_RTREE \
          689  +  -DSQLITE_ENABLE_FTS4 \
          690  +  -DSQLITE_EANBLE_FTS5
          691  +
          692  +dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
          693  +	clang-6.0 -I. -g -O0 -fsanitize=fuzzer,undefined -o dbfuzz2 \
          694  +		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
          695  +	mkdir -p dbfuzz2-dir
          696  +	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
          697  +
   680    698   mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
   681    699   	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
   682    700   		$(TLIBS) -rpath "$(libdir)"
   683    701   
   684    702   MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
   685    703   MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20
   686    704   mptest:	mptester$(TEXE)

Changes to Makefile.msc.

  1619   1619   #
  1620   1620   FUZZDATA = \
  1621   1621     $(TOP)\test\fuzzdata1.db \
  1622   1622     $(TOP)\test\fuzzdata2.db \
  1623   1623     $(TOP)\test\fuzzdata3.db \
  1624   1624     $(TOP)\test\fuzzdata4.db \
  1625   1625     $(TOP)\test\fuzzdata5.db \
  1626         -  $(TOP)\test\fuzzdata6.db
         1626  +  $(TOP)\test\fuzzdata6.db \
         1627  +  $(TOP)\test\fuzzdata7.db
  1627   1628   # <</mark>>
  1628   1629   
  1629   1630   # Additional compiler options for the shell.  These are only effective
  1630   1631   # when the shell is not being dynamically linked.
  1631   1632   #
  1632   1633   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
  1633   1634   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
  1634   1635   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
  1635   1636   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
         1637  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
  1636   1638   !ENDIF
  1637   1639   
  1638   1640   # <<mark>>
  1639   1641   # Extra compiler options for various test tools.
  1640   1642   #
  1641   1643   MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
  1642   1644   FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1

Changes to ext/session/session1.test.

   650    650     set CONFLICT_HANDLERS [list REPLACE REPLACE ABORT]
   651    651     execsql ROLLBACK
   652    652     execsql BEGIN
   653    653     catch { sqlite3changeset_apply_v2 -nosavepoint db $C xConflict } msg
   654    654     execsql { SELECT * FROM x1 }
   655    655   } {1 one 2 two 3 iii}
   656    656   execsql ROLLBACK
          657  +
          658  +do_test $tn.14.1 { sqlite3session_config strm_size -1 } 64
          659  +do_test $tn.14.2 { sqlite3session_config strm_size 65536 } 65536
          660  +do_test $tn.14.3 { sqlite3session_config strm_size 64 } 64
          661  +do_test $tn.14.4 { 
          662  +  list [catch {sqlite3session_config invalid 123} msg] $msg
          663  +} {1 SQLITE_MISUSE}
   657    664   
   658    665   }]
   659    666   }
   660    667   
   661    668   
   662    669   finish_test

Changes to ext/session/sqlite3session.c.

    21     21   # ifdef SQLITE_TEST
    22     22   #   define SESSIONS_STRM_CHUNK_SIZE 64
    23     23   # else
    24     24   #   define SESSIONS_STRM_CHUNK_SIZE 1024
    25     25   # endif
    26     26   #endif
    27     27   
           28  +static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE;
           29  +
    28     30   typedef struct SessionHook SessionHook;
    29     31   struct SessionHook {
    30     32     void *pCtx;
    31     33     int (*xOld)(void*,int,sqlite3_value**);
    32     34     int (*xNew)(void*,int,sqlite3_value**);
    33     35     int (*xCount)(void*);
    34     36     int (*xDepth)(void*);
................................................................................
   240    242   ** columns and columns that are modified by the UPDATE are set to "undefined".
   241    243   ** PRIMARY KEY fields contain the values identifying the table row to update,
   242    244   ** and fields associated with modified columns contain the new column values.
   243    245   **
   244    246   ** The records associated with INSERT changes are in the same format as for
   245    247   ** changesets. It is not possible for a record associated with an INSERT
   246    248   ** change to contain a field set to "undefined".
          249  +**
          250  +** REBASE BLOB FORMAT:
          251  +**
          252  +** A rebase blob may be output by sqlite3changeset_apply_v2() and its 
          253  +** streaming equivalent for use with the sqlite3_rebaser APIs to rebase
          254  +** existing changesets. A rebase blob contains one entry for each conflict
          255  +** resolved using either the OMIT or REPLACE strategies within the apply_v2()
          256  +** call.
          257  +**
          258  +** The format used for a rebase blob is very similar to that used for
          259  +** changesets. All entries related to a single table are grouped together.
          260  +**
          261  +** Each group of entries begins with a table header in changeset format:
          262  +**
          263  +**   1 byte: Constant 0x54 (capital 'T')
          264  +**   Varint: Number of columns in the table.
          265  +**   nCol bytes: 0x01 for PK columns, 0x00 otherwise.
          266  +**   N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated.
          267  +**
          268  +** Followed by one or more entries associated with the table.
          269  +**
          270  +**   1 byte: Either SQLITE_INSERT (0x12), DELETE (0x09).
          271  +**   1 byte: Flag. 0x01 for REPLACE, 0x00 for OMIT.
          272  +**   record: (in the record format defined above).
          273  +**
          274  +** In a rebase blob, the first field is set to SQLITE_INSERT if the change
          275  +** that caused the conflict was an INSERT or UPDATE, or to SQLITE_DELETE if
          276  +** it was a DELETE. The second field is set to 0x01 if the conflict 
          277  +** resolution strategy was REPLACE, or 0x00 if it was OMIT.
          278  +**
          279  +** If the change that caused the conflict was a DELETE, then the single
          280  +** record is a copy of the old.* record from the original changeset. If it
          281  +** was an INSERT, then the single record is a copy of the new.* record. If
          282  +** the conflicting change was an UPDATE, then the single record is a copy
          283  +** of the new.* record with the PK fields filled in based on the original
          284  +** old.* record.
   247    285   */
   248    286   
   249    287   /*
   250    288   ** For each row modified during a session, there exists a single instance of
   251    289   ** this structure stored in a SessionTable.aChange[] hash table.
   252    290   */
   253    291   struct SessionChange {
................................................................................
  2393   2431             }else if( p->op!=SQLITE_INSERT ){
  2394   2432               rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK);
  2395   2433             }
  2396   2434             if( rc==SQLITE_OK ){
  2397   2435               rc = sqlite3_reset(pSel);
  2398   2436             }
  2399   2437   
  2400         -          /* If the buffer is now larger than SESSIONS_STRM_CHUNK_SIZE, pass
         2438  +          /* If the buffer is now larger than sessions_strm_chunk_size, pass
  2401   2439             ** its contents to the xOutput() callback. */
  2402   2440             if( xOutput 
  2403   2441              && rc==SQLITE_OK 
  2404   2442              && buf.nBuf>nNoop 
  2405         -           && buf.nBuf>SESSIONS_STRM_CHUNK_SIZE 
         2443  +           && buf.nBuf>sessions_strm_chunk_size 
  2406   2444             ){
  2407   2445               rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf);
  2408   2446               nNoop = -1;
  2409   2447               buf.nBuf = 0;
  2410   2448             }
  2411   2449   
  2412   2450           }
................................................................................
  2610   2648   }
  2611   2649   
  2612   2650   /*
  2613   2651   ** If the SessionInput object passed as the only argument is a streaming
  2614   2652   ** object and the buffer is full, discard some data to free up space.
  2615   2653   */
  2616   2654   static void sessionDiscardData(SessionInput *pIn){
  2617         -  if( pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){
         2655  +  if( pIn->xInput && pIn->iNext>=sessions_strm_chunk_size ){
  2618   2656       int nMove = pIn->buf.nBuf - pIn->iNext;
  2619   2657       assert( nMove>=0 );
  2620   2658       if( nMove>0 ){
  2621   2659         memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove);
  2622   2660       }
  2623   2661       pIn->buf.nBuf -= pIn->iNext;
  2624   2662       pIn->iNext = 0;
................................................................................
  2633   2671   **
  2634   2672   ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
  2635   2673   */
  2636   2674   static int sessionInputBuffer(SessionInput *pIn, int nByte){
  2637   2675     int rc = SQLITE_OK;
  2638   2676     if( pIn->xInput ){
  2639   2677       while( !pIn->bEof && (pIn->iNext+nByte)>=pIn->nData && rc==SQLITE_OK ){
  2640         -      int nNew = SESSIONS_STRM_CHUNK_SIZE;
         2678  +      int nNew = sessions_strm_chunk_size;
  2641   2679   
  2642   2680         if( pIn->bNoDiscard==0 ) sessionDiscardData(pIn);
  2643   2681         if( SQLITE_OK==sessionBufferGrow(&pIn->buf, nNew, &rc) ){
  2644   2682           rc = pIn->xInput(pIn->pIn, &pIn->buf.aBuf[pIn->buf.nBuf], &nNew);
  2645   2683           if( nNew==0 ){
  2646   2684             pIn->bEof = 1;
  2647   2685           }else{
................................................................................
  3358   3396   
  3359   3397         default:
  3360   3398           rc = SQLITE_CORRUPT_BKPT;
  3361   3399           goto finished_invert;
  3362   3400       }
  3363   3401   
  3364   3402       assert( rc==SQLITE_OK );
  3365         -    if( xOutput && sOut.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){
         3403  +    if( xOutput && sOut.nBuf>=sessions_strm_chunk_size ){
  3366   3404         rc = xOutput(pOut, sOut.aBuf, sOut.nBuf);
  3367   3405         sOut.nBuf = 0;
  3368   3406         if( rc!=SQLITE_OK ) goto finished_invert;
  3369   3407       }
  3370   3408     }
  3371   3409   
  3372   3410     assert( rc==SQLITE_OK );
................................................................................
  3437   3475     int nCol;                       /* Size of azCol[] and abPK[] arrays */
  3438   3476     const char **azCol;             /* Array of column names */
  3439   3477     u8 *abPK;                       /* Boolean array - true if column is in PK */
  3440   3478     int bStat1;                     /* True if table is sqlite_stat1 */
  3441   3479     int bDeferConstraints;          /* True to defer constraints */
  3442   3480     SessionBuffer constraints;      /* Deferred constraints are stored here */
  3443   3481     SessionBuffer rebase;           /* Rebase information (if any) here */
  3444         -  int bRebaseStarted;             /* If table header is already in rebase */
         3482  +  u8 bRebaseStarted;              /* If table header is already in rebase */
         3483  +  u8 bRebase;                     /* True to collect rebase information */
  3445   3484   };
  3446   3485   
  3447   3486   /*
  3448   3487   ** Formulate a statement to DELETE a row from database db. Assuming a table
  3449   3488   ** structure like this:
  3450   3489   **
  3451   3490   **     CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c));
................................................................................
  3834   3873   */
  3835   3874   static int sessionRebaseAdd(
  3836   3875     SessionApplyCtx *p,             /* Apply context */
  3837   3876     int eType,                      /* Conflict resolution (OMIT or REPLACE) */
  3838   3877     sqlite3_changeset_iter *pIter   /* Iterator pointing at current change */
  3839   3878   ){
  3840   3879     int rc = SQLITE_OK;
  3841         -  int i;
  3842         -  int eOp = pIter->op;
  3843         -  if( p->bRebaseStarted==0 ){
  3844         -    /* Append a table-header to the rebase buffer */
  3845         -    const char *zTab = pIter->zTab;
  3846         -    sessionAppendByte(&p->rebase, 'T', &rc);
  3847         -    sessionAppendVarint(&p->rebase, p->nCol, &rc);
  3848         -    sessionAppendBlob(&p->rebase, p->abPK, p->nCol, &rc);
  3849         -    sessionAppendBlob(&p->rebase, (u8*)zTab, (int)strlen(zTab)+1, &rc);
  3850         -    p->bRebaseStarted = 1;
  3851         -  }
  3852         -
  3853         -  assert( eType==SQLITE_CHANGESET_REPLACE||eType==SQLITE_CHANGESET_OMIT );
  3854         -  assert( eOp==SQLITE_DELETE || eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE );
  3855         -
  3856         -  sessionAppendByte(&p->rebase, 
  3857         -      (eOp==SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT), &rc
  3858         -  );
  3859         -  sessionAppendByte(&p->rebase, (eType==SQLITE_CHANGESET_REPLACE), &rc);
  3860         -  for(i=0; i<p->nCol; i++){
  3861         -    sqlite3_value *pVal = 0;
  3862         -    if( eOp==SQLITE_DELETE || (eOp==SQLITE_UPDATE && p->abPK[i]) ){
  3863         -      sqlite3changeset_old(pIter, i, &pVal);
  3864         -    }else{
  3865         -      sqlite3changeset_new(pIter, i, &pVal);
  3866         -    }
  3867         -    sessionAppendValue(&p->rebase, pVal, &rc);
  3868         -  }
  3869         -
         3880  +  if( p->bRebase ){
         3881  +    int i;
         3882  +    int eOp = pIter->op;
         3883  +    if( p->bRebaseStarted==0 ){
         3884  +      /* Append a table-header to the rebase buffer */
         3885  +      const char *zTab = pIter->zTab;
         3886  +      sessionAppendByte(&p->rebase, 'T', &rc);
         3887  +      sessionAppendVarint(&p->rebase, p->nCol, &rc);
         3888  +      sessionAppendBlob(&p->rebase, p->abPK, p->nCol, &rc);
         3889  +      sessionAppendBlob(&p->rebase, (u8*)zTab, (int)strlen(zTab)+1, &rc);
         3890  +      p->bRebaseStarted = 1;
         3891  +    }
         3892  +
         3893  +    assert( eType==SQLITE_CHANGESET_REPLACE||eType==SQLITE_CHANGESET_OMIT );
         3894  +    assert( eOp==SQLITE_DELETE || eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE );
         3895  +
         3896  +    sessionAppendByte(&p->rebase, 
         3897  +        (eOp==SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT), &rc
         3898  +        );
         3899  +    sessionAppendByte(&p->rebase, (eType==SQLITE_CHANGESET_REPLACE), &rc);
         3900  +    for(i=0; i<p->nCol; i++){
         3901  +      sqlite3_value *pVal = 0;
         3902  +      if( eOp==SQLITE_DELETE || (eOp==SQLITE_UPDATE && p->abPK[i]) ){
         3903  +        sqlite3changeset_old(pIter, i, &pVal);
         3904  +      }else{
         3905  +        sqlite3changeset_new(pIter, i, &pVal);
         3906  +      }
         3907  +      sessionAppendValue(&p->rebase, pVal, &rc);
         3908  +    }
         3909  +  }
  3870   3910     return rc;
  3871   3911   }
  3872   3912   
  3873   3913   /*
  3874   3914   ** Invoke the conflict handler for the change that the changeset iterator
  3875   3915   ** currently points to.
  3876   3916   **
................................................................................
  4271   4311     SessionApplyCtx sApply;         /* changeset_apply() context object */
  4272   4312     int bPatchset;
  4273   4313   
  4274   4314     assert( xConflict!=0 );
  4275   4315   
  4276   4316     pIter->in.bNoDiscard = 1;
  4277   4317     memset(&sApply, 0, sizeof(sApply));
         4318  +  sApply.bRebase = (ppRebase && pnRebase);
  4278   4319     sqlite3_mutex_enter(sqlite3_db_mutex(db));
  4279   4320     if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){
  4280   4321       rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0);
  4281   4322     }
  4282   4323     if( rc==SQLITE_OK ){
  4283   4324       rc = sqlite3_exec(db, "PRAGMA defer_foreign_keys = 1", 0, 0, 0);
  4284   4325     }
................................................................................
  4421   4462         rc = sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0);
  4422   4463       }else{
  4423   4464         sqlite3_exec(db, "ROLLBACK TO changeset_apply", 0, 0, 0);
  4424   4465         sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0);
  4425   4466       }
  4426   4467     }
  4427   4468   
  4428         -  if( rc==SQLITE_OK && bPatchset==0 && ppRebase && pnRebase ){
         4469  +  assert( sApply.bRebase || sApply.rebase.nBuf==0 );
         4470  +  if( rc==SQLITE_OK && bPatchset==0 && sApply.bRebase ){
  4429   4471       *ppRebase = (void*)sApply.rebase.aBuf;
  4430   4472       *pnRebase = sApply.rebase.nBuf;
  4431   4473       sApply.rebase.aBuf = 0;
  4432   4474     }
  4433   4475     sqlite3_finalize(sApply.pInsert);
  4434   4476     sqlite3_finalize(sApply.pDelete);
  4435   4477     sqlite3_finalize(sApply.pUpdate);
................................................................................
  4891   4933       sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc);
  4892   4934       for(i=0; i<pTab->nChange; i++){
  4893   4935         SessionChange *p;
  4894   4936         for(p=pTab->apChange[i]; p; p=p->pNext){
  4895   4937           sessionAppendByte(&buf, p->op, &rc);
  4896   4938           sessionAppendByte(&buf, p->bIndirect, &rc);
  4897   4939           sessionAppendBlob(&buf, p->aRecord, p->nRecord, &rc);
  4898         -        if( rc==SQLITE_OK && xOutput && buf.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){
         4940  +        if( rc==SQLITE_OK && xOutput && buf.nBuf>=sessions_strm_chunk_size ){
  4899   4941             rc = xOutput(pOut, buf.aBuf, buf.nBuf);
  4900   4942             buf.nBuf = 0;
  4901   4943           }
  4902   4944         }
  4903   4945       }
  4904   4946     }
  4905   4947   
................................................................................
  5287   5329       }
  5288   5330   
  5289   5331       if( bDone==0 ){
  5290   5332         sessionAppendByte(&sOut, pIter->op, &rc);
  5291   5333         sessionAppendByte(&sOut, pIter->bIndirect, &rc);
  5292   5334         sessionAppendBlob(&sOut, aRec, nRec, &rc);
  5293   5335       }
  5294         -    if( rc==SQLITE_OK && xOutput && sOut.nBuf>SESSIONS_STRM_CHUNK_SIZE ){
         5336  +    if( rc==SQLITE_OK && xOutput && sOut.nBuf>sessions_strm_chunk_size ){
  5295   5337         rc = xOutput(pOut, sOut.aBuf, sOut.nBuf);
  5296   5338         sOut.nBuf = 0;
  5297   5339       }
  5298   5340       if( rc ) break;
  5299   5341     }
  5300   5342   
  5301   5343     if( rc!=SQLITE_OK ){
................................................................................
  5397   5439   */
  5398   5440   void sqlite3rebaser_delete(sqlite3_rebaser *p){
  5399   5441     if( p ){
  5400   5442       sessionDeleteTable(p->grp.pList);
  5401   5443       sqlite3_free(p);
  5402   5444     }
  5403   5445   }
         5446  +
         5447  +/* 
         5448  +** Global configuration
         5449  +*/
         5450  +int sqlite3session_config(int op, void *pArg){
         5451  +  int rc = SQLITE_OK;
         5452  +  switch( op ){
         5453  +    case SQLITE_SESSION_CONFIG_STRMSIZE: {
         5454  +      int *pInt = (int*)pArg;
         5455  +      if( *pInt>0 ){
         5456  +        sessions_strm_chunk_size = *pInt;
         5457  +      }
         5458  +      *pInt = sessions_strm_chunk_size;
         5459  +      break;
         5460  +    }
         5461  +    default:
         5462  +      rc = SQLITE_MISUSE;
         5463  +      break;
         5464  +  }
         5465  +  return rc;
         5466  +}
  5404   5467   
  5405   5468   #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */

Changes to ext/session/sqlite3session.h.

  1606   1606     sqlite3_rebaser *pRebaser,
  1607   1607     int (*xInput)(void *pIn, void *pData, int *pnData),
  1608   1608     void *pIn,
  1609   1609     int (*xOutput)(void *pOut, const void *pData, int nData),
  1610   1610     void *pOut
  1611   1611   );
  1612   1612   
         1613  +/*
         1614  +** CAPI3REF: Configure global parameters
         1615  +**
         1616  +** The sqlite3session_config() interface is used to make global configuration
         1617  +** changes to the sessions module in order to tune it to the specific needs 
         1618  +** of the application.
         1619  +**
         1620  +** The sqlite3session_config() interface is not threadsafe. If it is invoked
         1621  +** while any other thread is inside any other sessions method then the
         1622  +** results are undefined. Furthermore, if it is invoked after any sessions
         1623  +** related objects have been created, the results are also undefined. 
         1624  +**
         1625  +** The first argument to the sqlite3session_config() function must be one
         1626  +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The 
         1627  +** interpretation of the (void*) value passed as the second parameter and
         1628  +** the effect of calling this function depends on the value of the first
         1629  +** parameter.
         1630  +**
         1631  +** <dl>
         1632  +** <dt>SQLITE_SESSION_CONFIG_STRMSIZE<dd>
         1633  +**    By default, the sessions module streaming interfaces attempt to input
         1634  +**    and output data in approximately 1 KiB chunks. This operand may be used
         1635  +**    to set and query the value of this configuration setting. The pointer
         1636  +**    passed as the second argument must point to a value of type (int).
         1637  +**    If this value is greater than 0, it is used as the new streaming data
         1638  +**    chunk size for both input and output. Before returning, the (int) value
         1639  +**    pointed to by pArg is set to the final value of the streaming interface
         1640  +**    chunk size.
         1641  +** </dl>
         1642  +**
         1643  +** This function returns SQLITE_OK if successful, or an SQLite error code
         1644  +** otherwise.
         1645  +*/
         1646  +int sqlite3session_config(int op, void *pArg);
         1647  +
         1648  +/*
         1649  +** CAPI3REF: Values for sqlite3session_config().
         1650  +*/
         1651  +#define SQLITE_SESSION_CONFIG_STRMSIZE 1
  1613   1652   
  1614   1653   /*
  1615   1654   ** Make sure we can call this stuff from C++.
  1616   1655   */
  1617   1656   #ifdef __cplusplus
  1618   1657   }
  1619   1658   #endif
  1620   1659   
  1621   1660   #endif  /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */

Changes to ext/session/test_session.c.

  1234   1234   
  1235   1235     Tcl_CreateObjCommand(interp, Tcl_GetString(objv[1]), test_rebaser_cmd,
  1236   1236         (ClientData)pNew, test_rebaser_del
  1237   1237     );
  1238   1238     Tcl_SetObjResult(interp, objv[1]);
  1239   1239     return TCL_OK;
  1240   1240   }
         1241  +
         1242  +/*
         1243  +** tclcmd: sqlite3rebaser_configure OP VALUE
         1244  +*/
         1245  +static int SQLITE_TCLAPI test_sqlite3session_config(
         1246  +  void * clientData,
         1247  +  Tcl_Interp *interp,
         1248  +  int objc,
         1249  +  Tcl_Obj *CONST objv[]
         1250  +){
         1251  +  struct ConfigOpt {
         1252  +    const char *zSub;
         1253  +    int op;
         1254  +  } aSub[] = {
         1255  +    { "strm_size",    SQLITE_SESSION_CONFIG_STRMSIZE },
         1256  +    { "invalid",      0 },
         1257  +    { 0 }
         1258  +  };
         1259  +  int rc;
         1260  +  int iSub;
         1261  +  int iVal;
         1262  +
         1263  +  if( objc!=3 ){
         1264  +    Tcl_WrongNumArgs(interp, 1, objv, "OP VALUE");
         1265  +    return SQLITE_ERROR;
         1266  +  }
         1267  +  rc = Tcl_GetIndexFromObjStruct(interp, 
         1268  +      objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
         1269  +  );
         1270  +  if( rc!=TCL_OK ) return rc;
         1271  +  if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR;
         1272  +
         1273  +  rc = sqlite3session_config(aSub[iSub].op, (void*)&iVal);
         1274  +  if( rc!=SQLITE_OK ){
         1275  +    return test_session_error(interp, rc, 0);
         1276  +  }
         1277  +  Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
         1278  +  return TCL_OK;
         1279  +}
  1241   1280   
  1242   1281   int TestSession_Init(Tcl_Interp *interp){
  1243   1282     struct Cmd {
  1244   1283       const char *zCmd;
  1245   1284       Tcl_ObjCmdProc *xProc;
  1246   1285     } aCmd[] = {
  1247   1286       { "sqlite3session", test_sqlite3session },
................................................................................
  1250   1289       { "sqlite3changeset_concat", test_sqlite3changeset_concat },
  1251   1290       { "sqlite3changeset_apply", test_sqlite3changeset_apply },
  1252   1291       { "sqlite3changeset_apply_v2", test_sqlite3changeset_apply_v2 },
  1253   1292       { "sqlite3changeset_apply_replace_all", 
  1254   1293         test_sqlite3changeset_apply_replace_all },
  1255   1294       { "sql_exec_changeset", test_sql_exec_changeset },
  1256   1295       { "sqlite3rebaser_create", test_sqlite3rebaser_create },
         1296  +    { "sqlite3session_config", test_sqlite3session_config },
  1257   1297     };
  1258   1298     int i;
  1259   1299   
  1260   1300     for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
  1261   1301       struct Cmd *p = &aCmd[i];
  1262   1302       Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
  1263   1303     }
  1264   1304   
  1265   1305     return TCL_OK;
  1266   1306   }
  1267   1307   
  1268   1308   #endif /* SQLITE_TEST && SQLITE_SESSION && SQLITE_PREUPDATE_HOOK */

Changes to main.mk.

   508    508   #
   509    509   FUZZDATA = \
   510    510     $(TOP)/test/fuzzdata1.db \
   511    511     $(TOP)/test/fuzzdata2.db \
   512    512     $(TOP)/test/fuzzdata3.db \
   513    513     $(TOP)/test/fuzzdata4.db \
   514    514     $(TOP)/test/fuzzdata5.db \
   515         -  $(TOP)/test/fuzzdata6.db
          515  +  $(TOP)/test/fuzzdata6.db \
          516  +  $(TOP)/test/fuzzdata7.db
   516    517   
   517    518   # Standard options to testfixture
   518    519   #
   519    520   TESTOPTS = --verbose=file --output=test-out.txt
   520    521   
   521    522   # Extra compiler options for various shell tools
   522    523   #

Changes to src/btree.c.

  3588   3588     Pager *pPager = pBt->pPager;
  3589   3589     int rc;
  3590   3590   
  3591   3591     assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || 
  3592   3592         eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE );
  3593   3593     assert( sqlite3_mutex_held(pBt->mutex) );
  3594   3594     assert( pDbPage->pBt==pBt );
         3595  +  if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT;
  3595   3596   
  3596   3597     /* Move page iDbPage from its current location to page number iFreePage */
  3597   3598     TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", 
  3598   3599         iDbPage, iFreePage, iPtrPage, eType));
  3599   3600     rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit);
  3600   3601     if( rc!=SQLITE_OK ){
  3601   3602       return rc;

Changes to src/build.c.

  2418   2418   ** Also write code to modify the sqlite_master table and internal schema
  2419   2419   ** if a root-page of another table is moved by the btree-layer whilst
  2420   2420   ** erasing iTable (this can happen with an auto-vacuum database).
  2421   2421   */ 
  2422   2422   static void destroyRootPage(Parse *pParse, int iTable, int iDb){
  2423   2423     Vdbe *v = sqlite3GetVdbe(pParse);
  2424   2424     int r1 = sqlite3GetTempReg(pParse);
  2425         -  assert( iTable>1 );
         2425  +  if( iTable<2 ) sqlite3ErrorMsg(pParse, "corrupt schema");
  2426   2426     sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb);
  2427   2427     sqlite3MayAbort(pParse);
  2428   2428   #ifndef SQLITE_OMIT_AUTOVACUUM
  2429   2429     /* OP_Destroy stores an in integer r1. If this integer
  2430   2430     ** is non-zero, then it is the root page number of a table moved to
  2431   2431     ** location iTable. The following code modifies the sqlite_master table to
  2432   2432     ** reflect this.

Changes to src/dbstat.c.

   250    250       pCsr->iDb = pTab->iDb;
   251    251     }
   252    252   
   253    253     *ppCursor = (sqlite3_vtab_cursor *)pCsr;
   254    254     return SQLITE_OK;
   255    255   }
   256    256   
   257         -static void statClearPage(StatPage *p){
          257  +static void statClearCells(StatPage *p){
   258    258     int i;
   259    259     if( p->aCell ){
   260    260       for(i=0; i<p->nCell; i++){
   261    261         sqlite3_free(p->aCell[i].aOvfl);
   262    262       }
   263    263       sqlite3_free(p->aCell);
   264    264     }
          265  +  p->nCell = 0;
          266  +  p->aCell = 0;
          267  +}
          268  +
          269  +static void statClearPage(StatPage *p){
          270  +  statClearCells(p);
   265    271     sqlite3PagerUnref(p->pPg);
   266    272     sqlite3_free(p->zPath);
   267    273     memset(p, 0, sizeof(StatPage));
   268    274   }
   269    275   
   270    276   static void statResetCsr(StatCursor *pCsr){
   271    277     int i;
................................................................................
   320    326     int isLeaf;
   321    327     int szPage;
   322    328   
   323    329     u8 *aData = sqlite3PagerGetData(p->pPg);
   324    330     u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0];
   325    331   
   326    332     p->flags = aHdr[0];
          333  +  if( p->flags==0x0A || p->flags==0x0D ){
          334  +    isLeaf = 1;
          335  +    nHdr = 8;
          336  +  }else if( p->flags==0x05 || p->flags==0x02 ){
          337  +    isLeaf = 0;
          338  +    nHdr = 12;
          339  +  }else{
          340  +    goto statPageIsCorrupt;
          341  +  }
          342  +  if( p->iPgno==1 ) nHdr += 100;
   327    343     p->nCell = get2byte(&aHdr[3]);
   328    344     p->nMxPayload = 0;
   329         -
   330         -  isLeaf = (p->flags==0x0A || p->flags==0x0D);
   331         -  nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100;
          345  +  szPage = sqlite3BtreeGetPageSize(pBt);
   332    346   
   333    347     nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell;
   334    348     nUnused += (int)aHdr[7];
   335    349     iOff = get2byte(&aHdr[1]);
   336    350     while( iOff ){
          351  +    int iNext;
          352  +    if( iOff>=szPage ) goto statPageIsCorrupt;
   337    353       nUnused += get2byte(&aData[iOff+2]);
   338         -    iOff = get2byte(&aData[iOff]);
          354  +    iNext = get2byte(&aData[iOff]);
          355  +    if( iNext<iOff+4 && iNext>0 ) goto statPageIsCorrupt;
          356  +    iOff = iNext;
   339    357     }
   340    358     p->nUnused = nUnused;
   341    359     p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]);
   342         -  szPage = sqlite3BtreeGetPageSize(pBt);
   343    360   
   344    361     if( p->nCell ){
   345    362       int i;                        /* Used to iterate through cells */
   346    363       int nUsable;                  /* Usable bytes per page */
   347    364   
   348    365       sqlite3BtreeEnter(pBt);
   349    366       nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt);
................................................................................
   352    369       if( p->aCell==0 ) return SQLITE_NOMEM_BKPT;
   353    370       memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell));
   354    371   
   355    372       for(i=0; i<p->nCell; i++){
   356    373         StatCell *pCell = &p->aCell[i];
   357    374   
   358    375         iOff = get2byte(&aData[nHdr+i*2]);
          376  +      if( iOff<nHdr || iOff>=szPage ) goto statPageIsCorrupt;
   359    377         if( !isLeaf ){
   360    378           pCell->iChildPg = sqlite3Get4byte(&aData[iOff]);
   361    379           iOff += 4;
   362    380         }
   363    381         if( p->flags==0x05 ){
   364    382           /* A table interior node. nPayload==0. */
   365    383         }else{
................................................................................
   368    386           iOff += getVarint32(&aData[iOff], nPayload);
   369    387           if( p->flags==0x0D ){
   370    388             u64 dummy;
   371    389             iOff += sqlite3GetVarint(&aData[iOff], &dummy);
   372    390           }
   373    391           if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload;
   374    392           getLocalPayload(nUsable, p->flags, nPayload, &nLocal);
          393  +        if( nLocal<0 ) goto statPageIsCorrupt;
   375    394           pCell->nLocal = nLocal;
   376         -        assert( nLocal>=0 );
   377    395           assert( nPayload>=(u32)nLocal );
   378    396           assert( nLocal<=(nUsable-35) );
   379    397           if( nPayload>(u32)nLocal ){
   380    398             int j;
   381    399             int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4);
   382    400             pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4);
   383    401             pCell->nOvfl = nOvfl;
................................................................................
   398    416             }
   399    417           }
   400    418         }
   401    419       }
   402    420     }
   403    421   
   404    422     return SQLITE_OK;
          423  +
          424  +statPageIsCorrupt:
          425  +  p->flags = 0;
          426  +  statClearCells(p);
          427  +  return SQLITE_OK;
   405    428   }
   406    429   
   407    430   /*
   408    431   ** Populate the pCsr->iOffset and pCsr->szPage member variables. Based on
   409    432   ** the current value of pCsr->iPageno.
   410    433   */
   411    434   static void statSizeAndOffset(StatCursor *pCsr){

Changes to src/pragma.c.

  1033   1033   
  1034   1034   #ifndef SQLITE_OMIT_FLAG_PRAGMAS
  1035   1035     case PragTyp_FLAG: {
  1036   1036       if( zRight==0 ){
  1037   1037         setPragmaResultColumnNames(v, pPragma);
  1038   1038         returnSingleInt(v, (db->flags & pPragma->iArg)!=0 );
  1039   1039       }else{
  1040         -      int mask = pPragma->iArg;    /* Mask of bits to set or clear. */
         1040  +      u64 mask = pPragma->iArg;    /* Mask of bits to set or clear. */
  1041   1041         if( db->autoCommit==0 ){
  1042   1042           /* Foreign key support may not be enabled or disabled while not
  1043   1043           ** in auto-commit mode.  */
  1044   1044           mask &= ~(SQLITE_ForeignKeys);
  1045   1045         }
  1046   1046   #if SQLITE_USER_AUTHENTICATION
  1047   1047         if( db->auth.authLevel==UAUTH_User ){

Changes to src/pragma.h.

   123    123   /* Definitions of all built-in pragmas */
   124    124   typedef struct PragmaName {
   125    125     const char *const zName; /* Name of pragma */
   126    126     u8 ePragTyp;             /* PragTyp_XXX value */
   127    127     u8 mPragFlg;             /* Zero or more PragFlg_XXX values */
   128    128     u8 iPragCName;           /* Start of column names in pragCName[] */
   129    129     u8 nPragCName;           /* Num of col names. 0 means use pragma name */
   130         -  u32 iArg;                /* Extra argument */
          130  +  u64 iArg;                /* Extra argument */
   131    131   } PragmaName;
   132    132   static const PragmaName aPragmaName[] = {
   133    133   #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD)
   134    134    {/* zName:     */ "activate_extensions",
   135    135     /* ePragTyp:  */ PragTyp_ACTIVATE_EXTENSIONS,
   136    136     /* ePragFlg:  */ 0,
   137    137     /* ColNames:  */ 0, 0,
................................................................................
   659    659     /* iArg:      */ 0 },
   660    660   #endif
   661    661   #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   662    662    {/* zName:     */ "writable_schema",
   663    663     /* ePragTyp:  */ PragTyp_FLAG,
   664    664     /* ePragFlg:  */ PragFlg_Result0|PragFlg_NoColumns1,
   665    665     /* ColNames:  */ 0, 0,
   666         -  /* iArg:      */ SQLITE_WriteSchema },
          666  +  /* iArg:      */ SQLITE_WriteSchema|SQLITE_NoSchemaError },
   667    667   #endif
   668    668   };
   669    669   /* Number of pragmas: 62 on by default, 81 total. */

Changes to src/prepare.c.

   326    326       }
   327    327   #endif
   328    328     }
   329    329     if( db->mallocFailed ){
   330    330       rc = SQLITE_NOMEM_BKPT;
   331    331       sqlite3ResetAllSchemasOfConnection(db);
   332    332     }
   333         -  if( rc==SQLITE_OK || (db->flags&SQLITE_WriteSchema)){
   334         -    /* Black magic: If the SQLITE_WriteSchema flag is set, then consider
          333  +  if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){
          334  +    /* Black magic: If the SQLITE_NoSchemaError flag is set, then consider
   335    335       ** the schema loaded, even if errors occurred. In this situation the 
   336    336       ** current sqlite3_prepare() operation will fail, but the following one
   337    337       ** will attempt to compile the supplied statement against whatever subset
   338    338       ** of the schema was loaded before the error occurred. The primary
   339    339       ** purpose of this is to allow access to the sqlite_master table
   340    340       ** even when its contents have been corrupted.
   341    341       */

Changes to src/shell.c.in.

  1055   1055   #define AUTOEQP_off      0           /* Automatic EXPLAIN QUERY PLAN is off */
  1056   1056   #define AUTOEQP_on       1           /* Automatic EQP is on */
  1057   1057   #define AUTOEQP_trigger  2           /* On and also show plans for triggers */
  1058   1058   #define AUTOEQP_full     3           /* Show full EXPLAIN */
  1059   1059   
  1060   1060   /* Allowed values for ShellState.openMode
  1061   1061   */
  1062         -#define SHELL_OPEN_UNSPEC     0      /* No open-mode specified */
  1063         -#define SHELL_OPEN_NORMAL     1      /* Normal database file */
  1064         -#define SHELL_OPEN_APPENDVFS  2      /* Use appendvfs */
  1065         -#define SHELL_OPEN_ZIPFILE    3      /* Use the zipfile virtual table */
  1066         -#define SHELL_OPEN_READONLY   4      /* Open a normal database read-only */
         1062  +#define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
         1063  +#define SHELL_OPEN_NORMAL      1      /* Normal database file */
         1064  +#define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
         1065  +#define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
         1066  +#define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
         1067  +#define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
  1067   1068   
  1068   1069   /*
  1069   1070   ** These are the allowed shellFlgs values
  1070   1071   */
  1071   1072   #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
  1072   1073   #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
  1073   1074   #define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
................................................................................
  3428   3429     ".once (-e|-x|FILE)       Output for the next SQL command only to FILE",
  3429   3430     "     If FILE begins with '|' then open as a pipe",
  3430   3431     "     Other options:",
  3431   3432     "       -e    Invoke system text editor",
  3432   3433     "       -x    Open in a spreadsheet",
  3433   3434     ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  3434   3435     "     Options:",
  3435         -  "        --append      Use appendvfs to append database to the end of FILE",
  3436         -  "        --new         Initialize FILE to an empty database",
  3437         -  "        --readonly    Open FILE readonly",
  3438         -  "        --zip         FILE is a ZIP archive",
         3436  +  "        --append        Use appendvfs to append database to the end of FILE",
         3437  +#ifdef SQLITE_ENABLE_DESERIALIZE
         3438  +  "        --deserialize   Load into memory useing sqlite3_deserialize()",
         3439  +#endif
         3440  +  "        --new           Initialize FILE to an empty database",
         3441  +  "        --readonly      Open FILE readonly",
         3442  +  "        --zip           FILE is a ZIP archive",
  3439   3443     ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  3440   3444     "     If FILE begins with '|' then open it as a pipe.",
  3441   3445     ".print STRING...         Print literal STRING",
  3442   3446     ".prompt MAIN CONTINUE    Replace the standard prompts",
  3443   3447     ".quit                    Exit this program",
  3444   3448     ".read FILE               Read input from FILE",
  3445   3449     ".restore ?DB? FILE       Restore content of DB (default \"main\") from FILE",
................................................................................
  3669   3673     if( f==0 ){
  3670   3674       if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){
  3671   3675          return SHELL_OPEN_ZIPFILE;
  3672   3676       }else{
  3673   3677          return SHELL_OPEN_NORMAL;
  3674   3678       }
  3675   3679     }
         3680  +  n = fread(zBuf, 16, 1, f);
         3681  +  if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){
         3682  +    fclose(f);
         3683  +    return SHELL_OPEN_NORMAL;
         3684  +  }
  3676   3685     fseek(f, -25, SEEK_END);
  3677   3686     n = fread(zBuf, 25, 1, f);
  3678   3687     if( n==1 && memcmp(zBuf, "Start-Of-SQLite3-", 17)==0 ){
  3679   3688       rc = SHELL_OPEN_APPENDVFS;
  3680   3689     }else{
  3681   3690       fseek(f, -22, SEEK_END);
  3682   3691       n = fread(zBuf, 22, 1, f);
................................................................................
  3719   3728         }
  3720   3729       }
  3721   3730       switch( p->openMode ){
  3722   3731         case SHELL_OPEN_APPENDVFS: {
  3723   3732           sqlite3_open_v2(p->zDbFilename, &p->db, 
  3724   3733              SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
  3725   3734           break;
         3735  +      }
         3736  +      case SHELL_OPEN_DESERIALIZE: {
         3737  +        sqlite3_open(0, &p->db);
         3738  +        break;
  3726   3739         }
  3727   3740         case SHELL_OPEN_ZIPFILE: {
  3728   3741           sqlite3_open(":memory:", &p->db);
  3729   3742           break;
  3730   3743         }
  3731   3744         case SHELL_OPEN_READONLY: {
  3732   3745           sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0);
................................................................................
  3769   3782   #endif
  3770   3783       if( p->openMode==SHELL_OPEN_ZIPFILE ){
  3771   3784         char *zSql = sqlite3_mprintf(
  3772   3785            "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
  3773   3786         sqlite3_exec(p->db, zSql, 0, 0, 0);
  3774   3787         sqlite3_free(zSql);
  3775   3788       }
         3789  +#ifdef SQLITE_ENABLE_DESERIALIZE
         3790  +    else if( p->openMode==SHELL_OPEN_DESERIALIZE ){
         3791  +      int nData = 0;
         3792  +      unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData);
         3793  +      int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
         3794  +                   SQLITE_DESERIALIZE_RESIZEABLE |
         3795  +                   SQLITE_DESERIALIZE_FREEONCLOSE);
         3796  +      if( rc ){
         3797  +        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
         3798  +      }
         3799  +    }
         3800  +#endif
  3776   3801     }
  3777   3802   }
  3778   3803   
  3779   3804   /*
  3780   3805   ** Attempt to close the databaes connection.  Report errors.
  3781   3806   */
  3782   3807   void close_db(sqlite3 *db){
................................................................................
  6649   6674         }else if( optionMatch(z, "zip") ){
  6650   6675           p->openMode = SHELL_OPEN_ZIPFILE;
  6651   6676   #endif
  6652   6677         }else if( optionMatch(z, "append") ){
  6653   6678           p->openMode = SHELL_OPEN_APPENDVFS;
  6654   6679         }else if( optionMatch(z, "readonly") ){
  6655   6680           p->openMode = SHELL_OPEN_READONLY;
         6681  +#ifdef SQLITE_ENABLE_DESERIALIZE
         6682  +      }else if( optionMatch(z, "deserialize") ){
         6683  +        p->openMode = SHELL_OPEN_DESERIALIZE;
         6684  +#endif
  6656   6685         }else if( z[0]=='-' ){
  6657   6686           utf8_printf(stderr, "unknown option: %s\n", z);
  6658   6687           rc = 1;
  6659   6688           goto meta_command_exit;
  6660   6689         }
  6661   6690       }
  6662   6691       /* If a filename is specified, try to open it first */
................................................................................
  8617   8646         zVfs = cmdline_option_value(argc, argv, ++i);
  8618   8647   #ifdef SQLITE_HAVE_ZLIB
  8619   8648       }else if( strcmp(z,"-zip")==0 ){
  8620   8649         data.openMode = SHELL_OPEN_ZIPFILE;
  8621   8650   #endif
  8622   8651       }else if( strcmp(z,"-append")==0 ){
  8623   8652         data.openMode = SHELL_OPEN_APPENDVFS;
         8653  +#ifdef SQLITE_ENABLE_DESERIALIZE
         8654  +    }else if( strcmp(z,"-deserialize")==0 ){
         8655  +      data.openMode = SHELL_OPEN_DESERIALIZE;
         8656  +#endif
  8624   8657       }else if( strcmp(z,"-readonly")==0 ){
  8625   8658         data.openMode = SHELL_OPEN_READONLY;
  8626   8659   #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
  8627   8660       }else if( strncmp(z, "-A",2)==0 ){
  8628   8661         /* All remaining command-line arguments are passed to the ".archive"
  8629   8662         ** command, so ignore them */
  8630   8663         break;
................................................................................
  8712   8745         memcpy(data.colSeparator,",",2);
  8713   8746   #ifdef SQLITE_HAVE_ZLIB
  8714   8747       }else if( strcmp(z,"-zip")==0 ){
  8715   8748         data.openMode = SHELL_OPEN_ZIPFILE;
  8716   8749   #endif
  8717   8750       }else if( strcmp(z,"-append")==0 ){
  8718   8751         data.openMode = SHELL_OPEN_APPENDVFS;
         8752  +#ifdef SQLITE_ENABLE_DESERIALIZE
         8753  +    }else if( strcmp(z,"-deserialize")==0 ){
         8754  +      data.openMode = SHELL_OPEN_DESERIALIZE;
         8755  +#endif
  8719   8756       }else if( strcmp(z,"-readonly")==0 ){
  8720   8757         data.openMode = SHELL_OPEN_READONLY;
  8721   8758       }else if( strcmp(z,"-ascii")==0 ){
  8722   8759         data.mode = MODE_Ascii;
  8723   8760         sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
  8724   8761                          SEP_Unit);
  8725   8762         sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,

Changes to src/sqliteInt.h.

  1383   1383     sqlite3_vfs *pVfs;            /* OS Interface */
  1384   1384     struct Vdbe *pVdbe;           /* List of active virtual machines */
  1385   1385     CollSeq *pDfltColl;           /* The default collating sequence (BINARY) */
  1386   1386     sqlite3_mutex *mutex;         /* Connection mutex */
  1387   1387     Db *aDb;                      /* All backends */
  1388   1388     int nDb;                      /* Number of backends currently in use */
  1389   1389     u32 mDbFlags;                 /* flags recording internal state */
  1390         -  u32 flags;                    /* flags settable by pragmas. See below */
         1390  +  u64 flags;                    /* flags settable by pragmas. See below */
  1391   1391     i64 lastRowid;                /* ROWID of most recent insert (see above) */
  1392   1392     i64 szMmap;                   /* Default mmap_size setting */
  1393   1393     u32 nSchemaLock;              /* Do not reset the schema when non-zero */
  1394   1394     unsigned int openFlags;       /* Flags passed to sqlite3_vfs.xOpen() */
  1395   1395     int errCode;                  /* Most recent error code (SQLITE_*) */
  1396   1396     int errMask;                  /* & result codes with this before returning */
  1397   1397     int iSysErrno;                /* Errno value from last system error */
................................................................................
  1549   1549   #define SQLITE_QueryOnly      0x00100000  /* Disable database changes */
  1550   1550   #define SQLITE_CellSizeCk     0x00200000  /* Check btree cell sizes on load */
  1551   1551   #define SQLITE_Fts3Tokenizer  0x00400000  /* Enable fts3_tokenizer(2) */
  1552   1552   #define SQLITE_EnableQPSG     0x00800000  /* Query Planner Stability Guarantee*/
  1553   1553   #define SQLITE_TriggerEQP     0x01000000  /* Show trigger EXPLAIN QUERY PLAN */
  1554   1554   #define SQLITE_ResetDatabase  0x02000000  /* Reset the database */
  1555   1555   #define SQLITE_LegacyAlter    0x04000000  /* Legacy ALTER TABLE behaviour */
         1556  +#define SQLITE_NoSchemaError  0x08000000  /* Do not report schema parse errors*/
  1556   1557   
  1557   1558   /* Flags used only if debugging */
         1559  +#define HI(X)  ((u64)(X)<<32)
  1558   1560   #ifdef SQLITE_DEBUG
  1559         -#define SQLITE_SqlTrace       0x08000000  /* Debug print SQL as it executes */
  1560         -#define SQLITE_VdbeListing    0x10000000  /* Debug listings of VDBE programs */
  1561         -#define SQLITE_VdbeTrace      0x20000000  /* True to trace VDBE execution */
  1562         -#define SQLITE_VdbeAddopTrace 0x40000000  /* Trace sqlite3VdbeAddOp() calls */
  1563         -#define SQLITE_VdbeEQP        0x80000000  /* Debug EXPLAIN QUERY PLAN */
         1561  +#define SQLITE_SqlTrace       HI(0x0001)  /* Debug print SQL as it executes */
         1562  +#define SQLITE_VdbeListing    HI(0x0002)  /* Debug listings of VDBE progs */
         1563  +#define SQLITE_VdbeTrace      HI(0x0004)  /* True to trace VDBE execution */
         1564  +#define SQLITE_VdbeAddopTrace HI(0x0008)  /* Trace sqlite3VdbeAddOp() calls */
         1565  +#define SQLITE_VdbeEQP        HI(0x0010)  /* Debug EXPLAIN QUERY PLAN */
  1564   1566   #endif
  1565   1567   
  1566   1568   /*
  1567   1569   ** Allowed values for sqlite3.mDbFlags
  1568   1570   */
  1569   1571   #define DBFLAG_SchemaChange   0x0001  /* Uncommitted Hash table changes */
  1570   1572   #define DBFLAG_PreferBuiltin  0x0002  /* Preference to built-in funcs */

Changes to src/whereexpr.c.

  1569   1569     Expr *pTerm;
  1570   1570     if( pItem->fg.isTabFunc==0 ) return;
  1571   1571     pTab = pItem->pTab;
  1572   1572     assert( pTab!=0 );
  1573   1573     pArgs = pItem->u1.pFuncArg;
  1574   1574     if( pArgs==0 ) return;
  1575   1575     for(j=k=0; j<pArgs->nExpr; j++){
         1576  +    Expr *pRhs;
  1576   1577       while( k<pTab->nCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;}
  1577   1578       if( k>=pTab->nCol ){
  1578   1579         sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d",
  1579   1580                         pTab->zName, j);
  1580   1581         return;
  1581   1582       }
  1582   1583       pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0);
  1583   1584       if( pColRef==0 ) return;
  1584   1585       pColRef->iTable = pItem->iCursor;
  1585   1586       pColRef->iColumn = k++;
  1586   1587       pColRef->y.pTab = pTab;
  1587         -    pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef,
  1588         -                         sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0));
         1588  +    pRhs = sqlite3PExpr(pParse, TK_UPLUS, 
         1589  +        sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0);
         1590  +    pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs);
  1589   1591       whereClauseInsert(pWC, pTerm, TERM_DYNAMIC);
  1590   1592     }
  1591   1593   }

Changes to test/bestindex4.test.

   112    112               set {} {}
   113    113           } {}
   114    114         }
   115    115       }
   116    116   
   117    117     }
   118    118   }
          119  +
          120  +#-------------------------------------------------------------------------
          121  +# Test that a parameter passed to a table-valued function cannot be
          122  +# used to drive an index. i.e. that in the following:
          123  +#
          124  +#   SELECT * FROM tbl, vtab(tbl.x);
          125  +#
          126  +# The implicit constraint "tbl.x = vtab.hidden" is not optimized using
          127  +# an index on tbl.x.
          128  +#
          129  +reset_db
          130  +register_tcl_module db
          131  +proc vtab_command {method args} {
          132  +  switch -- $method {
          133  +    xConnect {
          134  +      return "CREATE TABLE t1(a, b, c, d HIDDEN)"
          135  +    }
          136  +
          137  +    xBestIndex {
          138  +      set clist [lindex $args 0]
          139  +      if {[llength $clist]!=1} { error "unexpected constraint list" }
          140  +      catch { array unset C }
          141  +      array set C [lindex $clist 0]
          142  +      if {$C(usable)} {
          143  +        return [list omit 0 idxnum 555 rows 10 cost 100]
          144  +      }
          145  +      return [list cost 100000000]
          146  +    }
          147  +
          148  +  }
          149  +
          150  +  return {}
          151  +}
          152  +
          153  +do_execsql_test 2.0 {
          154  +  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
          155  +  CREATE TABLE t1 (x INT PRIMARY KEY);
          156  +} {}
          157  +
          158  +do_execsql_test 2.1 {
          159  +  EXPLAIN QUERY PLAN SELECT * FROM t1, x1 WHERE x1.d=t1.x;
          160  +} {
          161  +  3 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 0:}
          162  +  7 0 0 {SEARCH TABLE t1 USING COVERING INDEX sqlite_autoindex_t1_1 (x=?)}
          163  +}
          164  +
          165  +do_execsql_test 2.2 {
          166  +  EXPLAIN QUERY PLAN SELECT * FROM t1, x1(t1.x)
          167  +} {
          168  +  3 0 0 {SCAN TABLE t1} 
          169  +  5 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:}
          170  +}
          171  +
   119    172   
   120    173   finish_test

Added test/dbfuzz2-seed1.db.

cannot compute difference between binary files

Added test/dbfuzz2.c.

            1  +/*
            2  +** 2018-10-26
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This program is designed for fuzz-testing SQLite database files using
           14  +** the -fsanitize=fuzzer option of clang.
           15  +**
           16  +** The -fsanitize=fuzzer option causes a main() to be inserted automatically.
           17  +** That main() invokes LLVMFuzzerTestOneInput(D,S) to be invoked repeatedly.
           18  +** Each D is a fuzzed database file.  The code in this file runs various
           19  +** SQL statements against that database, trying to provoke a failure.
           20  +**
           21  +** For best results the seed database files should have these tables:
           22  +**
           23  +**   Table "t1" with columns "a" and "b"
           24  +**   Tables "t2" and "t3 with the same number of compatible columns
           25  +**       "t3" should have a column names "x"
           26  +**   Table "t4" with a column "x" that is compatible with t3.x.
           27  +**
           28  +** Any of these tables can be virtual tables, for example FTS or RTree tables.
           29  +**
           30  +** To run this test:
           31  +**
           32  +**     mkdir dir
           33  +**     cp dbfuzz2-seed*.db dir
           34  +**     clang-6.0 -I. -g -O1 -fsanitize=fuzzer \
           35  +**       -DTHREADSAFE=0 -DSQLITE_ENABLE_DESERIALIZE \
           36  +**       -DSQLITE_ENABLE_DBSTAT_VTAB dbfuzz2.c sqlite3.c -ldl
           37  +**     ./a.out dir
           38  +*/
           39  +#include <assert.h>
           40  +#include <stdio.h>
           41  +#include <stdlib.h>
           42  +#include <string.h>
           43  +#include <stdarg.h>
           44  +#include <ctype.h>
           45  +#include <stdint.h>
           46  +#include "sqlite3.h"
           47  +
           48  +/*
           49  +** This is the is the SQL that is run against the database.
           50  +*/
           51  +static const char *azSql[] = {
           52  +  "PRAGMA integrity_check;",
           53  +  "SELECT * FROM sqlite_master;",
           54  +  "SELECT sum(length(name)) FROM dbstat;",
           55  +  "UPDATE t1 SET b=a, a=b WHERE a<b;",
           56  +  "ALTER TABLE t1 RENAME TO alkjalkjdfiiiwuer987lkjwer82mx97sf98788s9789s;",
           57  +  "INSERT INTO t3 SELECT * FROM t2;",
           58  +  "DELETE FROM t3 WHERE x IN (SELECT x FROM t4);",
           59  +  "REINDEX;",
           60  +  "DROP TABLE t3;",
           61  +  "VACUUM;",
           62  +};
           63  +
           64  +/* Output verbosity level.  0 means complete silence */
           65  +int eVerbosity = 0;
           66  +
           67  +/* libFuzzer invokes this routine with fuzzed database files (in aData).
           68  +** This routine run SQLite against the malformed database to see if it
           69  +** can provoke a failure or malfunction.
           70  +*/
           71  +int LLVMFuzzerTestOneInput(const uint8_t *aData, size_t nByte){
           72  +  unsigned char *a;
           73  +  sqlite3 *db;
           74  +  int rc;
           75  +  int i;
           76  +
           77  +  if( eVerbosity>=1 ){
           78  +    printf("************** nByte=%d ***************\n", (int)nByte);
           79  +    fflush(stdout);
           80  +  }
           81  +  rc = sqlite3_open(0, &db);
           82  +  if( rc ) return 1;
           83  +  a = sqlite3_malloc64(nByte+1);
           84  +  if( a==0 ) return 1;
           85  +  memcpy(a, aData, nByte);
           86  +  sqlite3_deserialize(db, "main", a, nByte, nByte,
           87  +        SQLITE_DESERIALIZE_RESIZEABLE |
           88  +        SQLITE_DESERIALIZE_FREEONCLOSE);
           89  +  for(i=0; i<sizeof(azSql)/sizeof(azSql[0]); i++){
           90  +    if( eVerbosity>=1 ){
           91  +      printf("%s\n", azSql[i]);
           92  +      fflush(stdout);
           93  +    }
           94  +    sqlite3_exec(db, azSql[i], 0, 0, 0);
           95  +  }
           96  +  rc = sqlite3_close(db);
           97  +  if( rc!=SQLITE_OK ){
           98  +    fprintf(stdout, "sqlite3_close() returns %d\n", rc);
           99  +  }
          100  +  if( sqlite3_memory_used()!=0 ){
          101  +    int nAlloc = 0;
          102  +    int nNotUsed = 0;
          103  +    sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &nAlloc, &nNotUsed, 0);
          104  +    fprintf(stderr,"Memory leak: %lld bytes in %d allocations\n",
          105  +            sqlite3_memory_used(), nAlloc);
          106  +    exit(1);
          107  +  }
          108  +  return 0;
          109  +}
          110  +
          111  +/* libFuzzer invokes this routine once when the executable starts, to
          112  +** process the command-line arguments.
          113  +*/
          114  +int LLVMFuzzerInitialize(int *pArgc, char ***pArgv){
          115  +  int i, j;
          116  +  int argc = *pArgc;
          117  +  char **newArgv;
          118  +  char **argv = *pArgv;
          119  +  newArgv = malloc( sizeof(char*)*(argc+1) );
          120  +  if( newArgv==0 ) return 0;
          121  +  newArgv[0] = argv[0];
          122  +  for(i=j=1; i<argc; i++){
          123  +    char *z = argv[i];
          124  +    if( z[0]=='-' ){
          125  +      z++;
          126  +      if( z[0]=='-' ) z++;
          127  +      if( strcmp(z,"v")==0 ){
          128  +        eVerbosity++;
          129  +        continue;
          130  +      }
          131  +    }
          132  +    newArgv[j++] = argv[i];
          133  +  }
          134  +  newArgv[j] = 0;
          135  +  *pArgv = newArgv;
          136  +  *pArgc = j;
          137  +  return 0;
          138  +}

Added test/fuzzdata7.db.

cannot compute difference between binary files

Changes to tool/mkpragmatab.tcl.

   116    116     TYPE: FLAG
   117    117     ARG:  SQLITE_IgnoreChecks
   118    118     IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   119    119     IF:   !defined(SQLITE_OMIT_CHECK)
   120    120   
   121    121     NAME: writable_schema
   122    122     TYPE: FLAG
   123         -  ARG:  SQLITE_WriteSchema
          123  +  ARG:  SQLITE_WriteSchema|SQLITE_NoSchemaError
   124    124     IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   125    125   
   126    126     NAME: read_uncommitted
   127    127     TYPE: FLAG
   128    128     ARG:  SQLITE_ReadUncommit
   129    129     IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
   130    130   
................................................................................
   575    575   puts $fd "typedef struct PragmaName \173"
   576    576   puts $fd "  const char *const zName; /* Name of pragma */"
   577    577   puts $fd "  u8 ePragTyp;             /* PragTyp_XXX value */"
   578    578   puts $fd "  u8 mPragFlg;             /* Zero or more PragFlg_XXX values */"
   579    579   puts $fd {  u8 iPragCName;           /* Start of column names in pragCName[] */}
   580    580   puts $fd "  u8 nPragCName;          \
   581    581   /* Num of col names. 0 means use pragma name */"
   582         -puts $fd "  u32 iArg;                /* Extra argument */"
          582  +puts $fd "  u64 iArg;                /* Extra argument */"
   583    583   puts $fd "\175 PragmaName;"
   584    584   puts $fd "static const PragmaName aPragmaName\[\] = \173"
   585    585   
   586    586   set current_if {}
   587    587   set spacer [format {    %26s } {}]
   588    588   foreach name $allnames {
   589    589     foreach {type arg if flag cx} $allbyname($name) break