SQLite

# 2017 March 02
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set ::testprefix rbucrash2

db close
forcedelete test.db-oal rbu.db
sqlite3_shutdown
sqlite3_config_uri 1
reset_db

# Set up a target database and an rbu update database. The target
# db is the usual "test.db", the rbu db is "test.db2".
#
forcedelete test.db2
do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b));
  INSERT INTO t1 VALUES(1, 2, 3);
  INSERT INTO t1 VALUES(4, 5, 6);
  INSERT INTO t1 VALUES(7, 8, 9);

  ATTACH 'test.db2' AS rbu;
  CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
  INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0);
}
db_save_and_close

proc do_rbu_crash_test2 {tn script} {

  foreach {f blksz} {
    test.db   512
    test.db2  512
    test.db   4096
    test.db2  4096
  } {
    set bDone 0
    for {set iDelay 1} {$bDone==0} {incr iDelay} {
      forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal
      db_restore
  
      set res [
        crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \
            -blocksize $blksz {}
      ]
  
      set bDone 1
      if {$res == "1 {child process exited abnormally}"} {
        set bDone 0
      } elseif {$res != "0 {}"} {
        error "unexected catchsql result: $res"
      }
  
      sqlite3rbu rbu test.db test.db2
      while {[rbu step]=="SQLITE_OK"} {}
      rbu close
  
      sqlite3 db test.db
      do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz {
        PRAGMA integrity_check;
      } {ok}
      db close
    }
  }
}

for {set x 1} {$x < 10} {incr x} {
  do_rbu_crash_test2 1.$x {
    sqlite3rbu rbu test.db test.db2
    while {[rbu step]=="SQLITE_OK"} {
      rbu savestate
    }
    rbu close
  }
}

for {set x 1} {$x < 2} {incr x} {
  do_rbu_crash_test2 2.$x {
    sqlite3rbu rbu test.db test.db2
    while {[rbu step]=="SQLITE_OK"} {
      rbu close
      sqlite3rbu rbu test.db test.db2
    }
    rbu close
  }
}

finish_test

    INSERT INTO rbu_count VALUES('data_t1', 3);
  }
  return $filename
}


do_execsql_test 1.0 {
  PRAGMA page_size = 4096;
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}

do_test 1.1 {
  create_rbu1 rbu.db
  sqlite3rbu rbu test.db rbu.db
  rbu bp_progress

  }] {SQLITE_DONE}]
}

foreach bReopen {0 1} {
  do_test 3.$bReopen.1.0 {
    reset_db
    execsql {
      PRAGMA page_size = 4096;
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, rbu_control);

  int rc;                         /* Value returned by last rbu_step() call */
  char *zErrmsg;                  /* Error message if rc!=SQLITE_OK */
  int nStep;                      /* Rows processed for current object */
  int nProgress;                  /* Rows processed for all objects */
  RbuObjIter objiter;             /* Iterator for skipping through tbl/idx */
  const char *zVfsName;           /* Name of automatically created rbu vfs */
  rbu_file *pTargetFd;            /* File handle open on target db */
  int nPagePerSector;             /* Pages per sector for pTargetFd */
  i64 iOalSz;
  i64 nPhaseOneStep;

  /* The following state variables are used as part of the incremental
  ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
  ** function rbuSetupCheckpoint() for details.  */
  u32 iMaxFrame;                  /* Largest iWalFrame value in aFrame[] */

    p->iWalCksum = rbuShmChecksum(p);
  }

  if( p->rc==SQLITE_OK ){
    if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
      p->rc = SQLITE_DONE;
      p->eStage = RBU_STAGE_DONE;
    }else{
      int nSectorSize;
      sqlite3_file *pDb = p->pTargetFd->pReal;
      sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
      assert( p->nPagePerSector==0 );
      nSectorSize = pDb->pMethods->xSectorSize(pDb);
      if( nSectorSize>p->pgsz ){
        p->nPagePerSector = nSectorSize / p->pgsz;
      }else{
        p->nPagePerSector = 1;
      }

      /* Call xSync() on the wal file. This causes SQLite to sync the 
      ** directory in which the target database and the wal file reside, in 
      ** case it has not been synced since the rename() call in 
      ** rbuMoveOalFile(). */
      p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL);
    }
  }
}

/*
** Called when iAmt bytes are read from offset iOff of the wal file while
** the rbu object is in capture mode. Record the frame number of the frame

            }
  
            if( p->rc==SQLITE_OK ){
              p->eStage = RBU_STAGE_DONE;
              p->rc = SQLITE_DONE;
            }
          }else{
            /* At one point the following block copied a single frame from the
            ** wal file to the database file. So that one call to sqlite3rbu_step()
            ** checkpointed a single frame. 
            **
            ** However, if the sector-size is larger than the page-size, and the
            ** application calls sqlite3rbu_savestate() or close() immediately
            ** after this step, then rbu_step() again, then a power failure occurs,
            ** then the database page written here may be damaged. Work around
            ** this by checkpointing frames until the next page in the aFrame[]
            ** lies on a different disk sector to the current one. */
            u32 iSector;
            do{
              RbuFrame *pFrame = &p->aFrame[p->nStep];
              iSector = (pFrame->iDbPage-1) / p->nPagePerSector;
              rbuCheckpointFrame(p, pFrame);
              p->nStep++;
            }while( p->nStep<p->nFrame 
                 && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector)
                 && p->rc==SQLITE_OK
            );
          }
          p->nProgress++;
        }
        break;
      }

      default:

  int rc;
  if( p ){

    /* Commit the transaction to the *-oal file. */
    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
      p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
    }

    /* Sync the db file if currently doing an incremental checkpoint */
    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
      sqlite3_file *pDb = p->pTargetFd->pReal;
      p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
    }

    rbuSaveState(p, p->eStage);

    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
      p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
    }

  if( rc==SQLITE_DONE ) return SQLITE_OK;

  assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
  if( p->eStage==RBU_STAGE_OAL ){
    assert( rc!=SQLITE_DONE );
    if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
  }

  /* Sync the db file */
  if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
    sqlite3_file *pDb = p->pTargetFd->pReal;
    rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
  }

  p->rc = rc;
  rbuSaveState(p, p->eStage);
  rc = p->rc;

  if( p->eStage==RBU_STAGE_OAL ){
    assert( rc!=SQLITE_DONE );

  temp = 0;
  src = data = pPage->aData;
  hdr = pPage->hdrOffset;
  cellOffset = pPage->cellOffset;
  nCell = pPage->nCell;
  assert( nCell==get2byte(&data[hdr+3]) );
  iCellFirst = cellOffset + 2*nCell;
  usableSize = pPage->pBt->usableSize;

  /* This block handles pages with two or fewer free blocks and nMaxFrag
  ** or fewer fragmented bytes. In this case it is faster to move the
  ** two (or one) blocks of cells using memmove() and add the required
  ** offsets to each pointer in the cell-pointer array than it is to 
  ** reconstruct the entire page.  */
  if( (int)data[hdr+7]<=nMaxFrag ){
    int iFree = get2byte(&data[hdr+1]);
    if( iFree ){
      int iFree2 = get2byte(&data[iFree]);

      /* pageFindSlot() has already verified that free blocks are sorted
      ** in order of offset within the page, and that no block extends
      ** past the end of the page. Provided the two free slots do not 
      ** overlap, this guarantees that the memmove() calls below will not
      ** overwrite the usableSize byte buffer, even if the database page
      ** is corrupt.  */
      assert( iFree2==0 || iFree2>iFree );
      assert( iFree+get2byte(&data[iFree+2]) <= usableSize );
      assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize );

      if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){
        u8 *pEnd = &data[cellOffset + nCell*2];
        u8 *pAddr;
        int sz2 = 0;
        int sz = get2byte(&data[iFree+2]);
        int top = get2byte(&data[hdr+5]);
        if( iFree2 ){
          if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_BKPT;
          sz2 = get2byte(&data[iFree2+2]);
          assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize );
          memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz));
          sz += sz2;
        }
        cbrk = top+sz;
        assert( cbrk+(iFree-top) <= usableSize );
        memmove(&data[cbrk], &data[top], iFree-top);
        for(pAddr=&data[cellOffset]; pAddr<pEnd; pAddr+=2){
          pc = get2byte(pAddr);
          if( pc<iFree ){ put2byte(pAddr, pc+sz); }
          else if( pc<iFree2 ){ put2byte(pAddr, pc+sz2); }
        }
        goto defragment_out;
      }
    }
  }


  cbrk = usableSize;
  iCellLast = usableSize - 4;

  for(i=0; i<nCell; i++){
    u8 *pAddr;     /* The i-th cell pointer */
    pAddr = &data[cellOffset + i*2];
    pc = get2byte(pAddr);
    testcase( pc==iCellFirst );
    testcase( pc==iCellLast );
    /* These conditions have already been verified in btreeInitPage()

      /*
      **    start of TTTTT
      **
      ** Move the date backwards to the beginning of the current day,
      ** or month or year.
      */
      if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break;
      if( !p->validJD && !p->validYMD && !p->validHMS ) break;
      z += 9;
      computeYMD(p);
      p->validHMS = 1;
      p->h = p->m = 0;
      p->s = 0.0;
      p->rawS = 0;
      p->validTZ = 0;
      p->validJD = 0;
      if( sqlite3_stricmp(z,"month")==0 ){
        p->D = 1;
        rc = 0;
      }else if( sqlite3_stricmp(z,"year")==0 ){

        p->M = 1;
        p->D = 1;
        rc = 0;
      }else if( sqlite3_stricmp(z,"day")==0 ){
        rc = 0;
      }
      break;

        u8 *zGarbage;
        int iFirst = (int)(pWrite->iOffset/g.iSectorSize);
        int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize);

        assert(pWrite->zBuf);

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", 
            1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize,
            pWrite->iOffset, iFirst, pWrite->pFile->zName
        );
#endif

        zGarbage = crash_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){

  }
  writeListSync(0, 1);
  assert( 0 );
  return TCL_OK;
}

/*
** tclcmd:   sqlite_crash_enable ENABLE ?DEFAULT?
**
** Parameter ENABLE must be a boolean value. If true, then the "crash"
** vfs is added to the system. If false, it is removed.
*/
static int SQLITE_TCLAPI crashEnableCmd(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int isEnable;
  int isDefault = 0;
  static sqlite3_vfs crashVfs = {
    2,                  /* iVersion */
    0,                  /* szOsFile */
    0,                  /* mxPathname */
    0,                  /* pNext */
    "crash",            /* zName */
    0,                  /* pAppData */

    cfRandomness,         /* xRandomness */
    cfSleep,              /* xSleep */
    cfCurrentTime,        /* xCurrentTime */
    cfGetLastError,       /* xGetLastError */
    0,                    /* xCurrentTimeInt64 */
  };

  if( objc!=2 && objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "ENABLE ?DEFAULT?");
    return TCL_ERROR;
  }

  if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){
    return TCL_ERROR;
  }
  if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){
    return TCL_ERROR;
  }

  if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){
    return TCL_OK;
  }

  if( crashVfs.pAppData==0 ){
    sqlite3_vfs *pOriginalVfs = sqlite3_vfs_find(0);
    crashVfs.mxPathname = pOriginalVfs->mxPathname;
    crashVfs.pAppData = (void *)pOriginalVfs;
    crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile;
    sqlite3_vfs_register(&crashVfs, isDefault);
  }else{
    crashVfs.pAppData = 0;
    sqlite3_vfs_unregister(&crashVfs);
  }

  return TCL_OK;
}

  ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
  ** parameter in the WHERE clause might influence the choice of query plan
  ** for a statement, then the statement will be automatically recompiled,
  ** as if there had been a schema change, on the first sqlite3_step() call
  ** following any change to the bindings of that parameter.
  */
  assert( p->isPrepareV2 || p->expmask==0 );
  if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<<i))!=0 ){
    p->expired = 1;
  }
  return SQLITE_OK;
}

/*
** Bind a text or BLOB value.

/*
** Configure SQL variable iVar so that binding a new value to it signals
** to sqlite3_reoptimize() that re-preparing the statement may result
** in a better query plan.
*/
void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){
  assert( iVar>0 );
  if( iVar>=32 ){
    v->expmask |= 0x80000000;
  }else{
    v->expmask |= ((u32)1 << (iVar-1));
  }
}

#ifndef SQLITE_OMIT_VIRTUALTABLE
/*

# 2017-03-03
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix corruptK

if {[permutation]=="mmap"} {
  finish_test
  return
}

# This module uses hard-coded offsets which do not work if the reserved_bytes
# value is nonzero.
if {[nonzero_reserved_bytes]} {finish_test; return;}
database_may_be_corrupt

# Initialize the database.
# 
do_execsql_test 1.1 {
  PRAGMA page_size=1024;
  PRAGMA auto_vacuum=0;
  CREATE TABLE t1(x);

  INSERT INTO t1 VALUES(randomblob(20));
  INSERT INTO t1 VALUES(randomblob(100));   -- make this into a free slot
  INSERT INTO t1 VALUES(randomblob(27));    -- this one will be corrupt
  INSERT INTO t1 VALUES(randomblob(800));

  DELETE FROM t1 WHERE rowid=2;  -- free the 100 byte slot
  PRAGMA page_count
} {2}


# Corrupt the database so that the blob stored immediately before 
# the free slot (rowid==3) has an overlarge length field. So that
# we can use sqlite3_blob_write() to manipulate the size field of
# the free slot.
#
# Then use sqlite3_blob_write() to set the size of said free slot
# to 24 bytes (instead of the actual 100).
#
# Then use the new 24 byte slot. Leaving the in-memory version of
# the page with zero free slots and a large nFree value. Then try
# to allocate another slot to get to defragmentPage().
#
do_test 1.2 {
  db close
  hexio_write test.db [expr 1024 + 0x360] 21
  hexio_write test.db [expr 1024 + 0x363] [format %x [expr 31*2 + 12]]
  sqlite3 db test.db

  set fd [db incrblob t1 x 3]
  fconfigure $fd -translation binary -encoding binary
  seek $fd 30
  puts -nonewline $fd "\x18"
  close $fd
} {}
do_execsql_test 1.3 {
  INSERT INTO t1 VALUES(randomblob(20));
}
do_catchsql_test 1.4 {
  INSERT INTO t1 VALUES(randomblob(90));
} {1 {database disk image is malformed}}

#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.1 {
  PRAGMA page_size=1024;
  PRAGMA auto_vacuum=0;
  CREATE TABLE t1(x);

  INSERT INTO t1 VALUES(randomblob(20));
  INSERT INTO t1 VALUES(randomblob(20));    -- free this one
  INSERT INTO t1 VALUES(randomblob(20));
  INSERT INTO t1 VALUES(randomblob(20));    -- and this one
  INSERT INTO t1 VALUES(randomblob(20));    -- corrupt this one.

  DELETE FROM t1 WHERE rowid IN(2, 4);
  PRAGMA page_count
} {2}

do_test 2.2 {
  db close
  hexio_write test.db [expr 1024 + 0x388] 53
  hexio_write test.db [expr 1024 + 0x38A] 03812C

  sqlite3 db test.db
  set fd [db incrblob t1 x 5]
  fconfigure $fd -translation binary -encoding binary

  seek $fd 22
  puts -nonewline $fd "\x5d"
  close $fd
} {}

do_catchsql_test 2.3 {
  INSERT INTO t1 VALUES(randomblob(900));
} {1 {database disk image is malformed}}




finish_test

datetest 16.25 {datetime(5373484,'-128963628 hours')} NULL
datetest 16.26 {datetime(5373484,'-5373484 days')} {-4713-11-24 12:00:00}
datetest 16.27 {datetime(5373484,'-5373485 days')} NULL
datetest 16.28 {datetime(5373484,'-176545 months')} {-4713-12-01 12:00:00}
datetest 16.29 {datetime(5373484,'-176546 months')} NULL
datetest 16.30 {datetime(5373484,'-14712 years')} {-4713-12-31 12:00:00}
datetest 16.31 {datetime(5373484,'-14713 years')} NULL

# 2017-03-02: Wrong 'start of day' computation.
# https://www.sqlite.org/src/info/6097cb92745327a1
#
datetest 17.1 {datetime(2457754, 'start of day')} {2016-12-31 00:00:00}
datetest 17.2 {datetime(2457828)} {2017-03-15 12:00:00}
datetest 17.3 {datetime(2457828,'start of day')} {2017-03-15 00:00:00}
datetest 17.4 {datetime(2457828,'start of month')} {2017-03-01 00:00:00}
datetest 17.5 {datetime(2457828,'start of year')} {2017-01-01 00:00:00}
datetest 17.6 {datetime(37,'start of year')} NULL
datetest 17.7 {datetime(38,'start of year')} {-4712-01-01 00:00:00}



finish_test

  set blocksize ""
  set crashdelay 1
  set prngseed 0
  set opendb { sqlite3 db test.db -vfs crash }
  set tclbody {}
  set crashfile ""
  set dc ""
  set dfltvfs 0
  set sql [lindex $args end]

  for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} {
    set z [lindex $args $ii]
    set n [string length $z]
    set z2 [lindex $args [expr $ii+1]]

    if     {$n>1 && [string first $z -delay]==0}     {set crashdelay $z2} \
    elseif {$n>1 && [string first $z -opendb]==0}    {set opendb $z2} \
    elseif {$n>1 && [string first $z -seed]==0}      {set prngseed $z2} \
    elseif {$n>1 && [string first $z -file]==0}      {set crashfile $z2}  \
    elseif {$n>1 && [string first $z -tclbody]==0}   {set tclbody $z2}  \
    elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \
    elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" }\
    elseif {$n>1 && [string first $z -dfltvfs]==0} {set dfltvfs $z2 }\
    else   { error "Unrecognized option: $z" }
  }

  if {$crashfile eq ""} {
    error "Compulsory option -file missing"
  }

  # $crashfile gets compared to the native filename in
  # cfSync(), which can be different then what TCL uses by
  # default, so here we force it to the "nativename" format.
  set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]]

  set f [open crash.tcl w]
  puts $f "sqlite3_crash_enable 1 $dfltvfs"
  puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile"
  puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte"

  # This block sets the cache size of the main database to 10
  # pages. This is done in case the build is configured to omit
  # "PRAGMA cache_size".
  if {$opendb!=""} {