/ Check-in [edee6a80]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: edee6a80e1cc7e6a2b8c3c7f76dd794fc8ab9a72
User & Date: dan 2017-03-02 14:51:47
References
2017-03-07
14:45
When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt. Cherrypick of [edee6a80]. check-in: 811a5599 user: dan tags: version-3.17.0-rbu-fixes
Context
2017-03-07
14:45
When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt. Cherrypick of [edee6a80]. check-in: 811a5599 user: dan tags: version-3.17.0-rbu-fixes
2017-03-02
16:56
Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size. check-in: 4012bb3a user: dan tags: trunk
14:51
When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt. check-in: edee6a80 user: dan tags: trunk
13:13
Make sure the Vdbe.expmask value is set correctly in sqlite3VdbeSwap(). This fixes a problem introduced by [a8fd7052]. check-in: 29f54b89 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Added ext/rbu/rbucrash2.test.











































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
# 2017 March 02
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set ::testprefix rbucrash2

db close
forcedelete test.db-oal rbu.db
sqlite3_shutdown
sqlite3_config_uri 1
reset_db

# Set up a target database and an rbu update database. The target
# db is the usual "test.db", the rbu db is "test.db2".
#
forcedelete test.db2
do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b));
  INSERT INTO t1 VALUES(1, 2, 3);
  INSERT INTO t1 VALUES(4, 5, 6);
  INSERT INTO t1 VALUES(7, 8, 9);

  ATTACH 'test.db2' AS rbu;
  CREATE TABLE rbu.data_t1(a, b, c, rbu_control);
  INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0);
  INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0);
}
db_save_and_close

proc do_rbu_crash_test2 {tn script} {

  foreach f {test.db test.db2} {
    set bDone 0
    for {set iDelay 1} {$bDone==0} {incr iDelay} {
      forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal
      db_restore
  
      set res [
        crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \
            -blocksize 512 {}
      ]
  
      set bDone 1
      if {$res == "1 {child process exited abnormally}"} {
        set bDone 0
      } elseif {$res != "0 {}"} {
        error "unexected catchsql result: $res"
      }
  
      sqlite3rbu rbu test.db test.db2
      while {[rbu step]=="SQLITE_OK"} {}
      rbu close
  
      sqlite3 db test.db
      do_execsql_test $tn.delay=$iDelay.f=$f {
        PRAGMA integrity_check;
      } {ok}
      db close
    }
  }
}

for {set x 1} {$x < 10} {incr x} {
  do_rbu_crash_test2 1.$x {
    sqlite3rbu rbu test.db test.db2
    while {[rbu step]=="SQLITE_OK"} {
      rbu savestate
    }
    rbu close
  }
}

for {set x 1} {$x < 2} {incr x} {
  do_rbu_crash_test2 2.$x {
    sqlite3rbu rbu test.db test.db2
    while {[rbu step]=="SQLITE_OK"} {
      rbu close
      sqlite3rbu rbu test.db test.db2
    }
    rbu close
  }
}

finish_test

Changes to ext/rbu/sqlite3rbu.c.

3713
3714
3715
3716
3717
3718
3719






3720
3721
3722
3723
3724
3725
3726
....
3837
3838
3839
3840
3841
3842
3843






3844
3845
3846
3847
3848
3849
3850
  int rc;
  if( p ){

    /* Commit the transaction to the *-oal file. */
    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
      p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
    }







    rbuSaveState(p, p->eStage);

    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
      p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
    }

................................................................................
  if( rc==SQLITE_DONE ) return SQLITE_OK;

  assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
  if( p->eStage==RBU_STAGE_OAL ){
    assert( rc!=SQLITE_DONE );
    if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
  }







  p->rc = rc;
  rbuSaveState(p, p->eStage);
  rc = p->rc;

  if( p->eStage==RBU_STAGE_OAL ){
    assert( rc!=SQLITE_DONE );







>
>
>
>
>
>







 







>
>
>
>
>
>







3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
....
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
  int rc;
  if( p ){

    /* Commit the transaction to the *-oal file. */
    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
      p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
    }

    /* Sync the db file if currently doing an incremental checkpoint */
    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
      sqlite3_file *pDb = p->pTargetFd->pReal;
      p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
    }

    rbuSaveState(p, p->eStage);

    if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
      p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
    }

................................................................................
  if( rc==SQLITE_DONE ) return SQLITE_OK;

  assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
  if( p->eStage==RBU_STAGE_OAL ){
    assert( rc!=SQLITE_DONE );
    if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
  }

  /* Sync the db file */
  if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
    sqlite3_file *pDb = p->pTargetFd->pReal;
    rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
  }

  p->rc = rc;
  rbuSaveState(p, p->eStage);
  rc = p->rc;

  if( p->eStage==RBU_STAGE_OAL ){
    assert( rc!=SQLITE_DONE );

Changes to src/test6.c.

311
312
313
314
315
316
317
318

319
320
321
322
323
324
325
326
...
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841

842
843
844
845
846
847
848
...
858
859
860
861
862
863
864
865
866
867
868
869
870
871



872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
        u8 *zGarbage;
        int iFirst = (int)(pWrite->iOffset/g.iSectorSize);
        int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize);

        assert(pWrite->zBuf);

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors @ %lld (sector %d) (%s)\n", 

            1+iLast-iFirst, pWrite->iOffset, iFirst, pWrite->pFile->zName
        );
#endif

        zGarbage = crash_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
................................................................................
  }
  writeListSync(0, 1);
  assert( 0 );
  return TCL_OK;
}

/*
** tclcmd:   sqlite_crash_enable ENABLE
**
** Parameter ENABLE must be a boolean value. If true, then the "crash"
** vfs is added to the system. If false, it is removed.
*/
static int SQLITE_TCLAPI crashEnableCmd(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int isEnable;

  static sqlite3_vfs crashVfs = {
    2,                  /* iVersion */
    0,                  /* szOsFile */
    0,                  /* mxPathname */
    0,                  /* pNext */
    "crash",            /* zName */
    0,                  /* pAppData */
................................................................................
    cfRandomness,         /* xRandomness */
    cfSleep,              /* xSleep */
    cfCurrentTime,        /* xCurrentTime */
    cfGetLastError,       /* xGetLastError */
    0,                    /* xCurrentTimeInt64 */
  };

  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "ENABLE");
    return TCL_ERROR;
  }

  if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){
    return TCL_ERROR;



  }

  if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){
    return TCL_OK;
  }

  if( crashVfs.pAppData==0 ){
    sqlite3_vfs *pOriginalVfs = sqlite3_vfs_find(0);
    crashVfs.mxPathname = pOriginalVfs->mxPathname;
    crashVfs.pAppData = (void *)pOriginalVfs;
    crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile;
    sqlite3_vfs_register(&crashVfs, 0);
  }else{
    crashVfs.pAppData = 0;
    sqlite3_vfs_unregister(&crashVfs);
  }

  return TCL_OK;
}







|
>
|







 







|











>







 







|
|





>
>
>











|







311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
...
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
...
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
        u8 *zGarbage;
        int iFirst = (int)(pWrite->iOffset/g.iSectorSize);
        int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize);

        assert(pWrite->zBuf);

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", 
            1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize,
            pWrite->iOffset, iFirst, pWrite->pFile->zName
        );
#endif

        zGarbage = crash_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
................................................................................
  }
  writeListSync(0, 1);
  assert( 0 );
  return TCL_OK;
}

/*
** tclcmd:   sqlite_crash_enable ENABLE ?DEFAULT?
**
** Parameter ENABLE must be a boolean value. If true, then the "crash"
** vfs is added to the system. If false, it is removed.
*/
static int SQLITE_TCLAPI crashEnableCmd(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int isEnable;
  int isDefault = 0;
  static sqlite3_vfs crashVfs = {
    2,                  /* iVersion */
    0,                  /* szOsFile */
    0,                  /* mxPathname */
    0,                  /* pNext */
    "crash",            /* zName */
    0,                  /* pAppData */
................................................................................
    cfRandomness,         /* xRandomness */
    cfSleep,              /* xSleep */
    cfCurrentTime,        /* xCurrentTime */
    cfGetLastError,       /* xGetLastError */
    0,                    /* xCurrentTimeInt64 */
  };

  if( objc!=2 && objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "ENABLE ?DEFAULT?");
    return TCL_ERROR;
  }

  if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){
    return TCL_ERROR;
  }
  if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){
    return TCL_ERROR;
  }

  if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){
    return TCL_OK;
  }

  if( crashVfs.pAppData==0 ){
    sqlite3_vfs *pOriginalVfs = sqlite3_vfs_find(0);
    crashVfs.mxPathname = pOriginalVfs->mxPathname;
    crashVfs.pAppData = (void *)pOriginalVfs;
    crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile;
    sqlite3_vfs_register(&crashVfs, isDefault);
  }else{
    crashVfs.pAppData = 0;
    sqlite3_vfs_unregister(&crashVfs);
  }

  return TCL_OK;
}

Changes to test/tester.tcl.

1529
1530
1531
1532
1533
1534
1535

1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549

1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
  set blocksize ""
  set crashdelay 1
  set prngseed 0
  set opendb { sqlite3 db test.db -vfs crash }
  set tclbody {}
  set crashfile ""
  set dc ""

  set sql [lindex $args end]

  for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} {
    set z [lindex $args $ii]
    set n [string length $z]
    set z2 [lindex $args [expr $ii+1]]

    if     {$n>1 && [string first $z -delay]==0}     {set crashdelay $z2} \
    elseif {$n>1 && [string first $z -opendb]==0}    {set opendb $z2} \
    elseif {$n>1 && [string first $z -seed]==0}      {set prngseed $z2} \
    elseif {$n>1 && [string first $z -file]==0}      {set crashfile $z2}  \
    elseif {$n>1 && [string first $z -tclbody]==0}   {set tclbody $z2}  \
    elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \
    elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \

    else   { error "Unrecognized option: $z" }
  }

  if {$crashfile eq ""} {
    error "Compulsory option -file missing"
  }

  # $crashfile gets compared to the native filename in
  # cfSync(), which can be different then what TCL uses by
  # default, so here we force it to the "nativename" format.
  set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]]

  set f [open crash.tcl w]
  puts $f "sqlite3_crash_enable 1"
  puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile"
  puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte"

  # This block sets the cache size of the main database to 10
  # pages. This is done in case the build is configured to omit
  # "PRAGMA cache_size".
  if {$opendb!=""} {







>













|
>













|







1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
  set blocksize ""
  set crashdelay 1
  set prngseed 0
  set opendb { sqlite3 db test.db -vfs crash }
  set tclbody {}
  set crashfile ""
  set dc ""
  set dfltvfs 0
  set sql [lindex $args end]

  for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} {
    set z [lindex $args $ii]
    set n [string length $z]
    set z2 [lindex $args [expr $ii+1]]

    if     {$n>1 && [string first $z -delay]==0}     {set crashdelay $z2} \
    elseif {$n>1 && [string first $z -opendb]==0}    {set opendb $z2} \
    elseif {$n>1 && [string first $z -seed]==0}      {set prngseed $z2} \
    elseif {$n>1 && [string first $z -file]==0}      {set crashfile $z2}  \
    elseif {$n>1 && [string first $z -tclbody]==0}   {set tclbody $z2}  \
    elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \
    elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" }\
    elseif {$n>1 && [string first $z -dfltvfs]==0} {set dfltvfs $z2 }\
    else   { error "Unrecognized option: $z" }
  }

  if {$crashfile eq ""} {
    error "Compulsory option -file missing"
  }

  # $crashfile gets compared to the native filename in
  # cfSync(), which can be different then what TCL uses by
  # default, so here we force it to the "nativename" format.
  set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]]

  set f [open crash.tcl w]
  puts $f "sqlite3_crash_enable 1 $dfltvfs"
  puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile"
  puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte"

  # This block sets the cache size of the main database to 10
  # pages. This is done in case the build is configured to omit
  # "PRAGMA cache_size".
  if {$opendb!=""} {