Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch version-3.17.0-rbu-fixes Excluding Merge-Ins
This is equivalent to a diff from ada05cfa to 694fe0b2
2017-03-07
| ||
14:47 | Before beginning an incremental checkpoint in RBU, sync the directory containing the target database file. This ensures that the new directory entry created by renaming the *-oal file to *-wal is synced to disk. Cherrypick of [915a9a28]. (Leaf check-in: 694fe0b2 user: dan tags: version-3.17.0-rbu-fixes) | |
14:46 | Fix another RBU case similar to the previous. This one for systems where the sector-size is larger than the page-size. Cherrypick of [4012bb3a]. (check-in: 59a11b7f user: dan tags: version-3.17.0-rbu-fixes) | |
14:45 | When saving the state of an RBU update in the incremental-checkpoint phase, sync the database file. Otherwise, if a power failure occurs and the RBU update resumed following system recovery, the database may become corrupt. Cherrypick of [edee6a80]. (check-in: 811a5599 user: dan tags: version-3.17.0-rbu-fixes) | |
2017-03-03
| ||
16:51 | Before beginning an incremental checkpoint in RBU, sync the directory containing the target database file. This ensures that the new directory entry created by renaming the *-oal file to *-wal is synced to disk. (check-in: 915a9a28 user: dan tags: trunk) | |
2017-02-15
| ||
15:11 | Remove the CLANG_VERSION macro, since we have learned that version numbers in clang are "marketing" and are inconsistent and unreliable. Builds using clang will still use the GCC_VERSION macro since clang works hard to be gcc compatible. (check-in: 8d3f485d user: drh tags: branch-3.17) | |
2017-02-13
| ||
16:17 | Merge all changes up through the 3.17.0 release. (check-in: 1913a758 user: drh tags: apple-osx) | |
16:02 | Version 3.17.0 (check-in: ada05cfa user: drh tags: release, version-3.17.0, branch-3.17) | |
11:35 | Fix typos in using the MSVC_VERSION macro. (check-in: 25ebadd0 user: drh tags: branch-3.17) | |
Added ext/rbu/rbucrash2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | # 2017 March 02 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } source $testdir/tester.tcl set ::testprefix rbucrash2 db close forcedelete test.db-oal rbu.db sqlite3_shutdown sqlite3_config_uri 1 reset_db # Set up a target database and an rbu update database. The target # db is the usual "test.db", the rbu db is "test.db2". # forcedelete test.db2 do_execsql_test 1.0 { CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b)); INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t1 VALUES(7, 8, 9); ATTACH 'test.db2' AS rbu; CREATE TABLE rbu.data_t1(a, b, c, rbu_control); INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0); INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0); INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0); INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0); INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0); INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0); } db_save_and_close proc do_rbu_crash_test2 {tn script} { foreach {f blksz} { test.db 512 test.db2 512 test.db 4096 test.db2 4096 } { set bDone 0 for {set iDelay 1} {$bDone==0} {incr iDelay} { forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal db_restore set res [ crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \ -blocksize $blksz {} ] set bDone 1 if {$res == "1 {child process exited abnormally}"} { set bDone 0 } elseif {$res != "0 {}"} { error "unexected catchsql result: $res" } sqlite3rbu rbu test.db test.db2 while {[rbu step]=="SQLITE_OK"} {} rbu close sqlite3 db test.db do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz { PRAGMA integrity_check; } {ok} db close } } } for {set x 1} {$x < 10} {incr x} { do_rbu_crash_test2 1.$x { sqlite3rbu rbu test.db test.db2 while {[rbu step]=="SQLITE_OK"} { rbu savestate } rbu close } } for {set x 1} {$x < 2} {incr x} { do_rbu_crash_test2 2.$x { sqlite3rbu rbu test.db test.db2 while {[rbu step]=="SQLITE_OK"} { rbu close sqlite3rbu rbu test.db test.db2 } rbu close } } finish_test |
Changes to ext/rbu/rbuprogress.test.
︙ | ︙ | |||
36 37 38 39 40 41 42 43 44 45 46 47 48 49 | INSERT INTO rbu_count VALUES('data_t1', 3); } return $filename } do_execsql_test 1.0 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); } do_test 1.1 { create_rbu1 rbu.db sqlite3rbu rbu test.db rbu.db rbu bp_progress | > | 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | INSERT INTO rbu_count VALUES('data_t1', 3); } return $filename } do_execsql_test 1.0 { PRAGMA page_size = 4096; CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); } do_test 1.1 { create_rbu1 rbu.db sqlite3rbu rbu test.db rbu.db rbu bp_progress |
︙ | ︙ | |||
262 263 264 265 266 267 268 269 270 271 272 273 274 275 | }] {SQLITE_DONE}] } foreach bReopen {0 1} { do_test 3.$bReopen.1.0 { reset_db execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b); CREATE TABLE t2(a INTEGER PRIMARY KEY, b); CREATE TABLE t3(a INTEGER PRIMARY KEY, b); CREATE TABLE t4(a INTEGER PRIMARY KEY, b); } create_db_file rbu.db { CREATE TABLE data_t1(a, b, rbu_control); | > | 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 | }] {SQLITE_DONE}] } foreach bReopen {0 1} { do_test 3.$bReopen.1.0 { reset_db execsql { PRAGMA page_size = 4096; CREATE TABLE t1(a INTEGER PRIMARY KEY, b); CREATE TABLE t2(a INTEGER PRIMARY KEY, b); CREATE TABLE t3(a INTEGER PRIMARY KEY, b); CREATE TABLE t4(a INTEGER PRIMARY KEY, b); } create_db_file rbu.db { CREATE TABLE data_t1(a, b, rbu_control); |
︙ | ︙ |
Changes to ext/rbu/sqlite3rbu.c.
︙ | ︙ | |||
352 353 354 355 356 357 358 359 360 361 362 363 364 365 | int rc; /* Value returned by last rbu_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ int nStep; /* Rows processed for current object */ int nProgress; /* Rows processed for all objects */ RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ i64 iOalSz; i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding ** function rbuSetupCheckpoint() for details. */ u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ | > | 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 | int rc; /* Value returned by last rbu_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ int nStep; /* Rows processed for current object */ int nProgress; /* Rows processed for all objects */ RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding ** function rbuSetupCheckpoint() for details. */ u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ |
︙ | ︙ | |||
2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 | p->iWalCksum = rbuShmChecksum(p); } if( p->rc==SQLITE_OK ){ if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ p->rc = SQLITE_DONE; p->eStage = RBU_STAGE_DONE; } } } /* ** Called when iAmt bytes are read from offset iOff of the wal file while ** the rbu object is in capture mode. Record the frame number of the frame | > > > > > > > > > > > > > > > > > | 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 | p->iWalCksum = rbuShmChecksum(p); } if( p->rc==SQLITE_OK ){ if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ p->rc = SQLITE_DONE; p->eStage = RBU_STAGE_DONE; }else{ int nSectorSize; sqlite3_file *pDb = p->pTargetFd->pReal; sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; assert( p->nPagePerSector==0 ); nSectorSize = pDb->pMethods->xSectorSize(pDb); if( nSectorSize>p->pgsz ){ p->nPagePerSector = nSectorSize / p->pgsz; }else{ p->nPagePerSector = 1; } /* Call xSync() on the wal file. This causes SQLite to sync the ** directory in which the target database and the wal file reside, in ** case it has not been synced since the rename() call in ** rbuMoveOalFile(). */ p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); } } } /* ** Called when iAmt bytes are read from offset iOff of the wal file while ** the rbu object is in capture mode. Record the frame number of the frame |
︙ | ︙ | |||
3271 3272 3273 3274 3275 3276 3277 | } if( p->rc==SQLITE_OK ){ p->eStage = RBU_STAGE_DONE; p->rc = SQLITE_DONE; } }else{ | > > > > > > > > > > > > | > | | > > > > | 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 | } if( p->rc==SQLITE_OK ){ p->eStage = RBU_STAGE_DONE; p->rc = SQLITE_DONE; } }else{ /* At one point the following block copied a single frame from the ** wal file to the database file. So that one call to sqlite3rbu_step() ** checkpointed a single frame. ** ** However, if the sector-size is larger than the page-size, and the ** application calls sqlite3rbu_savestate() or close() immediately ** after this step, then rbu_step() again, then a power failure occurs, ** then the database page written here may be damaged. Work around ** this by checkpointing frames until the next page in the aFrame[] ** lies on a different disk sector to the current one. */ u32 iSector; do{ RbuFrame *pFrame = &p->aFrame[p->nStep]; iSector = (pFrame->iDbPage-1) / p->nPagePerSector; rbuCheckpointFrame(p, pFrame); p->nStep++; }while( p->nStep<p->nFrame && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector) && p->rc==SQLITE_OK ); } p->nProgress++; } break; } default: |
︙ | ︙ | |||
3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 | int rc; if( p ){ /* Commit the transaction to the *-oal file. */ if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg); } rbuSaveState(p, p->eStage); if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg); } | > > > > > > | 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 | int rc; if( p ){ /* Commit the transaction to the *-oal file. */ if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg); } /* Sync the db file if currently doing an incremental checkpoint */ if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ sqlite3_file *pDb = p->pTargetFd->pReal; p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); } rbuSaveState(p, p->eStage); if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg); } |
︙ | ︙ | |||
3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 | if( rc==SQLITE_DONE ) return SQLITE_OK; assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); if( p->eStage==RBU_STAGE_OAL ){ assert( rc!=SQLITE_DONE ); if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); } p->rc = rc; rbuSaveState(p, p->eStage); rc = p->rc; if( p->eStage==RBU_STAGE_OAL ){ assert( rc!=SQLITE_DONE ); | > > > > > > | 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 | if( rc==SQLITE_DONE ) return SQLITE_OK; assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); if( p->eStage==RBU_STAGE_OAL ){ assert( rc!=SQLITE_DONE ); if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); } /* Sync the db file */ if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ sqlite3_file *pDb = p->pTargetFd->pReal; rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); } p->rc = rc; rbuSaveState(p, p->eStage); rc = p->rc; if( p->eStage==RBU_STAGE_OAL ){ assert( rc!=SQLITE_DONE ); |
︙ | ︙ |
Changes to src/test6.c.
︙ | ︙ | |||
311 312 313 314 315 316 317 | u8 *zGarbage; int iFirst = (int)(pWrite->iOffset/g.iSectorSize); int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize); assert(pWrite->zBuf); #ifdef TRACE_CRASHTEST | | > | | 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 | u8 *zGarbage; int iFirst = (int)(pWrite->iOffset/g.iSectorSize); int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize); assert(pWrite->zBuf); #ifdef TRACE_CRASHTEST printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", 1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize, pWrite->iOffset, iFirst, pWrite->pFile->zName ); #endif zGarbage = crash_malloc(g.iSectorSize); if( zGarbage ){ sqlite3_int64 i; for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){ |
︙ | ︙ | |||
823 824 825 826 827 828 829 | } writeListSync(0, 1); assert( 0 ); return TCL_OK; } /* | | > | 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 | } writeListSync(0, 1); assert( 0 ); return TCL_OK; } /* ** tclcmd: sqlite_crash_enable ENABLE ?DEFAULT? ** ** Parameter ENABLE must be a boolean value. If true, then the "crash" ** vfs is added to the system. If false, it is removed. */ static int SQLITE_TCLAPI crashEnableCmd( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ int isEnable; int isDefault = 0; static sqlite3_vfs crashVfs = { 2, /* iVersion */ 0, /* szOsFile */ 0, /* mxPathname */ 0, /* pNext */ "crash", /* zName */ 0, /* pAppData */ |
︙ | ︙ | |||
858 859 860 861 862 863 864 | cfRandomness, /* xRandomness */ cfSleep, /* xSleep */ cfCurrentTime, /* xCurrentTime */ cfGetLastError, /* xGetLastError */ 0, /* xCurrentTimeInt64 */ }; | | | > > > | | 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 | cfRandomness, /* xRandomness */ cfSleep, /* xSleep */ cfCurrentTime, /* xCurrentTime */ cfGetLastError, /* xGetLastError */ 0, /* xCurrentTimeInt64 */ }; if( objc!=2 && objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "ENABLE ?DEFAULT?"); return TCL_ERROR; } if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){ return TCL_ERROR; } if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){ return TCL_ERROR; } if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){ return TCL_OK; } if( crashVfs.pAppData==0 ){ sqlite3_vfs *pOriginalVfs = sqlite3_vfs_find(0); crashVfs.mxPathname = pOriginalVfs->mxPathname; crashVfs.pAppData = (void *)pOriginalVfs; crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile; sqlite3_vfs_register(&crashVfs, isDefault); }else{ crashVfs.pAppData = 0; sqlite3_vfs_unregister(&crashVfs); } return TCL_OK; } |
︙ | ︙ |
Changes to test/tester.tcl.
︙ | ︙ | |||
1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 | set blocksize "" set crashdelay 1 set prngseed 0 set opendb { sqlite3 db test.db -vfs crash } set tclbody {} set crashfile "" set dc "" set sql [lindex $args end] for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { set z [lindex $args $ii] set n [string length $z] set z2 [lindex $args [expr $ii+1]] if {$n>1 && [string first $z -delay]==0} {set crashdelay $z2} \ elseif {$n>1 && [string first $z -opendb]==0} {set opendb $z2} \ elseif {$n>1 && [string first $z -seed]==0} {set prngseed $z2} \ elseif {$n>1 && [string first $z -file]==0} {set crashfile $z2} \ elseif {$n>1 && [string first $z -tclbody]==0} {set tclbody $z2} \ elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \ | > | > | | 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 | set blocksize "" set crashdelay 1 set prngseed 0 set opendb { sqlite3 db test.db -vfs crash } set tclbody {} set crashfile "" set dc "" set dfltvfs 0 set sql [lindex $args end] for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { set z [lindex $args $ii] set n [string length $z] set z2 [lindex $args [expr $ii+1]] if {$n>1 && [string first $z -delay]==0} {set crashdelay $z2} \ elseif {$n>1 && [string first $z -opendb]==0} {set opendb $z2} \ elseif {$n>1 && [string first $z -seed]==0} {set prngseed $z2} \ elseif {$n>1 && [string first $z -file]==0} {set crashfile $z2} \ elseif {$n>1 && [string first $z -tclbody]==0} {set tclbody $z2} \ elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \ elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" }\ elseif {$n>1 && [string first $z -dfltvfs]==0} {set dfltvfs $z2 }\ else { error "Unrecognized option: $z" } } if {$crashfile eq ""} { error "Compulsory option -file missing" } # $crashfile gets compared to the native filename in # cfSync(), which can be different then what TCL uses by # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] set f [open crash.tcl w] puts $f "sqlite3_crash_enable 1 $dfltvfs" puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile" puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte" # This block sets the cache size of the main database to 10 # pages. This is done in case the build is configured to omit # "PRAGMA cache_size". if {$opendb!=""} { |
︙ | ︙ |