/ Check-in [97914266]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem preventing resumption of RBU operations after recovering from a process or system failure that occurs during the incremental-checkpoint phase.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 97914266cb4ec63b0c9185ab139673139bd2f0ed
User & Date: dan 2017-01-13 18:24:37
Context
2017-01-16
18:10
Back out check-in [0b3174e0b1364c] and replace it with a better fix for \ticket [91e2e8ba6ff2e2] - a fix that does not cause the problem identified by ticket [7ffd1ca1d2ad4ec]. Add new test cases for both tickets. check-in: 9b64af7b user: drh tags: trunk
16:01
Add test cases for tickets [91e2e8ba6ff2e2] and [7ffd1ca1d2ad4ec]. check-in: 9d0dfe0b user: drh tags: automatic-index-affinity
11:54
An example showing how to improve performance of sqlite3VdbeSerialPut() using the GCC intrinsic function __builtin_bswap64(). Leaf check-in: e42ed9b4 user: drh tags: builtin-bswap64
2017-01-13
22:21
Merge updates from trunk. check-in: 8b42b8e3 user: mistachkin tags: winSectorSize
18:24
Fix a problem preventing resumption of RBU operations after recovering from a process or system failure that occurs during the incremental-checkpoint phase. check-in: 97914266 user: dan tags: trunk
12:53
Fix the build for SQLITE_ENABLE_MEMORY_MANAGEMENT. check-in: 8c85b8fd user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added ext/rbu/rburesume.test.

            1  +# 2017 January 13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file contains tests for resumption of RBU operations in the
           13  +# case where the previous RBU process crashed.
           14  +#
           15  +
           16  +source [file join [file dirname [info script]] rbu_common.tcl]
           17  +set ::testprefix rburesume
           18  +
           19  +forcedelete test.db-shm test.db-oal
           20  +do_execsql_test 1.0 {
           21  +  CREATE TABLE t1(a PRIMARY KEY, b, c);
           22  +  CREATE INDEX t1a ON t1(a);
           23  +  CREATE INDEX t1b ON t1(b);
           24  +  CREATE INDEX t1c ON t1(c);
           25  +  WITH s(i) AS (
           26  +    VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50
           27  +  )
           28  +  INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s;
           29  +}
           30  +db_save_and_close
           31  +
           32  +do_test 1.1 {
           33  +  list [file exists test.db] \
           34  +       [file exists test.db-wal] \
           35  +       [file exists test.db-shm] \
           36  +       [file exists test.db-oal]
           37  +} {1 0 0 0}
           38  +
           39  +# Each iteration of the following loop:
           40  +#
           41  +#   1. Restores the db to the state it was in following test case 1.0
           42  +#   2. Opens an RBU vacuum and steps it $n times.
           43  +#   3. Closes the RBU vacuum handled opened in (2).
           44  +#   4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. 
           45  +#
           46  +# The loop runs until $n is large enough that step (2) vacuums the entire
           47  +# database.
           48  +#
           49  +for {set n 1} {$n < 5000} {incr n} {
           50  +  db_restore
           51  +  forcedelete state.db
           52  +  sqlite3rbu_vacuum rbu test.db state.db
           53  +  for {set i 0} {$i<$n} {incr i} {
           54  +    set rc [rbu step]
           55  +    if {$rc == "SQLITE_DONE"} break
           56  +  }
           57  +  rbu close
           58  +  if {$rc == "SQLITE_DONE"} break
           59  +
           60  +  do_test 1.2.$n.1 {
           61  +    sqlite3rbu_vacuum rbu test.db state.db
           62  +    while {[rbu step]=="SQLITE_OK"} {}
           63  +    rbu close
           64  +  } {SQLITE_DONE}
           65  +
           66  +  do_test 1.2.$n.2 {
           67  +    sqlite3 db2 test.db
           68  +    db2 eval { 
           69  +      SELECT count(*) FROM t1;
           70  +      PRAGMA integrity_check;
           71  +    }
           72  +  } {50 ok}
           73  +  db2 close
           74  +}
           75  +
           76  +# Each iteration of this loop:
           77  +#
           78  +#   1. Restores the db to the state it was in following test case 1.0
           79  +#   2. Opens an RBU vacuum and steps it $n times.
           80  +#   3. Takes a copy of all database files and the state db.
           81  +#   4. Opens a second RBU vacuum handle on the copy, resumes and completes the
           82  +#      vacuum op. 
           83  +#
           84  +# The loop runs until $n is large enough that step (2) vacuums the entire
           85  +# database.
           86  +#
           87  +for {set n 1} {$n < 5000} {incr n} {
           88  +  db_restore
           89  +  forcedelete state.db state.db-shm state.db-oal state.db-wal
           90  +  sqlite3rbu_vacuum rbu test.db state.db
           91  +  for {set i 0} {$i<$n} {incr i} {
           92  +    set rc [rbu step]
           93  +    if {$rc == "SQLITE_DONE"} break
           94  +  }
           95  +  if {$rc == "SQLITE_DONE"} {
           96  +    rbu close
           97  +    break
           98  +  }
           99  +
          100  +  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
          101  +    set f2 [string map [list test.db test.db2] $f]
          102  +    if {[file exists $f]} {
          103  +      forcecopy $f $f2
          104  +    } else {
          105  +      forcedelete $f2
          106  +    }
          107  +  }
          108  +  forcecopy state.db state.db2
          109  +  rbu close
          110  +
          111  +  do_test 1.3.$n.1 {
          112  +    sqlite3rbu_vacuum rbu test.db2 state.db2
          113  +    while {[rbu step]=="SQLITE_OK"} {}
          114  +    rbu close
          115  +  } {SQLITE_DONE}
          116  +
          117  +  do_test 1.3.$n.2 {
          118  +    sqlite3 db2 test.db2
          119  +    db2 eval { 
          120  +      SELECT count(*) FROM t1;
          121  +      PRAGMA integrity_check;
          122  +    }
          123  +  } {50 ok}
          124  +  db2 close
          125  +}
          126  +
          127  +# Each iteration of this loop:
          128  +#
          129  +#   1. Restores the db to the state it was in following test case 1.0
          130  +#   2. Opens an RBU vacuum and steps it 10 times. Then closes it.
          131  +#   2. Opens an RBU vacuum and steps it $n times.
          132  +#   3. Takes a copy of all database files and the state db.
          133  +#   4. Opens a second RBU vacuum handle on the copy, resumes and completes the
          134  +#      vacuum op. 
          135  +#
          136  +# The loop runs until $n is large enough that step (3) vacuums the entire
          137  +# database.
          138  +#
          139  +for {set n 1} {$n < 5000} {incr n} {
          140  +  db_restore
          141  +  forcedelete state.db state.db-shm state.db-oal state.db-wal
          142  +
          143  +  sqlite3rbu_vacuum rbu test.db state.db
          144  +  for {set i 0} {$i<10} {incr i} {
          145  +    rbu step
          146  +  }
          147  +  rbu close
          148  +
          149  +  sqlite3rbu_vacuum rbu test.db state.db
          150  +  for {set i 0} {$i<$n} {incr i} {
          151  +    set rc [rbu step]
          152  +    if {$rc == "SQLITE_DONE"} break
          153  +  }
          154  +  if {$rc == "SQLITE_DONE"} {
          155  +    rbu close
          156  +    break
          157  +  }
          158  +
          159  +  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
          160  +    set f2 [string map [list test.db test.db2] $f]
          161  +    if {[file exists $f]} {
          162  +      forcecopy $f $f2
          163  +    } else {
          164  +      forcedelete $f2
          165  +    }
          166  +  }
          167  +  forcecopy state.db state.db2
          168  +  rbu close
          169  +
          170  +  do_test 1.4.$n.1 {
          171  +    sqlite3rbu_vacuum rbu test.db2 state.db2
          172  +    while {[rbu step]=="SQLITE_OK"} {}
          173  +    rbu close
          174  +  } {SQLITE_DONE}
          175  +
          176  +  do_test 1.4.$n.2 {
          177  +    sqlite3 db2 test.db2
          178  +    db2 eval { 
          179  +      SELECT count(*) FROM t1;
          180  +      PRAGMA integrity_check;
          181  +    }
          182  +  } {50 ok}
          183  +  db2 close
          184  +}
          185  +
          186  +forcedelete rbu.db
          187  +do_test 2.0 {
          188  +  sqlite3 db2 rbu.db
          189  +  db2 eval {
          190  +    CREATE TABLE data_t1(a, b, c, rbu_control);
          191  +    WITH s(i) AS (
          192  +        VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10
          193  +    )
          194  +    INSERT INTO data_t1 
          195  +      SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s;
          196  +  }
          197  +  db2 close
          198  +} {}
          199  +
          200  +# Each iteration of this loop:
          201  +#
          202  +#   1. Restores the db to the state it was in following test case 1.0
          203  +#   2. Opens an RBU handle to apply the RBU update created in test case 2.0.
          204  +#   3. Steps the RBU handle $n times.
          205  +#   4. Takes a copy of all database files and the state db.
          206  +#   5. Opens a second RBU handle on the copy, resumes and completes the
          207  +#      RBU op. Checks it worked as expected.
          208  +#
          209  +# The loop runs until $n is large enough that step (3) applies the entire
          210  +# update.
          211  +#
          212  +for {set n 1} {$n < 5000} {incr n} {
          213  +  db_restore
          214  +  forcedelete state.db state.db-shm state.db-oal state.db-wal
          215  +  sqlite3rbu rbu test.db rbu.db state.db
          216  +
          217  +  for {set i 0} {$i<$n} {incr i} {
          218  +    set rc [rbu step]
          219  +    if {$rc == "SQLITE_DONE"} break
          220  +  }
          221  +  if {$rc == "SQLITE_DONE"} {
          222  +    rbu close
          223  +    break
          224  +  }
          225  +
          226  +  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
          227  +    set f2 [string map [list test.db test.db2] $f]
          228  +    if {[file exists $f]} {
          229  +      forcecopy $f $f2
          230  +    } else {
          231  +      forcedelete $f2
          232  +    }
          233  +  }
          234  +  forcecopy state.db state.db2
          235  +  rbu close
          236  +
          237  +  do_test 2.$n.1 {
          238  +    sqlite3rbu rbu test.db2 rbu.db state.db2
          239  +    while {[rbu step]=="SQLITE_OK"} {}
          240  +    rbu close
          241  +  } {SQLITE_DONE}
          242  +
          243  +  do_test 2.$n.2 {
          244  +    sqlite3 db2 test.db2
          245  +    db2 eval { 
          246  +      SELECT count(*) FROM t1;
          247  +      PRAGMA integrity_check;
          248  +    }
          249  +  } {60 ok}
          250  +  db2 close
          251  +}
          252  +
          253  +finish_test
          254  +

Changes to ext/rbu/sqlite3rbu.c.

  2404   2404       rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
  2405   2405       if( rc!=SQLITE_NOTFOUND ) p->rc = rc;
  2406   2406       if( p->eStage>=RBU_STAGE_MOVE ){
  2407   2407         bOpen = 1;
  2408   2408       }else{
  2409   2409         RbuState *pState = rbuLoadState(p);
  2410   2410         if( pState ){
  2411         -        bOpen = (pState->eStage>RBU_STAGE_MOVE);
         2411  +        bOpen = (pState->eStage>=RBU_STAGE_MOVE);
  2412   2412           rbuFreeState(pState);
  2413   2413         }
  2414   2414       }
  2415   2415       if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
  2416   2416     }
  2417   2417   
  2418   2418     p->eStage = 0;