/ Check-in [cc25cfa0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid dropping the checkpoint lock after a recovery run as a precursor to a checkpoint operation.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal-incr-ckpt
Files: files | file ages | folders
SHA1: cc25cfa04630a43c1de26f2dbdacbe46c110a2b5
User & Date: dan 2010-05-31 16:41:54
Context
2010-05-31
16:56
Fix an inconsistent #ifdef in wal.c. Fix os_unix.c so that it does not allow moving an SHM lock directly exclusive to shared without going through unlocked. check-in: 552658da user: drh tags: wal-incr-ckpt
16:41
Avoid dropping the checkpoint lock after a recovery run as a precursor to a checkpoint operation. check-in: cc25cfa0 user: dan tags: wal-incr-ckpt
16:17
Zero the checkpoint header as the last step of successful WAL recovery. Avoid an unnecessary lock/unlock in WalBeginReadTransaction. check-in: db3509c5 user: dan tags: wal-incr-ckpt
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
...
949
950
951
952
953
954
955


956












957
958
959
960
961
962
963
964
....
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;                    /* Counter incremented each transaction */
  u16 bigEndCksum;                /* True if checksums in WAL are big-endian */
  u16 szPage;                     /* Database page size in bytes */
    u32 mxFrame;                    /* Index of last valid frame in the WAL */
  u32 nPage;                      /* Size of database in pages */
  u32 aFrameCksum[2];             /* Checksum of last frame in log */
  u32 aSalt[2];                   /* Two salt values copied from WAL header */
  u32 aCksum[2];                  /* Checksum over all prior fields */
};

/*
................................................................................
** that this thread is running recovery.  If unable to establish
** the necessary locks, this routine returns SQLITE_BUSY.
*/
static int walIndexRecover(Wal *pWal){
  int rc;                         /* Return Code */
  i64 nSize;                      /* Size of log file */
  u32 aFrameCksum[2] = {0, 0};















  rc = walLockExclusive(pWal, WAL_ALL_BUT_WRITE, SQLITE_SHM_NLOCK-1);
  if( rc ){
    return rc;
  }
  WALTRACE(("WAL%p: recovery begin...\n", pWal));

  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));

................................................................................
    ** checkpointers.
    */
    memset((void *)walCkptInfo(pWal), 0, sizeof(WalCkptInfo));
  }

recovery_error:
  WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok"));
  walUnlockExclusive(pWal, WAL_ALL_BUT_WRITE, SQLITE_SHM_NLOCK-1);
  return rc;
}

/*
** Close an open wal-index.
*/
static void walIndexClose(Wal *pWal, int isDelete){







|







 







>
>

>
>
>
>
>
>
>
>
>
>
>
>
|







 







|







260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
...
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
....
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;                    /* Counter incremented each transaction */
  u16 bigEndCksum;                /* True if checksums in WAL are big-endian */
  u16 szPage;                     /* Database page size in bytes */
  u32 mxFrame;                    /* Index of last valid frame in the WAL */
  u32 nPage;                      /* Size of database in pages */
  u32 aFrameCksum[2];             /* Checksum of last frame in log */
  u32 aSalt[2];                   /* Two salt values copied from WAL header */
  u32 aCksum[2];                  /* Checksum over all prior fields */
};

/*
................................................................................
** that this thread is running recovery.  If unable to establish
** the necessary locks, this routine returns SQLITE_BUSY.
*/
static int walIndexRecover(Wal *pWal){
  int rc;                         /* Return Code */
  i64 nSize;                      /* Size of log file */
  u32 aFrameCksum[2] = {0, 0};
  int iLock;                      /* Lock offset to lock for checkpoint */
  int nLock;                      /* Number of locks to hold */

  /* Obtain an exclusive lock on all byte in the locking range not already
  ** locked by the caller. The caller is guaranteed to have locked the
  ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte.
  ** If successful, the same bytes that are locked here are unlocked before
  ** this function returns.
  */
  assert( pWal->ckptLock==1 || pWal->ckptLock==0 );
  assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 );
  assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE );
  assert( pWal->writeLock );
  iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock;
  nLock = SQLITE_SHM_NLOCK - iLock;
  rc = walLockExclusive(pWal, iLock, nLock);
  if( rc ){
    return rc;
  }
  WALTRACE(("WAL%p: recovery begin...\n", pWal));

  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));

................................................................................
    ** checkpointers.
    */
    memset((void *)walCkptInfo(pWal), 0, sizeof(WalCkptInfo));
  }

recovery_error:
  WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok"));
  walUnlockExclusive(pWal, iLock, nLock);
  return rc;
}

/*
** Close an open wal-index.
*/
static void walIndexClose(Wal *pWal, int isDelete){

Changes to test/wal2.test.

17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

32
33
34

35
36
37
38
39
40
41
..
88
89
90
91
92
93
94









95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
...
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
...
146
147
148
149
150
151
152





153
154
155
156
157
158
159
...
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
...
241
242
243
244
245
246
247


248
249
250
251
252
253
254
...
310
311
312
313
314
315
316


317
318
319
320
321
322
323
...
344
345
346
347
348
349
350




351
352
353
354
355
356
357
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }

proc set_tvfs_hdr {file args} {

  # Set $nHdr to the number of bytes in the wal-index header:
  set nHdr 80
  set nInt [expr {$nHdr/4}]

  if {[llength $args]>1} {
    return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
  }

  set blob [tvfs shm $file]

  if {[llength $args]} {
    set ia [lindex $args 0]
    set tail [string range $blob [expr $nHdr*2] end]

    set blob [binary format i${nInt}i${nInt}a* $ia $ia $tail]
    tvfs shm $file $blob
  }

  binary scan $blob i${nInt} ints
  return $ints
}
................................................................................
    SELECT count(a), sum(a) FROM t1;
  }
} {4 10}
do_test wal2-1.1 {
  execsql { SELECT count(a), sum(a) FROM t1 } db2
} {4 10}










foreach {tn iInsert res wal_index_hdr_mod wal_locks} {
         2    5   {5 15}    0             {READ RECOVER READ UNLOCK}
         3    6   {6 21}    1             {READ RECOVER READ UNLOCK}
         4    7   {7 28}    2             {READ RECOVER READ UNLOCK}
         5    8   {8 36}    3             {READ RECOVER READ UNLOCK}
         6    9   {9 45}    4             {READ RECOVER READ UNLOCK}
         7   10   {10 55}   5             {READ RECOVER READ UNLOCK}
         8   11   {11 66}   6             {READ RECOVER READ UNLOCK}
         9   12   {12 78}   7             {READ RECOVER READ UNLOCK}
        10   13   {13 91}   8             {READ RECOVER READ UNLOCK}
        11   14   {14 105}  9             {READ RECOVER READ UNLOCK}
        12   15   {15 120}  -1            {READ UNLOCK}
} {

  do_test wal2-1.$tn.1 {
    execsql { INSERT INTO t1 VALUES($iInsert) }

    set ::locks [list]
    set ::cb_done 0

................................................................................
    proc tvfs_cb {method args} {
      if {$::cb_done == 0 && $method == "xShmGet"} {
        set ::cb_done 1
        if {$::wal_index_hdr_mod >= 0} {
          incr_tvfs_hdr [lindex $args 0] $::wal_index_hdr_mod 1
        }
      }

      if {$method == "xShmLock"} { lappend ::locks [lindex $args 2] }
      return SQLITE_OK
    }

    execsql { SELECT count(a), sum(a) FROM t1 } db2
  } $res

................................................................................
# it simply drops back to a READ lock and proceeds. But because the
# header is out-of-date, the reader reads the out-of-date snapshot.
#
# After this, the header is corrupted again and the reader is allowed
# to run recovery. This time, it sees an up-to-date snapshot of the
# database file.
#





do_test wal2-2.0 {

  testvfs tvfs tvfs_cb
  proc tvfs_cb {method args} {
    if {$method == "xShmOpen"} { set ::shm_file [lindex $args 0] }
    return SQLITE_OK
  }
................................................................................
        if {$::wal_index_hdr_mod >= 0} {
          incr_tvfs_hdr $::shm_file $::wal_index_hdr_mod 1
        }
      }
      if {$method == "xShmLock"} {
        set lock [lindex $args 2]
        lappend ::locks $lock
        if {$lock == "RECOVER"} {
          set_tvfs_hdr $::shm_file $::oldhdr
        }
      }
      return SQLITE_OK
    }

    execsql { SELECT count(a), sum(a) FROM t1 } db2
  } $res0

  do_test wal2-2.$tn.3 {
    set ::locks
  } {READ RECOVER READ UNLOCK}

  do_test wal2-2.$tn.4 {
    set ::locks [list]
    set ::cb_done 0
    proc tvfs_cb {method args} {
      if {$::cb_done == 0 && $method == "xShmGet"} {
        set ::cb_done 1
................................................................................
  } $res1
}
db close
db2 close
tvfs delete
file delete -force test.db test.db-wal test.db-journal



#-------------------------------------------------------------------------
# This test case - wal2-3.* - tests the response of the library to an
# SQLITE_BUSY when attempting to obtain a READ or RECOVER lock.
#
#   wal2-3.0 - 2: SQLITE_BUSY when obtaining a READ lock
#   wal2-3.3 - 6: SQLITE_BUSY when obtaining a RECOVER lock
#
................................................................................
} {4 10}
do_test wal2-3.5 {
  list [info exists ::sabotage] [info exists ::locked]
} {0 0}
db close
tvfs delete
file delete -force test.db test.db-wal test.db-journal



#-------------------------------------------------------------------------
# Test that a database connection using a VFS that does not support the
# xShmXXX interfaces cannot open a WAL database.
#
do_test wal2-4.1 {
  sqlite3 db test.db
................................................................................
} {0 {{need xShmOpen to see this}}}
db close
tvfs delete

#-------------------------------------------------------------------------
# Test that if a database connection is forced to run recovery before it
# can perform a checkpoint, it does not transition into RECOVER state.




#
do_test wal2-5.1 {
  proc tvfs_cb {method args} {
    set ::shm_file [lindex $args 0]
    if {$method == "xShmLock"} { lappend ::locks [lindex $args 2] }
    return $::tvfs_cb_return
  }







|







>


<
>







 







>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
|







 







<







 







>
>
>
>
>







 







|











|







 







>
>







 







>
>







 







>
>
>
>







17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

35
36
37
38
39
40
41
42
..
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
...
125
126
127
128
129
130
131

132
133
134
135
136
137
138
...
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
...
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
...
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
...
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
...
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }

proc set_tvfs_hdr {file args} {

  # Set $nHdr to the number of bytes in the wal-index header:
  set nHdr 40
  set nInt [expr {$nHdr/4}]

  if {[llength $args]>1} {
    return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"}
  }

  set blob [tvfs shm $file]

  if {[llength $args]} {
    set ia [lindex $args 0]

    binary scan $blob a[expr $nHdr*2]a* dummy tail
    set blob [binary format i${nInt}i${nInt}a* $ia $ia $tail]
    tvfs shm $file $blob
  }

  binary scan $blob i${nInt} ints
  return $ints
}
................................................................................
    SELECT count(a), sum(a) FROM t1;
  }
} {4 10}
do_test wal2-1.1 {
  execsql { SELECT count(a), sum(a) FROM t1 } db2
} {4 10}

set RECOVER [list                                      \
  {0 1 lock exclusive}   {1 7 lock exclusive}          \
  {1 7 unlock exclusive} {0 1 unlock exclusive}        \
]
set READ [list                                         \
  {4 1 lock exclusive} {4 1 unlock exclusive}          \
  {4 1 lock shared}    {4 1 unlock shared}             \
]

foreach {tn iInsert res wal_index_hdr_mod wal_locks} "
         2    5   {5 15}    0             {$RECOVER $READ}
         3    6   {6 21}    1             {$RECOVER $READ}
         4    7   {7 28}    2             {$RECOVER $READ}
         5    8   {8 36}    3             {$RECOVER $READ}
         6    9   {9 45}    4             {$RECOVER $READ}
         7   10   {10 55}   5             {$RECOVER $READ}
         8   11   {11 66}   6             {$RECOVER $READ}
         9   12   {12 78}   7             {$RECOVER $READ}
        10   13   {13 91}   8             {$RECOVER $READ}
        11   14   {14 105}  9             {$RECOVER $READ}
        12   15   {15 120}  -1            {$READ}
" {

  do_test wal2-1.$tn.1 {
    execsql { INSERT INTO t1 VALUES($iInsert) }

    set ::locks [list]
    set ::cb_done 0

................................................................................
    proc tvfs_cb {method args} {
      if {$::cb_done == 0 && $method == "xShmGet"} {
        set ::cb_done 1
        if {$::wal_index_hdr_mod >= 0} {
          incr_tvfs_hdr [lindex $args 0] $::wal_index_hdr_mod 1
        }
      }

      if {$method == "xShmLock"} { lappend ::locks [lindex $args 2] }
      return SQLITE_OK
    }

    execsql { SELECT count(a), sum(a) FROM t1 } db2
  } $res

................................................................................
# it simply drops back to a READ lock and proceeds. But because the
# header is out-of-date, the reader reads the out-of-date snapshot.
#
# After this, the header is corrupted again and the reader is allowed
# to run recovery. This time, it sees an up-to-date snapshot of the
# database file.
#
set WRITER [list 0 1 lock exclusive]
set LOCKS  [list \
  {0 1 lock exclusive} {0 1 unlock exclusive} \
  {4 1 lock shared}    {4 1 unlock shared}    \
]
do_test wal2-2.0 {

  testvfs tvfs tvfs_cb
  proc tvfs_cb {method args} {
    if {$method == "xShmOpen"} { set ::shm_file [lindex $args 0] }
    return SQLITE_OK
  }
................................................................................
        if {$::wal_index_hdr_mod >= 0} {
          incr_tvfs_hdr $::shm_file $::wal_index_hdr_mod 1
        }
      }
      if {$method == "xShmLock"} {
        set lock [lindex $args 2]
        lappend ::locks $lock
        if {$lock == $::WRITER} {
          set_tvfs_hdr $::shm_file $::oldhdr
        }
      }
      return SQLITE_OK
    }

    execsql { SELECT count(a), sum(a) FROM t1 } db2
  } $res0

  do_test wal2-2.$tn.3 {
    set ::locks
  } $LOCKS

  do_test wal2-2.$tn.4 {
    set ::locks [list]
    set ::cb_done 0
    proc tvfs_cb {method args} {
      if {$::cb_done == 0 && $method == "xShmGet"} {
        set ::cb_done 1
................................................................................
  } $res1
}
db close
db2 close
tvfs delete
file delete -force test.db test.db-wal test.db-journal


if 0 {
#-------------------------------------------------------------------------
# This test case - wal2-3.* - tests the response of the library to an
# SQLITE_BUSY when attempting to obtain a READ or RECOVER lock.
#
#   wal2-3.0 - 2: SQLITE_BUSY when obtaining a READ lock
#   wal2-3.3 - 6: SQLITE_BUSY when obtaining a RECOVER lock
#
................................................................................
} {4 10}
do_test wal2-3.5 {
  list [info exists ::sabotage] [info exists ::locked]
} {0 0}
db close
tvfs delete
file delete -force test.db test.db-wal test.db-journal

}

#-------------------------------------------------------------------------
# Test that a database connection using a VFS that does not support the
# xShmXXX interfaces cannot open a WAL database.
#
do_test wal2-4.1 {
  sqlite3 db test.db
................................................................................
} {0 {{need xShmOpen to see this}}}
db close
tvfs delete

#-------------------------------------------------------------------------
# Test that if a database connection is forced to run recovery before it
# can perform a checkpoint, it does not transition into RECOVER state.
#
# UPDATE: This has now changed. When running a checkpoint, if recovery is
# required the client grabs all exclusive locks (just as it would for a
# recovery performed as a pre-cursor to a normal database transaction).
#
do_test wal2-5.1 {
  proc tvfs_cb {method args} {
    set ::shm_file [lindex $args 0]
    if {$method == "xShmLock"} { lappend ::locks [lindex $args 2] }
    return $::tvfs_cb_return
  }