SQLite

Check-in [02c4040ce2]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem where an SQLITE_BUSY in the checkpoint code was being treated as an IO error (abandoning, instead of just limiting, the checkpoint).
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 02c4040ce2b4c970b3dee09f7c9ad5a2a3a9aa49
User & Date: dan 2010-06-04 10:37:06.000
Context
2010-06-04
11:56
If an attempt to sync the database file as part of a checkpoint fails, do not update the shared "nBackfill" variable. Otherwise, another process could wrap the log and overwrite content before it is synced into the database. (check-in: b813233d76 user: dan tags: trunk)
10:37
Fix a problem where an SQLITE_BUSY in the checkpoint code was being treated as an IO error (abandoning, instead of just limiting, the checkpoint). (check-in: 02c4040ce2 user: dan tags: trunk)
2010-06-03
19:10
Fix another problem in test_vfs.c. (check-in: df7d59899c user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/test_vfs.c.
592
593
594
595
596
597
598
599

600
601
602
603
604
605
606
){
  int rc = SQLITE_OK;
  TestvfsFile *pFd = (TestvfsFile *)pFile;
  Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);

  if( p->pScript && p->mask&TESTVFS_SHMGET_MASK ){
    tvfsExecTcl(p, "xShmGet", 
        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0

    );
    tvfsResultCode(p, &rc);
  }
  if( rc==SQLITE_OK && p->mask&TESTVFS_SHMGET_MASK && tvfsInjectIoerr(p) ){
    rc = SQLITE_IOERR;
  }








|
>







592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
){
  int rc = SQLITE_OK;
  TestvfsFile *pFd = (TestvfsFile *)pFile;
  Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);

  if( p->pScript && p->mask&TESTVFS_SHMGET_MASK ){
    tvfsExecTcl(p, "xShmGet", 
        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 
        Tcl_NewIntObj(reqMapSize)
    );
    tvfsResultCode(p, &rc);
  }
  if( rc==SQLITE_OK && p->mask&TESTVFS_SHMGET_MASK && tvfsInjectIoerr(p) ){
    rc = SQLITE_IOERR;
  }

Changes to src/wal.c.
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
  WalIterator *p;       /* Return value */
  int nSegment;         /* Number of segments to merge */
  u32 iLast;            /* Last frame in log */
  int nByte;            /* Number of bytes to allocate */
  int i;                /* Iterator variable */
  int nFinal;           /* Number of unindexed entries */
  u8 *aTmp;             /* Temp space used by merge-sort */
  int rc;               /* Return code of walIndexMap() */
  u8 *aSpace;           /* Surplus space on the end of the allocation */

  /* Make sure the wal-index is mapped into local memory */
  rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame));
  if( rc!=SQLITE_OK ){
    return rc;
  }

  /* This routine only runs while holding SQLITE_SHM_CHECKPOINT.  No other
  ** thread is able to write to shared memory while this routine is
  ** running (or, indeed, while the WalIterator object exists).  Hence,
  ** we can cast off the volatile qualifacation from shared memory
  */
  assert( pWal->ckptLock );







<



|
<
<
<







1295
1296
1297
1298
1299
1300
1301

1302
1303
1304
1305



1306
1307
1308
1309
1310
1311
1312
  WalIterator *p;       /* Return value */
  int nSegment;         /* Number of segments to merge */
  u32 iLast;            /* Last frame in log */
  int nByte;            /* Number of bytes to allocate */
  int i;                /* Iterator variable */
  int nFinal;           /* Number of unindexed entries */
  u8 *aTmp;             /* Temp space used by merge-sort */

  u8 *aSpace;           /* Surplus space on the end of the allocation */

  /* Make sure the wal-index is mapped into local memory */
  assert( pWal->pWiData && pWal->szWIndex>=walMappingSize(pWal->hdr.mxFrame) );




  /* This routine only runs while holding SQLITE_SHM_CHECKPOINT.  No other
  ** thread is able to write to shared memory while this routine is
  ** running (or, indeed, while the WalIterator object exists).  Hence,
  ** we can cast off the volatile qualifacation from shared memory
  */
  assert( pWal->ckptLock );
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
/* 
** Free an iterator allocated by walIteratorInit().
*/
static void walIteratorFree(WalIterator *p){
  sqlite3_free(p);
}


/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
** The amount of information copies from WAL to database might be limited
** by active readers.  This routine will never overwrite a database page
** that a concurrent reader might be using.







<







1352
1353
1354
1355
1356
1357
1358

1359
1360
1361
1362
1363
1364
1365
/* 
** Free an iterator allocated by walIteratorInit().
*/
static void walIteratorFree(WalIterator *p){
  sqlite3_free(p);
}


/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
** The amount of information copies from WAL to database might be limited
** by active readers.  This routine will never overwrite a database page
** that a concurrent reader might be using.
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421

1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
  int i;                          /* Loop counter */
  volatile WalIndexHdr *pHdr;     /* The actual wal-index header in SHM */
  volatile WalCkptInfo *pInfo;    /* The checkpoint status information */

  /* Allocate the iterator */
  rc = walIteratorInit(pWal, &pIter);
  if( rc!=SQLITE_OK || pWal->hdr.mxFrame==0 ){
    walIteratorFree(pIter);
    return rc;
  }

  /*** TODO:  Move this test out to the caller.  Make it an assert() here ***/
  if( pWal->hdr.szPage!=nBuf ){
    walIteratorFree(pIter);
    return SQLITE_CORRUPT_BKPT;

  }

  /* Compute in mxSafeFrame the index of the last frame of the WAL that is
  ** safe to write into the database.  Frames beyond mxSafeFrame might
  ** overwrite database pages that are in use by active readers and thus
  ** cannot be backfilled from the WAL.
  */
  mxSafeFrame = pWal->hdr.mxFrame;
  pHdr = (volatile WalIndexHdr*)pWal->pWiData;
  pInfo = (volatile WalCkptInfo*)&pHdr[2];
  assert( pInfo==walCkptInfo(pWal) );
  for(i=1; i<WAL_NREADER; i++){
    u32 y = pInfo->aReadMark[i];
    if( y>0 && (mxSafeFrame==0 || mxSafeFrame>=y) ){
      if( y<=pWal->hdr.mxFrame
       && walLockExclusive(pWal, WAL_READ_LOCK(i), 1)==SQLITE_OK
      ){
        pInfo->aReadMark[i] = 0;
        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
      }else if( rc==SQLITE_BUSY ){
        mxSafeFrame = y-1;
      }else{
        walIteratorFree(pIter);
        return rc;
      }
    }
  }

  if( pInfo->nBackfill<mxSafeFrame
   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
  ){







|
<




<
|
>













|
|
|
|





|
<







1402
1403
1404
1405
1406
1407
1408
1409

1410
1411
1412
1413

1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438

1439
1440
1441
1442
1443
1444
1445
  int i;                          /* Loop counter */
  volatile WalIndexHdr *pHdr;     /* The actual wal-index header in SHM */
  volatile WalCkptInfo *pInfo;    /* The checkpoint status information */

  /* Allocate the iterator */
  rc = walIteratorInit(pWal, &pIter);
  if( rc!=SQLITE_OK || pWal->hdr.mxFrame==0 ){
    goto walcheckpoint_out;

  }

  /*** TODO:  Move this test out to the caller.  Make it an assert() here ***/
  if( pWal->hdr.szPage!=nBuf ){

    rc = SQLITE_CORRUPT_BKPT;
    goto walcheckpoint_out;
  }

  /* Compute in mxSafeFrame the index of the last frame of the WAL that is
  ** safe to write into the database.  Frames beyond mxSafeFrame might
  ** overwrite database pages that are in use by active readers and thus
  ** cannot be backfilled from the WAL.
  */
  mxSafeFrame = pWal->hdr.mxFrame;
  pHdr = (volatile WalIndexHdr*)pWal->pWiData;
  pInfo = (volatile WalCkptInfo*)&pHdr[2];
  assert( pInfo==walCkptInfo(pWal) );
  for(i=1; i<WAL_NREADER; i++){
    u32 y = pInfo->aReadMark[i];
    if( y>0 && mxSafeFrame>=y ){
      assert( y<=pWal->hdr.mxFrame );
      rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
      if( rc==SQLITE_OK ){
        pInfo->aReadMark[i] = 0;
        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
      }else if( rc==SQLITE_BUSY ){
        mxSafeFrame = y-1;
      }else{
        goto walcheckpoint_out;

      }
    }
  }

  if( pInfo->nBackfill<mxSafeFrame
   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
  ){
1484
1485
1486
1487
1488
1489
1490

1491
1492
1493
1494
1495
1496
1497
  }else if( rc==SQLITE_BUSY ){
    /* Reset the return code so as not to report a checkpoint failure
    ** just because active readers prevent any backfill.
    */
    rc = SQLITE_OK;
  }


  walIteratorFree(pIter);
  return rc;
}

/*
** Close a connection to a log file.
*/







>







1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
  }else if( rc==SQLITE_BUSY ){
    /* Reset the return code so as not to report a checkpoint failure
    ** just because active readers prevent any backfill.
    */
    rc = SQLITE_OK;
  }

 walcheckpoint_out:
  walIteratorFree(pIter);
  return rc;
}

/*
** Close a connection to a log file.
*/
Changes to test/wal3.test.
105
106
107
108
109
110
111



















































































112
113
  } $str
  do_test wal3-1.$i.7 {
    execsql { PRAGMA integrity_check } db2
  } {ok}
  db2 close
}




















































































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
  } $str
  do_test wal3-1.$i.7 {
    execsql { PRAGMA integrity_check } db2
  } {ok}
  db2 close
}

db close
foreach code [list {
  proc code2 {tcl} { uplevel #0 $tcl }
  proc code3 {tcl} { uplevel #0 $tcl }
  set tn singleproc
} {
  set ::code2_chan [launch_testfixture]
  set ::code3_chan [launch_testfixture]
  proc code2 {tcl} { testfixture $::code2_chan $tcl }
  proc code3 {tcl} { testfixture $::code3_chan $tcl }
  set tn multiproc
}] {
  file delete -force test.db test.db-wal test.db-journal
  sqlite3 db test.db
  eval $code

  # Open connections [db2] and [db3]. Depending on which iteration this
  # is, the connections may be created in this interpreter, or in 
  # interpreters running in other OS processes. As such, the [db2] and [db3]
  # commands should only be accessed within [code2] and [code3] blocks,
  # respectively.
  #
  code2 { sqlite3 db2 test.db ; db2 eval { PRAGMA journal_mode = WAL } }
  code3 { sqlite3 db3 test.db ; db3 eval { PRAGMA journal_mode = WAL } }

  # Shorthand commands. Execute SQL using database connection [db], [db2] 
  # or [db3]. Return the results.
  #
  proc sql  {sql} { db eval $sql }
  proc sql2 {sql} { code2 [list db2 eval $sql] }
  proc sql3 {sql} { code3 [list db3 eval $sql] }

  do_test wal3-2.$tn.1 {
    sql { 
      PRAGMA page_size = 1024;
      PRAGMA auto_vacuum = OFF; 
      PRAGMA journal_mode = WAL;
    }
    sql {
      CREATE TABLE t1(a, b);
      INSERT INTO t1 VALUES(1, 'one');
      BEGIN;
        SELECT * FROM t1;
    }
  } {1 one}
  do_test wal3-2.$tn.2 {
    sql2 {
      CREATE TABLE t2(a, b);
      INSERT INTO t2 VALUES(2, 'two');
      BEGIN;
        SELECT * FROM t2;
    }
  } {2 two}
  do_test wal3-2.$tn.3 {
    sql3 {
      CREATE TABLE t3(a, b);
      INSERT INTO t3 VALUES(3, 'three');
      BEGIN;
        SELECT * FROM t3;
    }
  } {3 three}

  # Try to checkpoint the database using [db]. It should be possible to
  # checkpoint everything except the frames added by [db3] (checkpointing
  # these frames would clobber the snapshot currently being used by [db2]).
  #
  do_test wal3-2.$tn.4 {
    sql {
      COMMIT;
      PRAGMA wal_checkpoint;
    }
  } {}
  do_test wal3-2.$tn.5 {
    file size test.db
  } [expr 3*1024]


  catch { db close }
  catch { code2 { db2 close } }
  catch { code3 { db3 close } }
  catch { close $::code2_chan }
  catch { close $::code3_chan }
}
finish_test

Changes to test/walfault.test.
12
13
14
15
16
17
18

19
20
21
22
23
24
25
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl


ifcapable !wal {finish_test ; return }

#-------------------------------------------------------------------------
# This test case, walfault-1-*, simulates faults while executing a
#
#   PRAGMA journal_mode = WAL;







>







12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
source $testdir/lock_common.tcl

ifcapable !wal {finish_test ; return }

#-------------------------------------------------------------------------
# This test case, walfault-1-*, simulates faults while executing a
#
#   PRAGMA journal_mode = WAL;
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402

  catch { db eval { ROLLBACK TO spoint } }
  catch { db eval { COMMIT } }
  set n [db one {SELECT count(*) FROM abc}]
  if {$n != 1 && $n != 2} { error "Incorrect number of rows: $n" }
}

#-------------------------------------------------------------------------
# When a database is checkpointed, SQLite does the following:
#
#   1. xShmLock(CHECKPOINT) to lock the WAL.
#   2. xShmGet(-1) to get a mapping to read the wal-index header.
#   3. If the mapping obtained in (2) is not large enough to cover the
#      entire wal-index, call xShmGet(nReq) to get a larger mapping.
#   4. Do the checkpoint.
#   5. Release the lock and mapping.
#
# This test case tests the outcome of an IO error in step 2.
#
proc walfault_10_vfs_cb {method args} {
  switch -- $::shm_state {
    0 { return SQLITE_OK }
    1 {
      if {$method == "xShmGet"} {
        set ::wal_index [tvfs shm [lindex $args 0]]
        tvfs shm [lindex $args 0] [string range $::wal_index 0 65535]
        set ::shm_state 2
      }
    }
    2 {
      if {$method == "xShmGet"} {
        tvfs shm [lindex $args 0] $::wal_index
        return SQLITE_IOERR
      }
    }
  }
  return SQLITE_OK
}
do_test walfault-10.1 {
  set ::shm_state 0
  testvfs tvfs 
  tvfs script walfault_10_vfs_cb

  sqlite3 db  test.db -vfs tvfs
  sqlite3 db2 test.db -vfs tvfs

  execsql {
    PRAGMA journal_mode = WAL;
    PRAGMA wal_autocheckpoint = 0;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(randomblob(900));
  }
} {wal 0}
do_test walfault-10.2 {
  execsql {
    PRAGMA wal_autocheckpoint = 0;
    BEGIN;
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 2 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 4 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 8 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 16 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 32 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 64 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 128 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 256 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 512 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 1024 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 2048 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 4096 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 8192 */
      INSERT INTO t1 SELECT randomblob(900) FROM t1;    /* 16384 */
    COMMIT;
  } db2
} {0}
do_test walfault-10.3 {
  set ::shm_state 1
  catchsql { PRAGMA wal_checkpoint } db2
} {1 {disk I/O error}}
set ::shm_state 0
db close
db2 close
tvfs delete
unset -nocomplain ::wal_index ::shm_state

finish_test








<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<


318
319
320
321
322
323
324













































































325
326

  catch { db eval { ROLLBACK TO spoint } }
  catch { db eval { COMMIT } }
  set n [db one {SELECT count(*) FROM abc}]
  if {$n != 1 && $n != 2} { error "Incorrect number of rows: $n" }
}














































































finish_test