/ Check-in [480d12db]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes and tests for backup of a WAL database.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: 480d12db4c0ebcc37598f7620d39193875eab15b
User & Date: dan 2010-04-23 19:15:00
Context
2010-04-24
04:49
Reserve some space at the start of the log-summary file to apply locks to. check-in: a031aa1f user: dan tags: wal
2010-04-23
19:15
Fixes and tests for backup of a WAL database. check-in: 480d12db user: dan tags: wal
11:44
Add very simple test cases for backup and VACUUM of WAL databases. More to come. check-in: 1077d813 user: dan tags: wal
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/log.c.

1563
1564
1565
1566
1567
1568
1569








1570
1571
1572
1573
1574
1575
1576
  }else if( pLog->isWriteLocked ){
    logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK);
    memcpy(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr));
    pLog->isWriteLocked = 0;
  }
  return SQLITE_OK;
}









/* 
** Write a set of frames to the log. The caller must hold at least a
** RESERVED lock on the database file.
*/
int sqlite3LogFrames(
  Log *pLog,                      /* Log handle to write to */







>
>
>
>
>
>
>
>







1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
  }else if( pLog->isWriteLocked ){
    logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK);
    memcpy(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr));
    pLog->isWriteLocked = 0;
  }
  return SQLITE_OK;
}

/* 
** Return true if data has been written but not committed to the log file. 
*/
int sqlite3LogDirty(Log *pLog){
  assert( pLog->isWriteLocked );
  return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg );
}

/* 
** Write a set of frames to the log. The caller must hold at least a
** RESERVED lock on the database file.
*/
int sqlite3LogFrames(
  Log *pLog,                      /* Log handle to write to */

Changes to src/log.h.

32
33
34
35
36
37
38



39
40
41
42
43
44
45

/* Read a page from the log, if it is present. */
int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut);
void sqlite3LogDbsize(Log *pLog, Pgno *pPgno);

/* Obtain or release the WRITER lock. */
int sqlite3LogWriteLock(Log *pLog, int op);




/* Write a frame or frames to the log. */
int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int);

/* Copy pages from the log to the database file */ 
int sqlite3LogCheckpoint(
  Log *pLog,                      /* Log connection */







>
>
>







32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

/* Read a page from the log, if it is present. */
int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut);
void sqlite3LogDbsize(Log *pLog, Pgno *pPgno);

/* Obtain or release the WRITER lock. */
int sqlite3LogWriteLock(Log *pLog, int op);

/* Return true if data has been written but not committed to the log file. */
int sqlite3LogDirty(Log *pLog);

/* Write a frame or frames to the log. */
int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int);

/* Copy pages from the log to the database file */ 
int sqlite3LogCheckpoint(
  Log *pLog,                      /* Log connection */

Changes to src/pager.c.

2230
2231
2232
2233
2234
2235
2236








2237
2238
2239













2240
2241
2242
2243
2244
2245
2246
....
3214
3215
3216
3217
3218
3219
3220



























3221
3222
3223
3224
3225
3226
3227
....
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
....
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
....
5115
5116
5117
5118
5119
5120
5121

5122
5123
5124
5125
5126
5127
5128
  IOTRACE(("PGIN %p %d\n", pPager, pgno));
  PAGERTRACE(("FETCH %d page %d hash(%08x)\n",
               PAGERID(pPager), pgno, pager_pagehash(pPg)));

  return rc;
}









static int pagerRollbackLog(Pager *pPager){
  int rc = SQLITE_OK;
  PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);













  pPager->dbSize = pPager->dbOrigSize;
  while( pList && rc==SQLITE_OK ){
    PgHdr *pNext = pList->pDirty;
    if( sqlite3PcachePageRefcount(pList)==0 ){
      sqlite3PagerLookup(pPager, pList->pgno);
      sqlite3PcacheDrop(pList);
    }else{
................................................................................
    pPager->nSubRec++;
    assert( pPager->nSavepoint>0 );
    rc = addToSavepointBitvecs(pPager, pPg->pgno);
  }
  return rc;
}





























/*
** This function is called by the pcache layer when it has reached some
** soft memory limit. The first argument is a pointer to a Pager object
** (cast as a void*). The pager is always 'purgeable' (not an in-memory
** database). The second argument is a reference to a page that is 
** currently dirty but has no outstanding references. The page
................................................................................

  assert( pPg->pPager==pPager );
  assert( pPg->flags&PGHDR_DIRTY );

  pPg->pDirty = 0;
  if( pagerUseLog(pPager) ){
    /* Write a single frame for this page to the log. */
    rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0);
  }else{
    /* The doNotSync flag is set by the sqlite3PagerWrite() function while it
    ** is journalling a set of two or more database pages that are stored
    ** on the same disk sector. Syncing the journal is not allowed while
    ** this is happening as it is important that all members of such a
    ** set of pages are synced to disk together. So, if the page this function
    ** is trying to make clean will require a journal sync and the doNotSync
................................................................................
    ** backup in progress needs to be restarted.
    */
    sqlite3BackupRestart(pPager->pBackup);
  }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){
    if( pagerUseLog(pPager) ){
      PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
      if( pList ){
        rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList,
            pPager->dbSize, 1, (pPager->fullSync ? pPager->sync_flags : 0)
        );
      }
      sqlite3PcacheCleanAll(pPager->pPCache);
    }else{
      /* The following block updates the change-counter. Exactly how it
      ** does this depends on whether or not the atomic-update optimization
      ** was enabled at compile time, and if this transaction meets the 
................................................................................
**   hot-journal rollback).
*/
int sqlite3PagerRollback(Pager *pPager){
  int rc = SQLITE_OK;                  /* Return code */
  PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager)));
  if( pagerUseLog(pPager) ){
    int rc2;

    rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1);
    rc2 = pager_end_transaction(pPager, pPager->setMaster);
    if( rc==SQLITE_OK ) rc = rc2;
  }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){
    rc = pager_end_transaction(pPager, pPager->setMaster);
  }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    if( pPager->state>=PAGER_EXCLUSIVE ){







>
>
>
>
>
>
>
>



>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







 







|
|







 







>







2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
....
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
....
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
....
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
....
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
  IOTRACE(("PGIN %p %d\n", pPager, pgno));
  PAGERTRACE(("FETCH %d page %d hash(%08x)\n",
               PAGERID(pPager), pgno, pager_pagehash(pPg)));

  return rc;
}

/*
** This function is called when a transaction on a WAL database is rolled
** back. For each dirty page in the cache, do one of the following:
**
**   * If the page has no outstanding references, simply discard it.
**   * Otherwise, if the page has one or more outstanding references, 
**     reload the original content from the database (or log file).
*/
static int pagerRollbackLog(Pager *pPager){
  int rc = SQLITE_OK;
  PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);

  /* Normally, if a transaction is rolled back, any backup processes are
  ** updated as data is copied out of the rollback journal and into the
  ** database. This is not generally possible with a WAL database, as
  ** rollback involves simply truncating the log file. Therefore, if one
  ** or more frames have already been written to the log (and therefore 
  ** also copied into the backup databases) as part of this transaction,
  ** the backups must be restarted.
  */
  if( sqlite3LogDirty(pPager->pLog) ){
    sqlite3BackupRestart(pPager->pBackup);
  }

  pPager->dbSize = pPager->dbOrigSize;
  while( pList && rc==SQLITE_OK ){
    PgHdr *pNext = pList->pDirty;
    if( sqlite3PcachePageRefcount(pList)==0 ){
      sqlite3PagerLookup(pPager, pList->pgno);
      sqlite3PcacheDrop(pList);
    }else{
................................................................................
    pPager->nSubRec++;
    assert( pPager->nSavepoint>0 );
    rc = addToSavepointBitvecs(pPager, pPg->pgno);
  }
  return rc;
}

/*
** This function is a wrapper around sqlite3LogFrames(). As well as logging
** the contents of the list of pages headed by pList (connected by pDirty),
** this function notifies any active backup processes that the pages have
** changed. 
*/ 
static int pagerLogFrames(
  Pager *pPager,                  /* Pager object */
  PgHdr *pList,                   /* List of frames to log */
  Pgno nTruncate,                 /* Database size after this commit */
  int isCommit,                   /* True if this is a commit */
  int sync_flags                  /* Flags to pass to OsSync() (or 0) */
){
  int rc;                         /* Return code */

  assert( pPager->pLog );
  rc = sqlite3LogFrames(pPager->pLog, 
      pPager->pageSize, pList, nTruncate, isCommit, sync_flags
  );
  if( rc==SQLITE_OK && pPager->pBackup ){
    PgHdr *p;
    for(p=pList; p; p=p->pDirty){
      sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData);
    }
  }
  return rc;
}

/*
** This function is called by the pcache layer when it has reached some
** soft memory limit. The first argument is a pointer to a Pager object
** (cast as a void*). The pager is always 'purgeable' (not an in-memory
** database). The second argument is a reference to a page that is 
** currently dirty but has no outstanding references. The page
................................................................................

  assert( pPg->pPager==pPager );
  assert( pPg->flags&PGHDR_DIRTY );

  pPg->pDirty = 0;
  if( pagerUseLog(pPager) ){
    /* Write a single frame for this page to the log. */
    rc = pagerLogFrames(pPager, pPg, 0, 0, 0);
  }else{
    /* The doNotSync flag is set by the sqlite3PagerWrite() function while it
    ** is journalling a set of two or more database pages that are stored
    ** on the same disk sector. Syncing the journal is not allowed while
    ** this is happening as it is important that all members of such a
    ** set of pages are synced to disk together. So, if the page this function
    ** is trying to make clean will require a journal sync and the doNotSync
................................................................................
    ** backup in progress needs to be restarted.
    */
    sqlite3BackupRestart(pPager->pBackup);
  }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){
    if( pagerUseLog(pPager) ){
      PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
      if( pList ){
        rc = pagerLogFrames(pPager, pList, pPager->dbSize, 1, 
            (pPager->fullSync ? pPager->sync_flags : 0)
        );
      }
      sqlite3PcacheCleanAll(pPager->pPCache);
    }else{
      /* The following block updates the change-counter. Exactly how it
      ** does this depends on whether or not the atomic-update optimization
      ** was enabled at compile time, and if this transaction meets the 
................................................................................
**   hot-journal rollback).
*/
int sqlite3PagerRollback(Pager *pPager){
  int rc = SQLITE_OK;                  /* Return code */
  PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager)));
  if( pagerUseLog(pPager) ){
    int rc2;

    rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1);
    rc2 = pager_end_transaction(pPager, pPager->setMaster);
    if( rc==SQLITE_OK ) rc = rc2;
  }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){
    rc = pager_end_transaction(pPager, pPager->setMaster);
  }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    if( pPager->state>=PAGER_EXCLUSIVE ){

Changes to test/backup.test.

34
35
36
37
38
39
40


41
42
43
44
45
46
47
...
901
902
903
904
905
906
907

908
909
910
911
912
913
914
#
# backup-7.*: Test SQLITE_BUSY and SQLITE_LOCKED errors.
#
# backup-8.*: Test multiple simultaneous backup operations.
#
# backup-9.*: Test that passing a negative argument to backup_step() is
#             interpreted as "copy the whole file".


#

proc data_checksum {db file} { $db one "SELECT md5sum(a, b) FROM ${file}.t1" }
proc test_contents {name db1 file1 db2 file2} {
  $db2 eval {select * from sqlite_master}
  $db1 eval {select * from sqlite_master}
  set checksum [data_checksum $db2 $file2]
................................................................................
  } {ok}

  db2 close
  db3 close
}



# Test that if the database is written to via the same database handle being
# used as the source by a backup operation:
#
#   10.1.*: If the db is in-memory, the backup is restarted.
#   10.2.*: If the db is a file, the backup is not restarted.
#
db close







>
>







 







>







34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
...
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
#
# backup-7.*: Test SQLITE_BUSY and SQLITE_LOCKED errors.
#
# backup-8.*: Test multiple simultaneous backup operations.
#
# backup-9.*: Test that passing a negative argument to backup_step() is
#             interpreted as "copy the whole file".
# 
# backup-10.*: Test writing the source database mid backup.
#

proc data_checksum {db file} { $db one "SELECT md5sum(a, b) FROM ${file}.t1" }
proc test_contents {name db1 file1 db2 file2} {
  $db2 eval {select * from sqlite_master}
  $db1 eval {select * from sqlite_master}
  set checksum [data_checksum $db2 $file2]
................................................................................
  } {ok}

  db2 close
  db3 close
}


#-----------------------------------------------------------------------
# Test that if the database is written to via the same database handle being
# used as the source by a backup operation:
#
#   10.1.*: If the db is in-memory, the backup is restarted.
#   10.2.*: If the db is a file, the backup is not restarted.
#
db close

Changes to test/walbak.test.

15
16
17
18
19
20
21






22
23
24
25
26
27
28
..
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85



























































































86


set testdir [file dirname $argv0]
source $testdir/tester.tcl

proc log_file_size {nFrame pgsz} {
  expr {12 + ($pgsz+16)*$nFrame}
}







# Make sure a simple backup from a WAL database works.
#
do_test walbak-1.0 {
  execsql { 
    PRAGMA synchronous = NORMAL;
    PRAGMA page_size = 1024;
................................................................................
do_test walbak-1.3 {
  execsql { PRAGMA integrity_check } db2
} {ok}
db2 close

# Try a VACUUM on a WAL database.
#
do_test walbak-2.1 {
  execsql { 
    VACUUM;
    PRAGMA main.journal_mode;
  }
} {wal}
do_test walbak-2.2 {
  list [file size test.db] [file size test.db-wal]
} [list 1024 [log_file_size 6 1024]]
do_test walbak-2.3 {
  execsql { PRAGMA checkpoint }
  list [file size test.db] [file size test.db-wal]
} [list [expr 3*1024] [log_file_size 6 1024]]
do_test walbak-2.4 {
  execsql { 
    CREATE TABLE t2(a, b);
    INSERT INTO t2 SELECT * FROM t1;
    DROP TABLE t1;
  }
  list [file size test.db] [file size test.db-wal]
} [list [expr 3*1024] [log_file_size 6 1024]]
do_test walbak-2.5 {
  execsql { VACUUM }
  list [file size test.db] [file size test.db-wal]
} [list [expr 3*1024] [log_file_size 8 1024]]
do_test walbak-2.6 {
  execsql { PRAGMA checkpoint }
  list [file size test.db] [file size test.db-wal]
} [list [expr 2*1024] [log_file_size 8 1024]]




























































































finish_test








>
>
>
>
>
>







 







|





|


|



|







|



|




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
..
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184

set testdir [file dirname $argv0]
source $testdir/tester.tcl

proc log_file_size {nFrame pgsz} {
  expr {12 + ($pgsz+16)*$nFrame}
}

# Test organization:
# 
#   walback-1.*: Simple tests.
#   walback-2.*: Test backups when the source db is modified mid-backup.
#

# Make sure a simple backup from a WAL database works.
#
do_test walbak-1.0 {
  execsql { 
    PRAGMA synchronous = NORMAL;
    PRAGMA page_size = 1024;
................................................................................
do_test walbak-1.3 {
  execsql { PRAGMA integrity_check } db2
} {ok}
db2 close

# Try a VACUUM on a WAL database.
#
do_test walbak-1.4 {
  execsql { 
    VACUUM;
    PRAGMA main.journal_mode;
  }
} {wal}
do_test walbak-1.5 {
  list [file size test.db] [file size test.db-wal]
} [list 1024 [log_file_size 6 1024]]
do_test walbak-1.6 {
  execsql { PRAGMA checkpoint }
  list [file size test.db] [file size test.db-wal]
} [list [expr 3*1024] [log_file_size 6 1024]]
do_test walbak-1.7 {
  execsql { 
    CREATE TABLE t2(a, b);
    INSERT INTO t2 SELECT * FROM t1;
    DROP TABLE t1;
  }
  list [file size test.db] [file size test.db-wal]
} [list [expr 3*1024] [log_file_size 6 1024]]
do_test walbak-1.8 {
  execsql { VACUUM }
  list [file size test.db] [file size test.db-wal]
} [list [expr 3*1024] [log_file_size 8 1024]]
do_test walbak-1.9 {
  execsql { PRAGMA checkpoint }
  list [file size test.db] [file size test.db-wal]
} [list [expr 2*1024] [log_file_size 8 1024]]

#-------------------------------------------------------------------------
# Backups when the source db is modified mid-backup.
#
proc sig {{db db}} {
  $db eval { 
    PRAGMA integrity_check;
    SELECT md5sum(a, b) FROM t1; 
  }
}
db close
file delete test.db
sqlite3 db test.db
do_test walbak-2.1 {
  execsql { PRAGMA journal_mode = WAL }
  execsql {
    CREATE TABLE t1(a PRIMARY KEY, b);
    BEGIN;
      INSERT INTO t1 VALUES(randomblob(500), randomblob(500));
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /*  2 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /*  4 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /*  8 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 16 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 32 */
      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 64 */
    COMMIT;
  }
} {}
do_test walbak-2.2 {
  db backup abc.db
  sqlite3 db2 abc.db
  string compare [sig db] [sig db2]
} {0}

do_test walbak-2.3 {
  sqlite3_backup B db2 main db main
  B step 50
  execsql { UPDATE t1 SET b = randomblob(500) }
  list [B step 1000] [B finish]
} {SQLITE_DONE SQLITE_OK}
do_test walbak-2.4 {
  string compare [sig db] [sig db2]
} {0}

do_test walbak-2.5 {
  db close
  sqlite3 db test.db
  execsql { PRAGMA cache_size = 10 }
  sqlite3_backup B db2 main db main
  B step 50
  execsql {
    BEGIN;
      UPDATE t1 SET b = randomblob(500);
  }
  expr [file size test.db-wal] > 10*1024
} {1}
do_test walbak-2.6 {
  B step 1000
} {SQLITE_BUSY}
do_test walbak-2.7 {
  execsql COMMIT
  list [B step 1000] [B finish]
} {SQLITE_DONE SQLITE_OK}
do_test walbak-2.8 {
  string compare [sig db] [sig db2]
} {0}

do_test walbak-2.9 {
  db close
  sqlite3 db test.db
  execsql { PRAGMA cache_size = 10 }
  sqlite3_backup B db2 main db main
  B step 50
  execsql {
    BEGIN;
      UPDATE t1 SET b = randomblob(500);
  }
  expr [file size test.db-wal] > 10*1024
} {1}
do_test walbak-2.10 {
  B step 1000
} {SQLITE_BUSY}
do_test walbak-2.11 {
  execsql ROLLBACK
set sigB [sig db]
  list [B step 1000] [B finish]
} {SQLITE_DONE SQLITE_OK}
do_test walbak-2.12 {
  string compare [sig db] [sig db2]
} {0}
db2 close

finish_test