/ Check-in [1a391f3c]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix problems with recovering wal files that use a page-size other than the default.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 1a391f3c55dc9d4266552fa26d2a9839c6bafce4
User & Date: dan 2010-05-04 14:47:40
Context
2010-05-04
15:20
Add a test case to verify that log files containing pages that are not a power-of-two bytes in size are handled correctly. check-in: c2bf693f user: dan tags: trunk
14:47
Fix problems with recovering wal files that use a page-size other than the default. check-in: 1a391f3c user: dan tags: trunk
11:06
Fix a typo in walfault.test. check-in: 232dbe8e user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to src/pager.c.

2192
2193
2194
2195
2196
2197
2198
2199
2200

2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
....
2828
2829
2830
2831
2832
2833
2834









2835
2836
2837
2838
2839
2840
2841
....
3051
3052
3053
3054
3055
3056
3057
3058

3059
3060
3061
3062
3063
3064
3065
....
5829
5830
5831
5832
5833
5834
5835

5836
5837
5838
5839
5840
5841
5842
5843
....
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
** If an IO error occurs, then the IO error is returned to the caller.
** Otherwise, SQLITE_OK is returned.
*/
static int readDbPage(PgHdr *pPg){
  Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
  Pgno pgno = pPg->pgno;       /* Page number to read */
  int rc = SQLITE_OK;          /* Return code */
  i64 iOffset;                 /* Byte offset of file to read from */
  int isInWal = 0;             /* True if page is in log file */


  assert( pPager->state>=PAGER_SHARED && !MEMDB );
  assert( isOpen(pPager->fd) );

  if( NEVER(!isOpen(pPager->fd)) ){
    assert( pPager->tempFile );
    memset(pPg->pData, 0, pPager->pageSize);
    return SQLITE_OK;
  }

  if( pagerUseWal(pPager) ){
    /* Try to pull the page from the write-ahead log. */
    rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pPg->pData);
  }
  if( rc==SQLITE_OK && !isInWal ){
    iOffset = (pgno-1)*(i64)pPager->pageSize;
    rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset);
    if( rc==SQLITE_IOERR_SHORT_READ ){
      rc = SQLITE_OK;
    }
  }

  if( pgno==1 ){
    if( rc ){
................................................................................
** the error code is returned to the caller and the contents of the
** output buffer undefined.
*/
int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){
  int rc = SQLITE_OK;
  memset(pDest, 0, N);
  assert( isOpen(pPager->fd) || pPager->tempFile );









  if( isOpen(pPager->fd) ){
    IOTRACE(("DBHDR %p 0 %d\n", pPager, N))
    rc = sqlite3OsRead(pPager->fd, pDest, N, 0);
    if( rc==SQLITE_IOERR_SHORT_READ ){
      rc = SQLITE_OK;
    }
  }
................................................................................

  disable_simulated_io_errors();
  sqlite3BeginBenignMalloc();
  pPager->errCode = 0;
  pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
  sqlite3WalClose(pPager->pWal, pPager->fd, 
    (pPager->noSync ? 0 : pPager->sync_flags), pTmp

  );
  pPager->pWal = 0;
#endif
  pager_reset(pPager);
  if( MEMDB ){
    pager_unlock(pPager);
  }else{
................................................................................
*/
int sqlite3PagerCheckpoint(Pager *pPager){
  int rc = SQLITE_OK;
  if( pPager->pWal ){
    u8 *zBuf = (u8 *)pPager->pTmpSpace;
    rc = sqlite3WalCheckpoint(pPager->pWal, pPager->fd, 
        (pPager->noSync ? 0 : pPager->sync_flags),

        zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg
    );
  }
  return rc;
}

int sqlite3PagerWalCallback(Pager *pPager){
  return sqlite3WalCallback(pPager->pWal);
................................................................................
  ** the database file, the log and log-summary files will be deleted.
  */
  if( rc==SQLITE_OK && pPager->pWal ){
    rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE);
    if( rc==SQLITE_OK ){
      rc = sqlite3WalClose(pPager->pWal, pPager->fd,
        (pPager->noSync ? 0 : pPager->sync_flags), 
        (u8*)pPager->pTmpSpace
      );
      pPager->pWal = 0;
    }
  }
  return rc;
}
#endif

#endif /* SQLITE_OMIT_DISKIO */







<

>












|


|
|







 







>
>
>
>
>
>
>
>
>







 







|
>







 







>
|







 







|









2192
2193
2194
2195
2196
2197
2198

2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
....
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
....
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
....
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
....
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
** If an IO error occurs, then the IO error is returned to the caller.
** Otherwise, SQLITE_OK is returned.
*/
static int readDbPage(PgHdr *pPg){
  Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */
  Pgno pgno = pPg->pgno;       /* Page number to read */
  int rc = SQLITE_OK;          /* Return code */

  int isInWal = 0;             /* True if page is in log file */
  int pgsz = pPager->pageSize; /* Number of bytes to read */

  assert( pPager->state>=PAGER_SHARED && !MEMDB );
  assert( isOpen(pPager->fd) );

  if( NEVER(!isOpen(pPager->fd)) ){
    assert( pPager->tempFile );
    memset(pPg->pData, 0, pPager->pageSize);
    return SQLITE_OK;
  }

  if( pagerUseWal(pPager) ){
    /* Try to pull the page from the write-ahead log. */
    rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData);
  }
  if( rc==SQLITE_OK && !isInWal ){
    i64 iOffset = (pgno-1)*(i64)pPager->pageSize;
    rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset);
    if( rc==SQLITE_IOERR_SHORT_READ ){
      rc = SQLITE_OK;
    }
  }

  if( pgno==1 ){
    if( rc ){
................................................................................
** the error code is returned to the caller and the contents of the
** output buffer undefined.
*/
int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){
  int rc = SQLITE_OK;
  memset(pDest, 0, N);
  assert( isOpen(pPager->fd) || pPager->tempFile );

  if( pagerUseWal(pPager) ){
    int isInWal = 0;
    rc = sqlite3WalRead(pPager->pWal, 1, &isInWal, N, pDest);
    if( rc!=SQLITE_OK || isInWal ){
      return rc;
    }
  }

  if( isOpen(pPager->fd) ){
    IOTRACE(("DBHDR %p 0 %d\n", pPager, N))
    rc = sqlite3OsRead(pPager->fd, pDest, N, 0);
    if( rc==SQLITE_IOERR_SHORT_READ ){
      rc = SQLITE_OK;
    }
  }
................................................................................

  disable_simulated_io_errors();
  sqlite3BeginBenignMalloc();
  pPager->errCode = 0;
  pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
  sqlite3WalClose(pPager->pWal, pPager->fd, 
    (pPager->noSync ? 0 : pPager->sync_flags), 
    pPager->pageSize, pTmp
  );
  pPager->pWal = 0;
#endif
  pager_reset(pPager);
  if( MEMDB ){
    pager_unlock(pPager);
  }else{
................................................................................
*/
int sqlite3PagerCheckpoint(Pager *pPager){
  int rc = SQLITE_OK;
  if( pPager->pWal ){
    u8 *zBuf = (u8 *)pPager->pTmpSpace;
    rc = sqlite3WalCheckpoint(pPager->pWal, pPager->fd, 
        (pPager->noSync ? 0 : pPager->sync_flags),
        pPager->pageSize, zBuf, 
        pPager->xBusyHandler, pPager->pBusyHandlerArg
    );
  }
  return rc;
}

int sqlite3PagerWalCallback(Pager *pPager){
  return sqlite3WalCallback(pPager->pWal);
................................................................................
  ** the database file, the log and log-summary files will be deleted.
  */
  if( rc==SQLITE_OK && pPager->pWal ){
    rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE);
    if( rc==SQLITE_OK ){
      rc = sqlite3WalClose(pPager->pWal, pPager->fd,
        (pPager->noSync ? 0 : pPager->sync_flags), 
        pPager->pageSize, (u8*)pPager->pTmpSpace
      );
      pPager->pWal = 0;
    }
  }
  return rc;
}
#endif

#endif /* SQLITE_OMIT_DISKIO */

Changes to src/wal.c.

706
707
708
709
710
711
712

713
714
715
716
717
718
719
720
721
722
723
724
725
726
727










728
729
730
731
732
733
734
...
784
785
786
787
788
789
790

791
792
793
794
795
796
797
798
...
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
...
949
950
951
952
953
954
955
956






957
958
959
960
961
962
963
....
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
....
1262
1263
1264
1265
1266
1267
1268

1269
1270
1271
1272
1273
1274
1275
....
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
/*
** Checkpoint the contents of the log file.
*/
static int walCheckpoint(
  Wal *pWal,                      /* Wal connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  int sync_flags,                 /* Flags for OsSync() (or 0) */

  u8 *zBuf                        /* Temporary buffer to use */
){
  int rc;                         /* Return code */
  int pgsz = pWal->hdr.pgsz;      /* Database page-size */
  WalIterator *pIter = 0;         /* Wal iterator context */
  u32 iDbpage = 0;                /* Next database page to write */
  u32 iFrame = 0;                 /* Wal frame containing data for iDbpage */

  if( pWal->hdr.iLastPg==0 ){
    return SQLITE_OK;
  }

  /* Allocate the iterator */
  pIter = walIteratorInit(pWal);
  if( !pIter ) return SQLITE_NOMEM;











  /* Sync the log file to disk */
  if( sync_flags ){
    rc = sqlite3OsSync(pWal->pFd, sync_flags);
    if( rc!=SQLITE_OK ) goto out;
  }

................................................................................
/*
** Close a connection to a log file.
*/
int sqlite3WalClose(
  Wal *pWal,                      /* Wal to close */
  sqlite3_file *pFd,              /* Database file */
  int sync_flags,                 /* Flags to pass to OsSync() (or 0) */

  u8 *zBuf                        /* Buffer of at least page-size bytes */
){
  int rc = SQLITE_OK;
  if( pWal ){
    int isDelete = 0;             /* True to unlink wal and wal-index files */

    /* If an EXCLUSIVE lock can be obtained on the database file (using the
    ** ordinary, rollback-mode locking methods, this guarantees that the
................................................................................
    ** the database. In this case checkpoint the database and unlink both
    ** the wal and wal-index files.
    **
    ** The EXCLUSIVE lock is not released before returning.
    */
    rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE);
    if( rc==SQLITE_OK ){
      rc = walCheckpoint(pWal, pFd, sync_flags, zBuf);
      if( rc==SQLITE_OK ){
        isDelete = 1;
      }
      walIndexUnmap(pWal);
    }

    pWal->pVfs->xShmClose(pWal->pVfs, pWal->pWIndex, isDelete);
................................................................................
  );
  walSetLock(pWal, SQLITE_SHM_UNLOCK);
}

/*
** Read a page from the log, if it is present. 
*/
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){






  u32 iRead = 0;
  u32 *aData; 
  int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00);

  assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE );
  walIndexMap(pWal, -1);

................................................................................

  /* If iRead is non-zero, then it is the log frame number that contains the
  ** required page. Read and return data from the log file.
  */
  if( iRead ){
    i64 iOffset = walFrameOffset(iRead, pWal->hdr.pgsz) + WAL_FRAME_HDRSIZE;
    *pInWal = 1;
    return sqlite3OsRead(pWal->pFd, pOut, pWal->hdr.pgsz, iOffset);
  }

  *pInWal = 0;
  return SQLITE_OK;
}


................................................................................
**   3. Zero the wal-index header (so new readers will ignore the log).
**   4. Drop the CHECKPOINT lock.
*/
int sqlite3WalCheckpoint(
  Wal *pWal,                      /* Wal connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  int sync_flags,                 /* Flags to sync db file with (or 0) */

  u8 *zBuf,                       /* Temporary buffer to use */
  int (*xBusyHandler)(void *),    /* Pointer to busy-handler function */
  void *pBusyHandlerArg           /* Argument to pass to xBusyHandler */
){
  int rc;                         /* Return code */
  int isChanged = 0;              /* True if a new wal-index header is loaded */

................................................................................
    walSetLock(pWal, SQLITE_SHM_UNLOCK);
    return rc;
  }

  /* Copy data from the log to the database file. */
  rc = walIndexReadHdr(pWal, &isChanged);
  if( rc==SQLITE_OK ){
    rc = walCheckpoint(pWal, pFd, sync_flags, zBuf);
  }
  if( isChanged ){
    /* If a new wal-index header was loaded before the checkpoint was 
    ** performed, then the pager-cache associated with log pWal is now
    ** out of date. So zero the cached wal-index header to ensure that
    ** next time the pager opens a snapshot on this database it knows that
    ** the cache needs to be reset.







>








<
<
<
<



>
>
>
>
>
>
>
>
>
>







 







>
|







 







|







 







|
>
>
>
>
>
>







 







|







 







>







 







|







706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721




722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
...
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
...
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
...
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
....
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
....
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
....
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
/*
** Checkpoint the contents of the log file.
*/
static int walCheckpoint(
  Wal *pWal,                      /* Wal connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  int sync_flags,                 /* Flags for OsSync() (or 0) */
  int nBuf,                       /* Size of zBuf in bytes */
  u8 *zBuf                        /* Temporary buffer to use */
){
  int rc;                         /* Return code */
  int pgsz = pWal->hdr.pgsz;      /* Database page-size */
  WalIterator *pIter = 0;         /* Wal iterator context */
  u32 iDbpage = 0;                /* Next database page to write */
  u32 iFrame = 0;                 /* Wal frame containing data for iDbpage */





  /* Allocate the iterator */
  pIter = walIteratorInit(pWal);
  if( !pIter ) return SQLITE_NOMEM;

  if( pWal->hdr.iLastPg==0 ){
    rc = SQLITE_OK;
    goto out;
  }

  if( pWal->hdr.pgsz!=nBuf ){
    rc = SQLITE_CORRUPT_BKPT;
    goto out;
  }

  /* Sync the log file to disk */
  if( sync_flags ){
    rc = sqlite3OsSync(pWal->pFd, sync_flags);
    if( rc!=SQLITE_OK ) goto out;
  }

................................................................................
/*
** Close a connection to a log file.
*/
int sqlite3WalClose(
  Wal *pWal,                      /* Wal to close */
  sqlite3_file *pFd,              /* Database file */
  int sync_flags,                 /* Flags to pass to OsSync() (or 0) */
  int nBuf,
  u8 *zBuf                        /* Buffer of at least nBuf bytes */
){
  int rc = SQLITE_OK;
  if( pWal ){
    int isDelete = 0;             /* True to unlink wal and wal-index files */

    /* If an EXCLUSIVE lock can be obtained on the database file (using the
    ** ordinary, rollback-mode locking methods, this guarantees that the
................................................................................
    ** the database. In this case checkpoint the database and unlink both
    ** the wal and wal-index files.
    **
    ** The EXCLUSIVE lock is not released before returning.
    */
    rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE);
    if( rc==SQLITE_OK ){
      rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf);
      if( rc==SQLITE_OK ){
        isDelete = 1;
      }
      walIndexUnmap(pWal);
    }

    pWal->pVfs->xShmClose(pWal->pVfs, pWal->pWIndex, isDelete);
................................................................................
  );
  walSetLock(pWal, SQLITE_SHM_UNLOCK);
}

/*
** Read a page from the log, if it is present. 
*/
int sqlite3WalRead(
  Wal *pWal, 
  Pgno pgno, 
  int *pInWal, 
  int nOut,
  u8 *pOut
){
  u32 iRead = 0;
  u32 *aData; 
  int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00);

  assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE );
  walIndexMap(pWal, -1);

................................................................................

  /* If iRead is non-zero, then it is the log frame number that contains the
  ** required page. Read and return data from the log file.
  */
  if( iRead ){
    i64 iOffset = walFrameOffset(iRead, pWal->hdr.pgsz) + WAL_FRAME_HDRSIZE;
    *pInWal = 1;
    return sqlite3OsRead(pWal->pFd, pOut, nOut, iOffset);
  }

  *pInWal = 0;
  return SQLITE_OK;
}


................................................................................
**   3. Zero the wal-index header (so new readers will ignore the log).
**   4. Drop the CHECKPOINT lock.
*/
int sqlite3WalCheckpoint(
  Wal *pWal,                      /* Wal connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  int sync_flags,                 /* Flags to sync db file with (or 0) */
  int nBuf,                       /* Size of temporary buffer */
  u8 *zBuf,                       /* Temporary buffer to use */
  int (*xBusyHandler)(void *),    /* Pointer to busy-handler function */
  void *pBusyHandlerArg           /* Argument to pass to xBusyHandler */
){
  int rc;                         /* Return code */
  int isChanged = 0;              /* True if a new wal-index header is loaded */

................................................................................
    walSetLock(pWal, SQLITE_SHM_UNLOCK);
    return rc;
  }

  /* Copy data from the log to the database file. */
  rc = walIndexReadHdr(pWal, &isChanged);
  if( rc==SQLITE_OK ){
    rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf);
  }
  if( isChanged ){
    /* If a new wal-index header was loaded before the checkpoint was 
    ** performed, then the pager-cache associated with log pWal is now
    ** out of date. So zero the cached wal-index header to ensure that
    ** next time the pager opens a snapshot on this database it knows that
    ** the cache needs to be reset.

Changes to src/wal.h.

20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
..
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
..
79
80
81
82
83
84
85

86
87
88
89
90
91
92
#include "sqliteInt.h"

#ifdef SQLITE_OMIT_WAL
# define sqlite3WalOpen(x,y,z)             0
# define sqlite3WalClose(w,x,y,z)          0
# define sqlite3WalOpenSnapshot(y,z)       0
# define sqlite3WalCloseSnapshot(z) 
# define sqlite3WalRead(w,x,y,z)           0
# define sqlite3WalDbsize(y,z)
# define sqlite3WalWriteLock(y,z)          0
# define sqlite3WalUndo(x,y,z)             0
# define sqlite3WalSavepoint(z)            0
# define sqlite3WalSavepointUndo(y,z)      0
# define sqlite3WalFrames(u,v,w,x,y,z)     0
# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0
................................................................................
/* Connection to a write-ahead log (WAL) file. 
** There is one object of this type for each pager. 
*/
typedef struct Wal Wal;

/* Open and close a connection to a write-ahead log. */
int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Wal **ppWal);
int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, u8 *zBuf);

/* Used by readers to open (lock) and close (unlock) a snapshot.  A 
** snapshot is like a read-transaction.  It is the state of the database
** at an instant in time.  sqlite3WalOpenSnapshot gets a read lock and
** preserves the current state even if the other threads or processes
** write to or checkpoint the WAL.  sqlite3WalCloseSnapshot() closes the
** transaction and releases the lock.
*/
int sqlite3WalOpenSnapshot(Wal *pWal, int *);
void sqlite3WalCloseSnapshot(Wal *pWal);

/* Read a page from the write-ahead log, if it is present. */
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut);

/* Return the size of the database as it existed at the beginning
** of the snapshot */
void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno);

/* Obtain or release the WRITER lock. */
int sqlite3WalWriteLock(Wal *pWal, int op);
................................................................................
int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);

/* Copy pages from the log to the database file */ 
int sqlite3WalCheckpoint(
  Wal *pWal,                      /* Write-ahead log connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  int sync_flags,                 /* Flags to sync db file with (or 0) */

  u8 *zBuf,                       /* Temporary buffer to use */
  int (*xBusyHandler)(void *),    /* Pointer to busy-handler function */
  void *pBusyHandlerArg           /* Argument to pass to xBusyHandler */
);

/* Return the value to pass to a sqlite3_wal_hook callback, the
** number of frames in the WAL at the point of the last commit since







|







 







|












|







 







>







20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
..
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
..
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
#include "sqliteInt.h"

#ifdef SQLITE_OMIT_WAL
# define sqlite3WalOpen(x,y,z)             0
# define sqlite3WalClose(w,x,y,z)          0
# define sqlite3WalOpenSnapshot(y,z)       0
# define sqlite3WalCloseSnapshot(z) 
# define sqlite3WalRead(v,w,x,y,z)         0
# define sqlite3WalDbsize(y,z)
# define sqlite3WalWriteLock(y,z)          0
# define sqlite3WalUndo(x,y,z)             0
# define sqlite3WalSavepoint(z)            0
# define sqlite3WalSavepointUndo(y,z)      0
# define sqlite3WalFrames(u,v,w,x,y,z)     0
# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0
................................................................................
/* Connection to a write-ahead log (WAL) file. 
** There is one object of this type for each pager. 
*/
typedef struct Wal Wal;

/* Open and close a connection to a write-ahead log. */
int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Wal **ppWal);
int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, int, u8 *);

/* Used by readers to open (lock) and close (unlock) a snapshot.  A 
** snapshot is like a read-transaction.  It is the state of the database
** at an instant in time.  sqlite3WalOpenSnapshot gets a read lock and
** preserves the current state even if the other threads or processes
** write to or checkpoint the WAL.  sqlite3WalCloseSnapshot() closes the
** transaction and releases the lock.
*/
int sqlite3WalOpenSnapshot(Wal *pWal, int *);
void sqlite3WalCloseSnapshot(Wal *pWal);

/* Read a page from the write-ahead log, if it is present. */
int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut);

/* Return the size of the database as it existed at the beginning
** of the snapshot */
void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno);

/* Obtain or release the WRITER lock. */
int sqlite3WalWriteLock(Wal *pWal, int op);
................................................................................
int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);

/* Copy pages from the log to the database file */ 
int sqlite3WalCheckpoint(
  Wal *pWal,                      /* Write-ahead log connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  int sync_flags,                 /* Flags to sync db file with (or 0) */
  int nBuf,                       /* Size of buffer nBuf */
  u8 *zBuf,                       /* Temporary buffer to use */
  int (*xBusyHandler)(void *),    /* Pointer to busy-handler function */
  void *pBusyHandlerArg           /* Argument to pass to xBusyHandler */
);

/* Return the value to pass to a sqlite3_wal_hook callback, the
** number of frames in the WAL at the point of the last commit since

Changes to test/wal.test.

1188
1189
1190
1191
1192
1193
1194
1195

















































































































































1196
1197
1198
1199
1200

  do_test wal-17.$tn.3 {
    db close
    file size test.db
  } [expr 512*171]
}
sqlite3_test_control_pending_byte $old_pending_byte


















































































































































catch { db2 close }
catch { db close }
finish_test










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




<
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344


  do_test wal-17.$tn.3 {
    db close
    file size test.db
  } [expr 512*171]
}
sqlite3_test_control_pending_byte $old_pending_byte

#-------------------------------------------------------------------------
# This test - wal-18.* - verifies a couple of specific conditions that
# may be encountered while recovering a log file are handled correctly:
#
#   wal-18.1.* When the first 32-bits of a frame checksum is correct but 
#              the second 32-bits are false, and
#
#   wal-18.2.* When the page-size field that occurs at the start of a log
#              file is a power of 2 greater than 16384 or smaller than 512.
#
file delete -force test.db test.db-wal test.db-journal
do_test wal-18.0 {
  sqlite3 db test.db
  execsql {
    PRAGMA page_size = 1024;
    PRAGMA auto_vacuum = 0;
    PRAGMA journal_mode = WAL;
    PRAGMA synchronous = OFF;

    CREATE TABLE t1(a, b, UNIQUE(a, b));
    INSERT INTO t1 VALUES(0, 0);
    PRAGMA wal_checkpoint;

    INSERT INTO t1 VALUES(1, 2);          -- frames 1 and 2
    INSERT INTO t1 VALUES(3, 4);          -- frames 3 and 4
    INSERT INTO t1 VALUES(5, 6);          -- frames 5 and 6
  }

  file copy -force test.db testX.db
  file copy -force test.db-wal testX.db-wal
  db close
  list [file size testX.db] [file size testX.db-wal]
} [list [expr 3*1024] [log_file_size 6 1024]]

foreach {nFrame result} {
         0      {0 0}
         1      {0 0}
         2      {0 0 1 2}
         3      {0 0 1 2}
         4      {0 0 1 2 3 4}
         5      {0 0 1 2 3 4}
         6      {0 0 1 2 3 4 5 6}
} {
  do_test wal-18.1.$nFrame {
    file copy -force testX.db test.db
    file copy -force testX.db-wal test.db-wal

    hexio_write test.db-wal [expr 12 + $nFrame*(16+1024) + 12] 00000000

    sqlite3 db test.db
    execsql { 
      SELECT * FROM t1;
      PRAGMA integrity_check; 
    }
  } [concat $result ok]
  db close
} 

proc randomblob {pgsz} {
  sqlite3 rbdb :memory:
  set blob [rbdb one {SELECT randomblob($pgsz)}]
  rbdb close
  set blob
}

proc logcksum {ckv1 ckv2 blob} {
  upvar $ckv1 c1
  upvar $ckv2 c2

  binary scan $blob iu* values
  foreach v $values {
    incr c1 $v
    incr c2 $c1
  }

  set c1 [expr ($c1 + ($c1>>24))&0xFFFFFFFF]
  set c2 [expr ($c2 + ($c2>>24))&0xFFFFFFFF]
}

file copy -force test.db testX.db
foreach {tn pgsz works} { 
  1    128    0
  2    256    0
  3    512    1
  4   1024    1
  5   2048    1
  6   4096    1
  7   8192    1
  8  16384    1
  9  32768    1
 10  65536    0
} {

  for {set pg 1} {$pg <= 3} {incr pg} {
    file copy -force testX.db test.db
    file delete -force test.db-wal
  
    # Check that the database now exists and consists of three pages. And
    # that there is no associated wal file.
    #
    do_test wal-18.2.$tn.$pg.1 { file exists test.db-wal } 0
    do_test wal-18.2.$tn.$pg.2 { file exists test.db } 1
    do_test wal-18.2.$tn.$pg.3 { file size test.db } [expr 1024*3]
  
    do_test wal-18.2.$tn.$pg.4 {

      # Create a wal file that contains a single frame (database page
      # number $pg) with the commit flag set. The frame checksum is
      # correct, but the contents of the database page are corrupt.
      #
      # The page-size in the log file header is set to $pgsz. If the
      # WAL code considers $pgsz to be a valid SQLite database file page-size,
      # the database will be corrupt (because the garbage frame contents
      # will be treated as valid content). If $pgsz is invalid (too small
      # or too large), the db will not be corrupt as the log file will
      # be ignored.
      #
      set c1 22
      set c2 23
      set walhdr [binary format III $pgsz $c1 $c2]
      set framebody [randomblob $pgsz]
      set framehdr  [binary format II $pg 5]
      logcksum c1 c2 $framehdr
      logcksum c1 c2 $framebody
      set framehdr [binary format IIII $pg 5 $c1 $c2]
      set fd [open test.db-wal w]
      fconfigure $fd -encoding binary -translation binary
      puts -nonewline $fd $walhdr
      puts -nonewline $fd $framehdr
      puts -nonewline $fd $framebody
      close $fd
  
      file size test.db-wal
    } [log_file_size 1 $pgsz]
  
    do_test wal-18.2.$tn.$pg.5 {
      sqlite3 db test.db
      set rc [catch { db one {PRAGMA integrity_check} } msg]
      expr { $rc!=0 || $msg!="ok" }
    } $works
  
    db close
  }
}

catch { db2 close }
catch { db close }
finish_test


Changes to test/walcrash.test.

246
247
248
249
250
251
252











253
254
255




256
257

258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
  } {1}
  do_test walcrash-6.$i.3 { execsql { PRAGMA main.integrity_check } } {ok}
  do_test walcrash-6.$i.4 { execsql { PRAGMA main.journal_mode } } {wal}

  db close
}












for {set i 1} {$i < $REPEATS} {incr i} {
  file delete -force test.db test.db-wal





  do_test walcrash-7.$i.1 {
    crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 {

      PRAGMA journal_mode = wal;
      BEGIN;
        CREATE TABLE t1(a, b);
        INSERT INTO t1 VALUES(1, 2);
      COMMIT;
      PRAGMA wal_checkpoint;
      CREATE INDEX i1 ON t1(a);
      PRAGMA wal_checkpoint;
    }
  } {1 {child process exited abnormally}}

  do_test walcrash-7.$i.2 {
    sqlite3 db test.db
    execsql { SELECT b FROM t1 WHERE a = 1 }
  } {2}
  do_test walcrash-7.$i.3 { execsql { PRAGMA main.integrity_check } } {ok}







>
>
>
>
>
>
>
>
>
>
>



>
>
>
>

|
>








|







246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
  } {1}
  do_test walcrash-6.$i.3 { execsql { PRAGMA main.integrity_check } } {ok}
  do_test walcrash-6.$i.4 { execsql { PRAGMA main.journal_mode } } {wal}

  db close
}

#-------------------------------------------------------------------------
# This test case simulates a crash while checkpointing the database. Page
# 1 is one of the pages overwritten by the checkpoint. This is a special
# case because it means the content of page 1 may be damaged. SQLite will
# have to determine:
#
#   (a) that the database is a WAL database, and 
#   (b) the database page-size
#
# based on the log file.
#
for {set i 1} {$i < $REPEATS} {incr i} {
  file delete -force test.db test.db-wal

  # Select a page-size for this test.
  #
  set pgsz [lindex {512 1024 2048 4096 8192 16384} [expr $i%6]]

  do_test walcrash-7.$i.1 {
    crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 "
      PRAGMA page_size = $pgsz;
      PRAGMA journal_mode = wal;
      BEGIN;
        CREATE TABLE t1(a, b);
        INSERT INTO t1 VALUES(1, 2);
      COMMIT;
      PRAGMA wal_checkpoint;
      CREATE INDEX i1 ON t1(a);
      PRAGMA wal_checkpoint;
    "
  } {1 {child process exited abnormally}}

  do_test walcrash-7.$i.2 {
    sqlite3 db test.db
    execsql { SELECT b FROM t1 WHERE a = 1 }
  } {2}
  do_test walcrash-7.$i.3 { execsql { PRAGMA main.integrity_check } } {ok}