/ Check-in [65ba804d]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the WAL file format to support two kinds of checksums - one that is fast to calculate on little-endian architectures and another that is fast on big-endian architectures. A flag in the wal-header indicates which the file uses.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 65ba804dd1d31d1eef6ae3f40a3ade344a410b84
User & Date: dan 2010-05-24 10:39:36
Context
2010-05-24
12:34
Fix up test_osinst.c to work with SQLITE_OMIT_VIRTUALTABLE. check-in: 51fd3815 user: drh tags: trunk
10:39
Change the WAL file format to support two kinds of checksums - one that is fast to calculate on little-endian architectures and another that is fast on big-endian architectures. A flag in the wal-header indicates which the file uses. check-in: 65ba804d user: dan tags: trunk
2010-05-22
08:22
Add a couple of missing methods to test_osinst.c.. check-in: 5c9e9c06 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

211
212
213
214
215
216
217

218
219
220
221
222
223
224
225
...
231
232
233
234
235
236
237











238
239
240
241
242
243
244
...
289
290
291
292
293
294
295











296
297
298
299
300
301
302
303
304
305
306

307
308
309
310
311
312

313

314
315
316
317
318
319
320
321
322
323

324
325
326






327
328


329
330
331
332
333
334
335
...
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
...
385
386
387
388
389
390
391

392
393
394
395
396
397

398
399
400
401
402
403
404
405
406
...
410
411
412
413
414
415
416

417
418
419
420
421
422
423
...
424
425
426
427
428
429
430

431
432
433
434
435
436
437
438
439
...
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713

714
715
716
717
718
719
720
721
722
723
724


725

726


727


728
729

730
731
732
733
734
735
736
....
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
....
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636

1637
1638
1639
1640
1641
1642
1643
** The following object holds a copy of the wal-index header content.
**
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;      /* Counter incremented each transaction */

  u32 szPage;       /* Database page size in bytes */
  u32 mxFrame;      /* Index of last valid frame in the WAL */
  u32 nPage;        /* Size of database in pages */
  u32 aSalt[2];     /* Salt-1 and salt-2 values copied from WAL header */
  u32 aCksum[2];    /* Checksum over all prior fields */
};

/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
................................................................................
#define WALINDEX_LOCK_RESERVED 8

/* Size of header before each frame in wal */
#define WAL_FRAME_HDRSIZE 24

/* Size of write ahead log header */
#define WAL_HDRSIZE 24












/*
** Return the offset of frame iFrame in the write-ahead log file, 
** assuming a database page size of szPage bytes. The offset returned
** is to the start of the write-ahead log frame-header.
*/
#define walFrameOffset(iFrame, szPage) (                               \
................................................................................
  struct WalSegment {
    int iNext;              /* Next slot in aIndex[] not previously returned */
    u8 *aIndex;             /* i0, i1, i2... such that aPgno[iN] ascending */
    u32 *aPgno;             /* 256 page numbers.  Pointer to Wal.pWiData */
  } aSegment[1];        /* One for every 256 entries in the WAL */
};













/*
** Generate or extend an 8 byte checksum based on the data in 
** array aByte[] and the initial values of aIn[0] and aIn[1] (or
** initial values of 0 and 0 if aIn==NULL).
**
** The checksum is written back into aOut[] before returning.
**
** nByte must be a positive multiple of 8.
*/
static void walChecksumBytes(

  u8 *a,           /* Content to be checksummed */
  int nByte,       /* Bytes of content in a[].  Must be a multiple of 8. */
  const u32 *aIn,  /* Initial checksum value input */
  u32 *aOut        /* OUT: Final checksum value output */
){
  u32 s1, s2;

  u8 *aEnd = (u8*)&a[nByte];

  if( aIn ){
    s1 = aIn[0];
    s2 = aIn[1];
  }else{
    s1 = s2 = 0;
  }

  assert( nByte>=8 );
  assert( (nByte&0x00000003)==0 );


  do {
    s1 += (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3] + s2;
    s2 += (a[4]<<24) + (a[5]<<16) + (a[6]<<8) + a[7] + s1;






    a += 8;
  }while( a<aEnd );


  aOut[0] = s1;
  aOut[1] = s2;
}

/*
** Attempt to change the lock status.
**
................................................................................
/*
** Write the header information in pWal->hdr into the wal-index.
**
** The checksum on pWal->hdr is updated before it is written.
*/
static void walIndexWriteHdr(Wal *pWal){
  WalIndexHdr *aHdr;
  walChecksumBytes((u8*)&pWal->hdr,
                   sizeof(pWal->hdr) - sizeof(pWal->hdr.aCksum),
                   0, pWal->hdr.aCksum);
  aHdr = (WalIndexHdr*)pWal->pWiData;
  memcpy(&aHdr[1], &pWal->hdr, sizeof(pWal->hdr));
  sqlite3OsShmBarrier(pWal->pDbFd);
  memcpy(&aHdr[0], &pWal->hdr, sizeof(pWal->hdr));
}
................................................................................
static void walEncodeFrame(
  Wal *pWal,                      /* The write-ahead log */
  u32 iPage,                      /* Database page number for frame */
  u32 nTruncate,                  /* New db size (or 0 for non-commit frames) */
  u8 *aData,                      /* Pointer to page data */
  u8 *aFrame                      /* OUT: Write encoded frame here */
){

  u32 aCksum[2];
  assert( WAL_FRAME_HDRSIZE==24 );
  sqlite3Put4byte(&aFrame[0], iPage);
  sqlite3Put4byte(&aFrame[4], nTruncate);
  memcpy(&aFrame[8], pWal->hdr.aSalt, 8);


  walChecksumBytes(aFrame, 16, 0, aCksum);
  walChecksumBytes(aData, pWal->szPage, aCksum, aCksum);

  sqlite3Put4byte(&aFrame[16], aCksum[0]);
  sqlite3Put4byte(&aFrame[20], aCksum[1]);
}

/*
** Check to see if the frame with header in aFrame[] and content
................................................................................
static int walDecodeFrame(
  Wal *pWal,                      /* The write-ahead log */
  u32 *piPage,                    /* OUT: Database page number for frame */
  u32 *pnTruncate,                /* OUT: New db size (or 0 if not commit) */
  u8 *aData,                      /* Pointer to page data (for checksum) */
  u8 *aFrame                      /* Frame data */
){

  u32 aCksum[2];
  assert( WAL_FRAME_HDRSIZE==24 );

  /* A frame is only valid if the salt values in the frame-header
  ** match the salt values in the wal-header. 
  */
  if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){
................................................................................
    return 0;
  }

  /* A frame is only valid if a checksum of the first 16 bytes
  ** of the frame-header, and the frame-data matches
  ** the checksum in the last 8 bytes of the frame-header.
  */

  walChecksumBytes(aFrame, 16, 0, aCksum);
  walChecksumBytes(aData, pWal->szPage, aCksum, aCksum);
  if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) 
   || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) 
  ){
    /* Checksum failed. */
    return 0;
  }

................................................................................
/*
** Recover the wal-index by reading the write-ahead log file. 
** The caller must hold RECOVER lock on the wal-index file.
*/
static int walIndexRecover(Wal *pWal){
  int rc;                         /* Return Code */
  i64 nSize;                      /* Size of log file */
  WalIndexHdr hdr;              /* Recovered wal-index header */

  assert( pWal->lockState>SQLITE_SHM_READ );
  memset(&hdr, 0, sizeof(hdr));

  rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  if( nSize>WAL_FRAME_HDRSIZE ){
    u8 aBuf[WAL_HDRSIZE];         /* Buffer to load first frame header into */
    u8 *aFrame = 0;               /* Malloc'd buffer to load entire frame */
    int szFrame;                  /* Number of bytes in buffer aFrame[] */
    u8 *aData;                    /* Pointer to data part of aFrame buffer */
    int iFrame;                   /* Index of last frame read */
    i64 iOffset;                  /* Next offset to read from log file */
    int szPage;                   /* Page size according to the log */


    /* Read in the first frame header in the file (to determine the 
    ** database page size).
    */
    rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    /* If the database page size is not a power of two, or is greater than
    ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid data.


    */

    szPage = sqlite3Get4byte(&aBuf[8]);


    if( szPage&(szPage-1) || szPage>SQLITE_MAX_PAGE_SIZE || szPage<512 ){


      goto finished;
    }

    pWal->szPage = szPage;
    pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
    memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);

    /* Malloc a buffer to read frames into. */
    szFrame = szPage + WAL_FRAME_HDRSIZE;
    aFrame = (u8 *)sqlite3_malloc(szFrame);
................................................................................

  if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
    return 1;   /* Dirty read */
  }  
  if( h1.szPage==0 ){
    return 1;   /* Malformed header - probably all zeros */
  }
  walChecksumBytes((u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
  if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){
    return 1;   /* Checksum does not match */
  }

  if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){
    *pChanged = 1;
    memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr));
................................................................................
  /* If this is the first frame written into the log, write the WAL
  ** header to the start of the WAL file. See comments at the top of
  ** this source file for a description of the WAL header format.
  */
  iFrame = pWal->hdr.mxFrame;
  if( iFrame==0 ){
    u8 aWalHdr[WAL_HDRSIZE];        /* Buffer to assembly wal-header in */
    sqlite3Put4byte(&aWalHdr[0], 0x377f0682);
    sqlite3Put4byte(&aWalHdr[4], 3007000);
    sqlite3Put4byte(&aWalHdr[8], szPage);
    pWal->szPage = szPage;

    sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
    memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }







>
|







 







>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>











>






>
|
>








|

>
|
<
|
>
>
>
>
>
>
|
|
>
>







 







|







 







>






>
|
|







 







>







 







>
|
|







 







|









|
|






>

<
|
<






|
>
>

>

>
>
|
>
>


>







 







|







 







|



>







211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
...
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
...
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351

352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
...
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
...
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
...
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
...
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
...
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753

754

755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
....
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
....
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
** The following object holds a copy of the wal-index header content.
**
** The actual header in the wal-index consists of two copies of this
** object.
*/
struct WalIndexHdr {
  u32 iChange;      /* Counter incremented each transaction */
  u16 bigEndCksum;  /* True if checksums in WAL are big-endian */
  u16 szPage;       /* Database page size in bytes */
  u32 mxFrame;      /* Index of last valid frame in the WAL */
  u32 nPage;        /* Size of database in pages */
  u32 aSalt[2];     /* Salt-1 and salt-2 values copied from WAL header */
  u32 aCksum[2];    /* Checksum over all prior fields */
};

/* A block of WALINDEX_LOCK_RESERVED bytes beginning at
................................................................................
#define WALINDEX_LOCK_RESERVED 8

/* Size of header before each frame in wal */
#define WAL_FRAME_HDRSIZE 24

/* Size of write ahead log header */
#define WAL_HDRSIZE 24

/* WAL magic value. Either this value, or the same value with the least
** significant bit also set (WAL_MAGIC | 0x00000001) is stored in 32-bit
** big-endian format in the first 4 bytes of a WAL file.
**
** If the LSB is set, then the checksums for each frame within the WAL
** file are calculated by treating all data as an array of 32-bit 
** big-endian words. Otherwise, they are calculated by interpreting 
** all data as 32-bit little-endian words.
*/
#define WAL_MAGIC 0x377f0682

/*
** Return the offset of frame iFrame in the write-ahead log file, 
** assuming a database page size of szPage bytes. The offset returned
** is to the start of the write-ahead log frame-header.
*/
#define walFrameOffset(iFrame, szPage) (                               \
................................................................................
  struct WalSegment {
    int iNext;              /* Next slot in aIndex[] not previously returned */
    u8 *aIndex;             /* i0, i1, i2... such that aPgno[iN] ascending */
    u32 *aPgno;             /* 256 page numbers.  Pointer to Wal.pWiData */
  } aSegment[1];        /* One for every 256 entries in the WAL */
};

/*
** The argument to this macro must be of type u32. On a little-endian
** architecture, it returns the u32 value that results from interpreting
** the 4 bytes as a big-endian value. On a big-endian architecture, it
** returns the value that would be produced by intepreting the 4 bytes
** of the input value as a little-endian integer.
*/
#define BYTESWAP32(x) ( \
    (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8)  \
  + (((x)&0x00FF0000)>>8)  + (((x)&0xFF000000)>>24) \
)

/*
** Generate or extend an 8 byte checksum based on the data in 
** array aByte[] and the initial values of aIn[0] and aIn[1] (or
** initial values of 0 and 0 if aIn==NULL).
**
** The checksum is written back into aOut[] before returning.
**
** nByte must be a positive multiple of 8.
*/
static void walChecksumBytes(
  int nativeCksum, /* True for native byte-order, false for non-native */
  u8 *a,           /* Content to be checksummed */
  int nByte,       /* Bytes of content in a[].  Must be a multiple of 8. */
  const u32 *aIn,  /* Initial checksum value input */
  u32 *aOut        /* OUT: Final checksum value output */
){
  u32 s1, s2;
  u32 *aData = (u32 *)a;
  u32 *aEnd = (u32 *)&a[nByte];

  if( aIn ){
    s1 = aIn[0];
    s2 = aIn[1];
  }else{
    s1 = s2 = 0;
  }

  assert( nByte>=8 );
  assert( (nByte&0x00000007)==0 );

  if( nativeCksum ){
    do {

      s1 += *aData++ + s2;
      s2 += *aData++ + s1;
    }while( aData<aEnd );
  }else{
    do {
      s1 += BYTESWAP32(aData[0]) + s2;
      s2 += BYTESWAP32(aData[1]) + s1;
      aData += 2;
    }while( aData<aEnd );
  }

  aOut[0] = s1;
  aOut[1] = s2;
}

/*
** Attempt to change the lock status.
**
................................................................................
/*
** Write the header information in pWal->hdr into the wal-index.
**
** The checksum on pWal->hdr is updated before it is written.
*/
static void walIndexWriteHdr(Wal *pWal){
  WalIndexHdr *aHdr;
  walChecksumBytes(1, (u8*)&pWal->hdr,
                   sizeof(pWal->hdr) - sizeof(pWal->hdr.aCksum),
                   0, pWal->hdr.aCksum);
  aHdr = (WalIndexHdr*)pWal->pWiData;
  memcpy(&aHdr[1], &pWal->hdr, sizeof(pWal->hdr));
  sqlite3OsShmBarrier(pWal->pDbFd);
  memcpy(&aHdr[0], &pWal->hdr, sizeof(pWal->hdr));
}
................................................................................
static void walEncodeFrame(
  Wal *pWal,                      /* The write-ahead log */
  u32 iPage,                      /* Database page number for frame */
  u32 nTruncate,                  /* New db size (or 0 for non-commit frames) */
  u8 *aData,                      /* Pointer to page data */
  u8 *aFrame                      /* OUT: Write encoded frame here */
){
  int nativeCksum;                /* True for native byte-order checksums */
  u32 aCksum[2];
  assert( WAL_FRAME_HDRSIZE==24 );
  sqlite3Put4byte(&aFrame[0], iPage);
  sqlite3Put4byte(&aFrame[4], nTruncate);
  memcpy(&aFrame[8], pWal->hdr.aSalt, 8);

  nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
  walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
  walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);

  sqlite3Put4byte(&aFrame[16], aCksum[0]);
  sqlite3Put4byte(&aFrame[20], aCksum[1]);
}

/*
** Check to see if the frame with header in aFrame[] and content
................................................................................
static int walDecodeFrame(
  Wal *pWal,                      /* The write-ahead log */
  u32 *piPage,                    /* OUT: Database page number for frame */
  u32 *pnTruncate,                /* OUT: New db size (or 0 if not commit) */
  u8 *aData,                      /* Pointer to page data (for checksum) */
  u8 *aFrame                      /* Frame data */
){
  int nativeCksum;                /* True for native byte-order checksums */
  u32 aCksum[2];
  assert( WAL_FRAME_HDRSIZE==24 );

  /* A frame is only valid if the salt values in the frame-header
  ** match the salt values in the wal-header. 
  */
  if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){
................................................................................
    return 0;
  }

  /* A frame is only valid if a checksum of the first 16 bytes
  ** of the frame-header, and the frame-data matches
  ** the checksum in the last 8 bytes of the frame-header.
  */
  nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
  walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
  walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
  if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) 
   || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) 
  ){
    /* Checksum failed. */
    return 0;
  }

................................................................................
/*
** Recover the wal-index by reading the write-ahead log file. 
** The caller must hold RECOVER lock on the wal-index file.
*/
static int walIndexRecover(Wal *pWal){
  int rc;                         /* Return Code */
  i64 nSize;                      /* Size of log file */
  WalIndexHdr hdr;                /* Recovered wal-index header */

  assert( pWal->lockState>SQLITE_SHM_READ );
  memset(&hdr, 0, sizeof(hdr));

  rc = sqlite3OsFileSize(pWal->pWalFd, &nSize);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  if( nSize>WAL_HDRSIZE ){
    u8 aBuf[WAL_HDRSIZE];         /* Buffer to load WAL header into */
    u8 *aFrame = 0;               /* Malloc'd buffer to load entire frame */
    int szFrame;                  /* Number of bytes in buffer aFrame[] */
    u8 *aData;                    /* Pointer to data part of aFrame buffer */
    int iFrame;                   /* Index of last frame read */
    i64 iOffset;                  /* Next offset to read from log file */
    int szPage;                   /* Page size according to the log */
    u32 magic;                    /* Magic value read from WAL header */


    /* Read in the WAL header. */

    rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    /* If the database page size is not a power of two, or is greater than
    ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid 
    ** data. Similarly, if the 'magic' value is invalid, ignore the whole
    ** WAL file.
    */
    magic = sqlite3Get4byte(&aBuf[0]);
    szPage = sqlite3Get4byte(&aBuf[8]);
    if( (magic&0xFFFFFFFE)!=WAL_MAGIC 
     || szPage&(szPage-1) 
     || szPage>SQLITE_MAX_PAGE_SIZE 
     || szPage<512 
    ){
      goto finished;
    }
    hdr.bigEndCksum = pWal->hdr.bigEndCksum = (magic&0x00000001);
    pWal->szPage = szPage;
    pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
    memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);

    /* Malloc a buffer to read frames into. */
    szFrame = szPage + WAL_FRAME_HDRSIZE;
    aFrame = (u8 *)sqlite3_malloc(szFrame);
................................................................................

  if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
    return 1;   /* Dirty read */
  }  
  if( h1.szPage==0 ){
    return 1;   /* Malformed header - probably all zeros */
  }
  walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
  if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){
    return 1;   /* Checksum does not match */
  }

  if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){
    *pChanged = 1;
    memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr));
................................................................................
  /* If this is the first frame written into the log, write the WAL
  ** header to the start of the WAL file. See comments at the top of
  ** this source file for a description of the WAL header format.
  */
  iFrame = pWal->hdr.mxFrame;
  if( iFrame==0 ){
    u8 aWalHdr[WAL_HDRSIZE];        /* Buffer to assembly wal-header in */
    sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN));
    sqlite3Put4byte(&aWalHdr[4], 3007000);
    sqlite3Put4byte(&aWalHdr[8], szPage);
    pWal->szPage = szPage;
    pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
    sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
    memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
    rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }

Changes to test/wal.test.

1284
1285
1286
1287
1288
1289
1290



1291

1292
1293
1294
1295
1296
1297
1298
1299
....
1343
1344
1345
1346
1347
1348
1349

1350
1351
1352
1353
1354
1355
1356
  set blob
}

proc logcksum {ckv1 ckv2 blob} {
  upvar $ckv1 c1
  upvar $ckv2 c2






  binary scan $blob I* values
  foreach {v1 v2} $values {
    set c1 [expr {($c1 + $v1 + $c2)&0xFFFFFFFF}]
    set c2 [expr {($c2 + $v2 + $c1)&0xFFFFFFFF}]
  }
}

file copy -force test.db testX.db
................................................................................
      set framebody [randomblob $pgsz]
      set framehdr  [binary format IIII $pg 5 22 23]
      set c1 0
      set c2 0
      logcksum c1 c2 $framehdr
      logcksum c1 c2 $framebody
      set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]

      set fd [open test.db-wal w]
      fconfigure $fd -encoding binary -translation binary
      puts -nonewline $fd $walhdr
      puts -nonewline $fd $framehdr
      puts -nonewline $fd $framebody
      close $fd
  







>
>
>
|
>
|







 







>







1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
....
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
  set blob
}

proc logcksum {ckv1 ckv2 blob} {
  upvar $ckv1 c1
  upvar $ckv2 c2

  set scanpattern I*
  if {$::tcl_platform(byteOrder) eq "littleEndian"} {
    set scanpattern i*
  }

  binary scan $blob $scanpattern values
  foreach {v1 v2} $values {
    set c1 [expr {($c1 + $v1 + $c2)&0xFFFFFFFF}]
    set c2 [expr {($c2 + $v2 + $c1)&0xFFFFFFFF}]
  }
}

file copy -force test.db testX.db
................................................................................
      set framebody [randomblob $pgsz]
      set framehdr  [binary format IIII $pg 5 22 23]
      set c1 0
      set c2 0
      logcksum c1 c2 $framehdr
      logcksum c1 c2 $framebody
      set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]

      set fd [open test.db-wal w]
      fconfigure $fd -encoding binary -translation binary
      puts -nonewline $fd $walhdr
      puts -nonewline $fd $framehdr
      puts -nonewline $fd $framebody
      close $fd
  

Added test/walcksum.test.















































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
# 2010 May 24
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl

ifcapable !wal {finish_test ; return }

# This proc calculates checksums in the same way as those used by SQLite 
# in WAL files. If the $endian argument is "big", then checksums are
# calculated by interpreting data as an array of big-endian integers. If
# it is "little", data is interpreted as an array of little-endian integers.
#
proc log_cksum {endian ckv1 ckv2 blob} {
  upvar $ckv1 c1
  upvar $ckv2 c2

  if {$endian!="big" && $endian!="little"} {
    return -error "Bad value \"$endian\" - must be \"big\" or \"little\""
  }
  set scanpattern I*
  if {$endian == "little"} { set scanpattern i* }

  binary scan $blob $scanpattern values
  foreach {v1 v2} $values {
    set c1 [expr {($c1 + $v1 + $c2)&0xFFFFFFFF}]
    set c2 [expr {($c2 + $v2 + $c1)&0xFFFFFFFF}]
  }
}

proc log_file_size {nFrame pgsz} {
  expr {24 + ($pgsz+24)*$nFrame}
}

# Read and return the contents of file $filename. Treat the content as
# binary data.
#
proc readfile {filename} {
  set fd [open $filename]
  fconfigure $fd -encoding binary
  fconfigure $fd -translation binary
  set data [read $fd]
  close $fd
  return $data
}

#
# File $filename must be a WAL file on disk. Check that the checksum of frame
# $iFrame in the file is correct when interpreting data as $endian-endian
# integers ($endian must be either "big" or "little"). If the checksum looks
# correct, return 1. Otherwise 0.
#
proc log_checksum_verify {filename iFrame endian} {
  set data [readfile $filename]
  set c1 0
  set c2 0
  
  binary scan [string range $data 8 11] I pgsz

  set n [log_file_size [expr $iFrame-1] $pgsz]
  binary scan [string range $data [expr $n+16] [expr $n+23]] II expect1 expect2
  log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
  log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]

  set expect1 [expr $expect1&0xFFFFFFFF]
  set expect2 [expr $expect2&0xFFFFFFFF]
  expr {$c1==$expect1 && $c2==$expect2}
}

#
# File $filename must be a WAL file on disk. Compute the checksum for frame
# $iFrame in the file by interpreting data as $endian-endian integers 
# ($endian must be either "big" or "little"). Then write the computed 
# checksum into the file.
#
proc log_checksum_write {filename iFrame endian} {
  set data [readfile $filename]
  set c1 0
  set c2 0
  
  binary scan [string range $data 8 11] I pgsz

  set n [log_file_size [expr $iFrame-1] $pgsz]
  log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
  log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]

  set bin [binary format II $c1 $c2]
  set fd [open $filename r+]
  fconfigure $fd -encoding binary
  fconfigure $fd -translation binary
  seek $fd [expr $n+16]
  puts -nonewline $fd $bin
  close $fd
}

#
# File $filename must be a WAL file on disk. Set the 'magic' field of the
# WAL header to indicate that checksums are $endian-endian ($endian must be
# either "big" or "little").
#
proc log_checksum_writemagic {filename endian} {
  set val [expr {0x377f0682 | ($endian == "big" ? 1 : 0)}]
  set bin [binary format I $val]
  set fd [open $filename r+]
  fconfigure $fd -encoding binary
  fconfigure $fd -translation binary
  puts -nonewline $fd $bin
  close $fd
}

#-------------------------------------------------------------------------
# Test cases walcksum-1.* attempt to verify the following:
#
#   * That both native and non-native order checksum log files can 
#      be recovered.
#
#   * That when appending to native or non-native checksum log files 
#     SQLite continues to use the right kind of checksums.
#
#   * Test point 2 when the appending process is not one that recovered
#     the log file.
#
#   * Test that both native and non-native checksum log files can be
#     checkpointed. And that after doing so the next write to the log
#     file occurs using native byte-order checksums. 
#
set native "big"
if {$::tcl_platform(byteOrder) == "littleEndian"} { set native "little" }
foreach endian {big little} {

  # Create a database. Leave some data in the log file.
  #
  do_test walcksum-1.$endian.1 {
    catch { db close }
    file delete -force test.db test.db-wal test.db-journal
    sqlite3 db test.db
    execsql {
      PRAGMA page_size = 1024;
      PRAGMA auto_vacuum = 0;
      PRAGMA synchronous = NORMAL;

      CREATE TABLE t1(a PRIMARY KEY, b);
      INSERT INTO t1 VALUES(1,  'one');
      INSERT INTO t1 VALUES(2,  'two');
      INSERT INTO t1 VALUES(3,  'three');
      INSERT INTO t1 VALUES(5,  'five');

      PRAGMA journal_mode = WAL;
      INSERT INTO t1 VALUES(8,  'eight');
      INSERT INTO t1 VALUES(13, 'thirteen');
      INSERT INTO t1 VALUES(21, 'twentyone');
    }

    file copy -force test.db test2.db
    file copy -force test.db-wal test2.db-wal
    db close

    list [file size test2.db] [file size test2.db-wal]
  } [list [expr 1024*3] [log_file_size 6 1024]]

  # Verify that the checksums are valid for all frames and that they
  # are calculated by interpreting data in native byte-order.
  #
  for {set f 1} {$f <= 6} {incr f} {
    do_test walcksum-1.$endian.2.$f {
      log_checksum_verify test2.db-wal $f $native
    } 1
  }

  # Replace all checksums in the current WAL file with $endian versions.
  # Then check that it is still possible to recover and read the database.
  #
  for {set f 1} {$f <= 6} {incr f} {
    do_test walcksum-1.$endian.3.$f {
      log_checksum_write test2.db-wal $f $endian
      log_checksum_verify test2.db-wal $f $endian
    } {1}
  }
  do_test walcksum-1.$endian.4.1 {
    log_checksum_writemagic test2.db-wal $endian
    file copy -force test2.db test.db
    file copy -force test2.db-wal test.db-wal
    sqlite3 db test.db
    execsql { SELECT a FROM t1 }
  } {1 2 3 5 8 13 21}

  # Following recovery, any frames written to the log should use the same 
  # endianness as the existing frames. Check that this is the case.
  #
  do_test walcksum-1.$endian.5.0 {
    execsql { 
      PRAGMA synchronous = NORMAL;
      INSERT INTO t1 VALUES(34, 'thirtyfour');
    }
    list [file size test.db] [file size test.db-wal]
  } [list [expr 1024*3] [log_file_size 8 1024]]
  for {set f 1} {$f <= 8} {incr f} {
    do_test walcksum-1.$endian.5.$f {
      log_checksum_verify test.db-wal $f $endian
    } {1}
  }

  # Now connect a second connection to the database. Check that this one
  # (not the one that did recovery) also appends frames to the log using
  # the same endianness for checksums as the existing frames.
  #
  do_test walcksum-1.$endian.6 {
    sqlite3 db2 test.db
    execsql { 
      PRAGMA integrity_check;
      SELECT a FROM t1;
    } db2
  } {ok 1 2 3 5 8 13 21 34}
  do_test walcksum-1.$endian.7.0 {
    execsql { 
      PRAGMA synchronous = NORMAL;
      INSERT INTO t1 VALUES(55, 'fiftyfive');
    } db2
    list [file size test.db] [file size test.db-wal]
  } [list [expr 1024*3] [log_file_size 10 1024]]
  for {set f 1} {$f <= 10} {incr f} {
    do_test walcksum-1.$endian.7.$f {
      log_checksum_verify test.db-wal $f $endian
    } {1}
  }

  # Now that both the recoverer and non-recoverer have added frames to the
  # log file, check that it can still be recovered.
  #
  file copy -force test.db test2.db
  file copy -force test.db-wal test2.db-wal
  do_test walcksum-1.$endian.7.11 {
    sqlite3 db3 test2.db
    execsql { 
      PRAGMA integrity_check;
      SELECT a FROM t1;
    } db3
  } {ok 1 2 3 5 8 13 21 34 55}
  db3 close

  # Run a checkpoint on the database file. Then, check that any frames written
  # to the start of the log use native byte-order checksums.
  #
  do_test walcksum-1.$endian.8.1 {
    execsql {
      PRAGMA wal_checkpoint;
      INSERT INTO t1 VALUES(89, 'eightynine');
    }
    log_checksum_verify test.db-wal 1 $native
  } {1}
  do_test walcksum-1.$endian.8.2 {
    log_checksum_verify test.db-wal 2 $native
  } {1}
  do_test walcksum-1.$endian.8.3 {
    log_checksum_verify test.db-wal 3 $native
  } [expr {$native == $endian}]

  do_test walcksum-1.$endian.9 {
    execsql { 
      PRAGMA integrity_check;
      SELECT a FROM t1;
    } db2
  } {ok 1 2 3 5 8 13 21 34 55 89}

  catch { db close }
  catch { db2 close }
}

finish_test