SQLite

Check-in [a5ecffcf02]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid writing the 8-byte journal-header magic until the journal-header is synced. In persistent journal-mode, this prevents any old content that follows an unsynced journal-header from being interpreted as part of the rollback journal. (CVS 6817)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: a5ecffcf025da2fcb241e83c7bebc1095a3b51d6
User & Date: danielk1977 2009-06-26 07:12:07.000
Context
2009-06-26
09:01
Update test_journal.c to account for (6817). Changes to test code only. (CVS 6818) (check-in: 542ee8cced user: danielk1977 tags: trunk)
07:12
Avoid writing the 8-byte journal-header magic until the journal-header is synced. In persistent journal-mode, this prevents any old content that follows an unsynced journal-header from being interpreted as part of the rollback journal. (CVS 6817) (check-in: a5ecffcf02 user: danielk1977 tags: trunk)
2009-06-25
16:11
Fix a problem with a return code being ignored in insertCell(). (CVS 6816) (check-in: bb5f1c0143 user: danielk1977 tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/pager.c.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.601 2009/06/22 05:43:24 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"

/*
** Macros for troubleshooting.  Normally turned off
*/







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.602 2009/06/26 07:12:07 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"

/*
** Macros for troubleshooting.  Normally turned off
*/
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
  for(ii=0; ii<pPager->nSavepoint; ii++){
    if( pPager->aSavepoint[ii].iHdrOffset==0 ){
      pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff;
    }
  }

  pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager);
  memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic));

  /* 
  ** Write the nRec Field - the number of page records that follow this
  ** journal header. Normally, zero is written to this value at this time.
  ** After the records are added to the journal (and the journal synced, 
  ** if in full-sync mode), the zero is overwritten with the true number
  ** of records (see syncJournal()).







<







753
754
755
756
757
758
759

760
761
762
763
764
765
766
  for(ii=0; ii<pPager->nSavepoint; ii++){
    if( pPager->aSavepoint[ii].iHdrOffset==0 ){
      pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff;
    }
  }

  pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager);


  /* 
  ** Write the nRec Field - the number of page records that follow this
  ** journal header. Normally, zero is written to this value at this time.
  ** After the records are added to the journal (and the journal synced, 
  ** if in full-sync mode), the zero is overwritten with the true number
  ** of records (see syncJournal()).
779
780
781
782
783
784
785

786
787

788
789
790
791
792
793
794
  **   * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees
  **     that garbage data is never appended to the journal file.
  */
  assert( isOpen(pPager->fd) || pPager->noSync );
  if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY)
   || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) 
  ){

    put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff);
  }else{

    put32bits(&zHeader[sizeof(aJournalMagic)], 0);
  }

  /* The random check-hash initialiser */ 
  sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit);
  put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit);
  /* The initial database size */







>


>







778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
  **   * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees
  **     that garbage data is never appended to the journal file.
  */
  assert( isOpen(pPager->fd) || pPager->noSync );
  if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY)
   || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) 
  ){
    memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic));
    put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff);
  }else{
    zHeader[0] = '\0';
    put32bits(&zHeader[sizeof(aJournalMagic)], 0);
  }

  /* The random check-hash initialiser */ 
  sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit);
  put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit);
  /* The initial database size */
848
849
850
851
852
853
854

855
856
857
858
859
860
861
**
** If the journal header file appears to be corrupted, SQLITE_DONE is
** returned and *pNRec and *PDbSize are undefined.  If JOURNAL_HDR_SZ bytes
** cannot be read from the journal file an error code is returned.
*/
static int readJournalHdr(
  Pager *pPager,               /* Pager object */

  i64 journalSize,             /* Size of the open journal file in bytes */
  u32 *pNRec,                  /* OUT: Value read from the nRec field */
  u32 *pDbSize                 /* OUT: Value of original database size field */
){
  int rc;                      /* Return code */
  unsigned char aMagic[8];     /* A buffer to hold the magic header */
  i64 iHdrOff;                 /* Offset of journal header being read */







>







849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
**
** If the journal header file appears to be corrupted, SQLITE_DONE is
** returned and *pNRec and *PDbSize are undefined.  If JOURNAL_HDR_SZ bytes
** cannot be read from the journal file an error code is returned.
*/
static int readJournalHdr(
  Pager *pPager,               /* Pager object */
  int isHot,
  i64 journalSize,             /* Size of the open journal file in bytes */
  u32 *pNRec,                  /* OUT: Value read from the nRec field */
  u32 *pDbSize                 /* OUT: Value of original database size field */
){
  int rc;                      /* Return code */
  unsigned char aMagic[8];     /* A buffer to hold the magic header */
  i64 iHdrOff;                 /* Offset of journal header being read */
873
874
875
876
877
878
879

880
881
882
883
884
885

886
887
888
889
890
891
892
  iHdrOff = pPager->journalOff;

  /* Read in the first 8 bytes of the journal header. If they do not match
  ** the  magic string found at the start of each journal header, return
  ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise,
  ** proceed.
  */

  rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff);
  if( rc ){
    return rc;
  }
  if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){
    return SQLITE_DONE;

  }

  /* Read the first three 32-bit fields of the journal header: The nRec
  ** field, the checksum-initializer and the database size at the start
  ** of the transaction. Return an error code if anything goes wrong.
  */
  if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec))







>
|
|
|
|
|
|
>







875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
  iHdrOff = pPager->journalOff;

  /* Read in the first 8 bytes of the journal header. If they do not match
  ** the  magic string found at the start of each journal header, return
  ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise,
  ** proceed.
  */
  if( isHot || iHdrOff!=pPager->journalHdr ){
    rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff);
    if( rc ){
      return rc;
    }
    if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){
      return SQLITE_DONE;
    }
  }

  /* Read the first three 32-bit fields of the journal header: The nRec
  ** field, the checksum-initializer and the database size at the start
  ** of the transaction. Return an error code if anything goes wrong.
  */
  if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec))
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
    int isUnsync = 0;

    /* Read the next journal header from the journal file.  If there are
    ** not enough bytes left in the journal file for a complete header, or
    ** it is corrupted, then a process must of failed while writing it.
    ** This indicates nothing more needs to be rolled back.
    */
    rc = readJournalHdr(pPager, szJ, &nRec, &mxPg);
    if( rc!=SQLITE_OK ){ 
      if( rc==SQLITE_DONE ){
        rc = SQLITE_OK;
      }
      goto end_playback;
    }








|







1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
    int isUnsync = 0;

    /* Read the next journal header from the journal file.  If there are
    ** not enough bytes left in the journal file for a complete header, or
    ** it is corrupted, then a process must of failed while writing it.
    ** This indicates nothing more needs to be rolled back.
    */
    rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg);
    if( rc!=SQLITE_OK ){ 
      if( rc==SQLITE_DONE ){
        rc = SQLITE_OK;
      }
      goto end_playback;
    }

2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
  ** of the main journal file.  Continue to skip out-of-range pages and
  ** continue adding pages rolled back to pDone.
  */
  while( rc==SQLITE_OK && pPager->journalOff<szJ ){
    u32 ii;            /* Loop counter */
    u32 nJRec = 0;     /* Number of Journal Records */
    u32 dummy;
    rc = readJournalHdr(pPager, szJ, &nJRec, &dummy);
    assert( rc!=SQLITE_DONE );

    /*
    ** The "pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff"
    ** test is related to ticket #2565.  See the discussion in the
    ** pager_playback() function for additional information.
    */







|







2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
  ** of the main journal file.  Continue to skip out-of-range pages and
  ** continue adding pages rolled back to pDone.
  */
  while( rc==SQLITE_OK && pPager->journalOff<szJ ){
    u32 ii;            /* Loop counter */
    u32 nJRec = 0;     /* Number of Journal Records */
    u32 dummy;
    rc = readJournalHdr(pPager, 0, szJ, &nJRec, &dummy);
    assert( rc!=SQLITE_DONE );

    /*
    ** The "pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff"
    ** test is related to ticket #2565.  See the discussion in the
    ** pager_playback() function for additional information.
    */
2760
2761
2762
2763
2764
2765
2766
2767


2768
2769
2770
2771
2772
2773
2774
      assert( isOpen(pPager->jfd) );

      if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){
        /* Variable iNRecOffset is set to the offset in the journal file
        ** of the nRec field of the most recently written journal header.
        ** This field will be updated following the xSync() operation
        ** on the journal file. */
        i64 iNRecOffset = pPager->journalHdr + sizeof(aJournalMagic);



        /* This block deals with an obscure problem. If the last connection
        ** that wrote to this database was operating in persistent-journal
        ** mode, then the journal file may at this point actually be larger
        ** than Pager.journalOff bytes. If the next thing in the journal
        ** file happens to be a journal-header (written as part of the
        ** previous connections transaction), and a crash or power-failure 







|
>
>







2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
      assert( isOpen(pPager->jfd) );

      if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){
        /* Variable iNRecOffset is set to the offset in the journal file
        ** of the nRec field of the most recently written journal header.
        ** This field will be updated following the xSync() operation
        ** on the journal file. */
	u8 zHeader[sizeof(aJournalMagic)+4];
	memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic));
	put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec);

        /* This block deals with an obscure problem. If the last connection
        ** that wrote to this database was operating in persistent-journal
        ** mode, then the journal file may at this point actually be larger
        ** than Pager.journalOff bytes. If the next thing in the journal
        ** file happens to be a journal-header (written as part of the
        ** previous connections transaction), and a crash or power-failure 
2813
2814
2815
2816
2817
2818
2819

2820

2821
2822
2823
2824
2825
2826
2827
        if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
          PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
          IOTRACE(("JSYNC %p\n", pPager))
          rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags);
          if( rc!=SQLITE_OK ) return rc;
        }
        IOTRACE(("JHDR %p %lld %d\n", pPager, iNRecOffset, 4));

        rc = write32bits(pPager->jfd, iNRecOffset, pPager->nRec);

        if( rc!=SQLITE_OK ) return rc;
      }
      if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
        PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
        IOTRACE(("JSYNC %p\n", pPager))
        rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags| 
          (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)







>
|
>







2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
        if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
          PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
          IOTRACE(("JSYNC %p\n", pPager))
          rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags);
          if( rc!=SQLITE_OK ) return rc;
        }
        IOTRACE(("JHDR %p %lld %d\n", pPager, iNRecOffset, 4));
        rc = sqlite3OsWrite(
            pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr
	);
        if( rc!=SQLITE_OK ) return rc;
      }
      if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
        PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
        IOTRACE(("JSYNC %p\n", pPager))
        rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags| 
          (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)
Changes to test/rollback.test.
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is verifying that a rollback in one statement
# caused by an ON CONFLICT ROLLBACK clause aborts any other pending
# statements.
#
# $Id: rollback.test,v 1.10 2008/10/17 18:51:53 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

set DB [sqlite3_connection_pointer db]

do_test rollback-1.1 {







|







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is verifying that a rollback in one statement
# caused by an ON CONFLICT ROLLBACK clause aborts any other pending
# statements.
#
# $Id: rollback.test,v 1.11 2009/06/26 07:12:07 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

set DB [sqlite3_connection_pointer db]

do_test rollback-1.1 {
110
111
112
113
114
115
116







117
118
119
120
121
122
123
    "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7"
  ]
  set iOffset [expr (([file size testA.db-journal] + 511)/512)*512]
  set fd [open testA.db-journal a+]
  fconfigure $fd -encoding binary -translation binary
  seek $fd $iOffset
  puts -nonewline $fd $zAppend







  close $fd

  # Open a handle on testA.db and use it to query the database. At one
  # point the first query would attempt a hot rollback, attempt to open
  # the master-journal file and return SQLITE_CANTOPEN when it could not
  # be opened. This is incorrect, it should simply delete the journal
  # file and proceed with the query.







>
>
>
>
>
>
>







110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
    "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7"
  ]
  set iOffset [expr (([file size testA.db-journal] + 511)/512)*512]
  set fd [open testA.db-journal a+]
  fconfigure $fd -encoding binary -translation binary
  seek $fd $iOffset
  puts -nonewline $fd $zAppend

  # Also, fix the first journal-header in the journal-file. Because the
  # journal file has not yet been synced, the 8-byte magic string at the
  # start of the first journal-header has not been written by SQLite.
  # So write it now.
  seek $fd 0
  puts -nonewline $fd "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7"
  close $fd

  # Open a handle on testA.db and use it to query the database. At one
  # point the first query would attempt a hot rollback, attempt to open
  # the master-journal file and return SQLITE_CANTOPEN when it could not
  # be opened. This is incorrect, it should simply delete the journal
  # file and proceed with the query.
Changes to test/tkt3457.test.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 2008 October 29
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# $Id: tkt3457.test,v 1.2 2009/06/05 17:09:12 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

if {$tcl_platform(platform) != "unix"} {
  finish_test
  return












|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 2008 October 29
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# $Id: tkt3457.test,v 1.3 2009/06/26 07:12:07 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

if {$tcl_platform(platform) != "unix"} {
  finish_test
  return
44
45
46
47
48
49
50










51
52
53
54
55
56
57
    INSERT INTO t1 VALUES(1, 2, 3);
    BEGIN;
    INSERT INTO t1 VALUES(4, 5, 6);
  }

  file copy -force test.db bak.db
  file copy -force test.db-journal bak.db-journal











  execsql COMMIT
} {}

do_test tkt3457-1.2 {
  file copy -force bak.db-journal test.db-journal
  file attributes test.db-journal -permissions ---------







>
>
>
>
>
>
>
>
>
>







44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    INSERT INTO t1 VALUES(1, 2, 3);
    BEGIN;
    INSERT INTO t1 VALUES(4, 5, 6);
  }

  file copy -force test.db bak.db
  file copy -force test.db-journal bak.db-journal

  # Fix the first journal-header in the journal-file. Because the
  # journal file has not yet been synced, the 8-byte magic string at the
  # start of the first journal-header has not been written by SQLite.
  # So write it now.
  set fd [open bak.db-journal a+]
  fconfigure $fd -encoding binary -translation binary
  seek $fd 0
  puts -nonewline $fd "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7"
  close $fd

  execsql COMMIT
} {}

do_test tkt3457-1.2 {
  file copy -force bak.db-journal test.db-journal
  file attributes test.db-journal -permissions ---------