SQLite

Check-in [e98a7f87f9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix some problems with multi-file transactions in persistent journal mode. (CVS 5102)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e98a7f87f91c62676f94ad5a0c4980ab929ca79d
User & Date: danielk1977 2008-05-07 19:11:03.000
Context
2008-05-08
01:11
Fix the new ioerr4.test so that it plays well with others. (CVS 5103) (check-in: 75df2d3d50 user: drh tags: trunk)
2008-05-07
19:11
Fix some problems with multi-file transactions in persistent journal mode. (CVS 5102) (check-in: e98a7f87f9 user: danielk1977 tags: trunk)
18:59
Added test cases for corrupt SerialTypeLen header values, and additional check to improve detection of corrupt values. (CVS 5101) (check-in: 530c636061 user: shane tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/btree.c.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.456 2008/05/07 07:13:16 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
#include "btreeInt.h"












|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.457 2008/05/07 19:11:03 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
#include "btreeInt.h"

6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855

6856
6857
6858
6859
6860
6861
6862
    ** present in pTo before the copy operation, journal page i from pTo.
    */
    if( i!=iSkip && i<=nToPage ){
      DbPage *pDbPage = 0;
      rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage);
      if( rc==SQLITE_OK ){
        rc = sqlite3PagerWrite(pDbPage);
      }
      if( rc==SQLITE_OK && i>nFromPage ){
        /* Yeah.  It seems wierd to call DontWrite() right after Write(). But
        ** that is because the names of those procedures do not exactly 
        ** represent what they do.  Write() really means "put this page in the
        ** rollback journal and mark it as dirty so that it will be written
        ** to the database file later."  DontWrite() undoes the second part of
        ** that and prevents the page from being written to the database. The
        ** page is still on the rollback journal, though.  And that is the 
        ** whole point of this block: to put pages on the rollback journal. 
        */
        sqlite3PagerDontWrite(pDbPage);
      }
      sqlite3PagerUnref(pDbPage);

    }

    /* Overwrite the data in page i of the target database */
    if( rc==SQLITE_OK && i!=iSkip && i<=nNewPage ){

      DbPage *pToPage = 0;
      sqlite3_int64 iOff;







<
|
|
|
|
|
|
|
|
|
|
|
|
|
>







6835
6836
6837
6838
6839
6840
6841

6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
    ** present in pTo before the copy operation, journal page i from pTo.
    */
    if( i!=iSkip && i<=nToPage ){
      DbPage *pDbPage = 0;
      rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage);
      if( rc==SQLITE_OK ){
        rc = sqlite3PagerWrite(pDbPage);

        if( rc==SQLITE_OK && i>nFromPage ){
          /* Yeah.  It seems wierd to call DontWrite() right after Write(). But
          ** that is because the names of those procedures do not exactly 
          ** represent what they do.  Write() really means "put this page in the
          ** rollback journal and mark it as dirty so that it will be written
          ** to the database file later."  DontWrite() undoes the second part of
          ** that and prevents the page from being written to the database. The
          ** page is still on the rollback journal, though.  And that is the 
          ** whole point of this block: to put pages on the rollback journal. 
          */
          sqlite3PagerDontWrite(pDbPage);
        }
        sqlite3PagerUnref(pDbPage);
      }
    }

    /* Overwrite the data in page i of the target database */
    if( rc==SQLITE_OK && i!=iSkip && i<=nNewPage ){

      DbPage *pToPage = 0;
      sqlite3_int64 iOff;
Changes to src/pager.c.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.442 2008/05/07 12:45:41 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.443 2008/05/07 19:11:03 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
964
965
966
967
968
969
970
971
972
973
974

975



976

977
978

979
980
981
982
983
984
985
}

/*
** Write zeros over the header of the journal file.  This has the
** effect of invalidating the journal file and committing the
** transaction.
*/
static int zeroJournalHdr(Pager *pPager){
  int rc;
  static const char zeroHdr[28];


  IOTRACE(("JZEROHDR %p\n", pPager))



  rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0);

  if( rc==SQLITE_OK ){
    rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY | pPager->sync_flags);

  }
  return rc;
}

/*
** The journal file must be open when this routine is called. A journal
** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the







|
|


>
|
>
>
>
|
>
|
|
>







964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
}

/*
** Write zeros over the header of the journal file.  This has the
** effect of invalidating the journal file and committing the
** transaction.
*/
static int zeroJournalHdr(Pager *pPager, int doTruncate){
  int rc = SQLITE_OK;
  static const char zeroHdr[28];

  if( pPager->journalOff ){
    IOTRACE(("JZEROHDR %p\n", pPager))
    if( doTruncate ){
      rc = sqlite3OsTruncate(pPager->jfd, 0);
    }else{
      rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0);
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->sync_flags);
    }
  }
  return rc;
}

/*
** The journal file must be open when this routine is called. A journal
** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the
1159
1160
1161
1162
1163
1164
1165

1166
1167
1168
1169
1170
1171
1172
** this call is a no-op.
*/
static int writeMasterJournal(Pager *pPager, const char *zMaster){
  int rc;
  int len; 
  int i; 
  i64 jrnlOff;

  u32 cksum = 0;
  char zBuf[sizeof(aJournalMagic)+2*4];

  if( !zMaster || pPager->setMaster) return SQLITE_OK;
  pPager->setMaster = 1;

  len = strlen(zMaster);







>







1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
** this call is a no-op.
*/
static int writeMasterJournal(Pager *pPager, const char *zMaster){
  int rc;
  int len; 
  int i; 
  i64 jrnlOff;
  i64 jrnlSize;
  u32 cksum = 0;
  char zBuf[sizeof(aJournalMagic)+2*4];

  if( !zMaster || pPager->setMaster) return SQLITE_OK;
  pPager->setMaster = 1;

  len = strlen(zMaster);
1192
1193
1194
1195
1196
1197
1198

1199

















1200
1201
1202
1203
1204
1205
1206
  if( rc!=SQLITE_OK ) return rc;
  jrnlOff += len;

  put32bits(zBuf, len);
  put32bits(&zBuf[4], cksum);
  memcpy(&zBuf[8], aJournalMagic, sizeof(aJournalMagic));
  rc = sqlite3OsWrite(pPager->jfd, zBuf, 8+sizeof(aJournalMagic), jrnlOff);

  pPager->needSync = !pPager->noSync;

















  return rc;
}

/*
** Add or remove a page from the list of all pages that are in the
** statement journal.
**







>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
  if( rc!=SQLITE_OK ) return rc;
  jrnlOff += len;

  put32bits(zBuf, len);
  put32bits(&zBuf[4], cksum);
  memcpy(&zBuf[8], aJournalMagic, sizeof(aJournalMagic));
  rc = sqlite3OsWrite(pPager->jfd, zBuf, 8+sizeof(aJournalMagic), jrnlOff);
  jrnlOff += 8+sizeof(aJournalMagic);
  pPager->needSync = !pPager->noSync;

  /* If the pager is in peristent-journal mode, then the physical 
  ** journal-file may extend past the end of the master-journal name
  ** and 8 bytes of magic data just written to the file. This is 
  ** dangerous because the code to rollback a hot-journal file
  ** will not be able to find the master-journal name to determine 
  ** whether or not the journal is hot. 
  **
  ** Easiest thing to do in this scenario is to truncate the journal 
  ** file to the required size.
  */ 
  if( (rc==SQLITE_OK)
   && (rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize))==SQLITE_OK
   && jrnlSize>jrnlOff
  ){
    rc = sqlite3OsTruncate(pPager->jfd, jrnlOff);
  }
  return rc;
}

/*
** Add or remove a page from the list of all pages that are in the
** statement journal.
**
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
**
** The journal file is either deleted or truncated.
**
** TODO: Consider keeping the journal file open for temporary databases.
** This might give a performance improvement on windows where opening
** a file is an expensive operation.
*/
static int pager_end_transaction(Pager *pPager){
  PgHdr *pPg;
  int rc = SQLITE_OK;
  int rc2 = SQLITE_OK;
  assert( !MEMDB );
  if( pPager->state<PAGER_RESERVED ){
    return SQLITE_OK;
  }
  sqlite3PagerStmtCommit(pPager);
  if( pPager->stmtOpen && !pPager->exclusiveMode ){
    sqlite3OsClose(pPager->stfd);
    pPager->stmtOpen = 0;
  }
  if( pPager->journalOpen ){
    if( (pPager->exclusiveMode ||
         pPager->journalMode==PAGER_JOURNALMODE_PERSIST) 
       && (rc = zeroJournalHdr(pPager))==SQLITE_OK ){
      pPager->journalOff = 0;
      pPager->journalStarted = 0;
    }else{
      sqlite3OsClose(pPager->jfd);
      pPager->journalOpen = 0;
      if( rc==SQLITE_OK ){
        rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0);







|















|







1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
**
** The journal file is either deleted or truncated.
**
** TODO: Consider keeping the journal file open for temporary databases.
** This might give a performance improvement on windows where opening
** a file is an expensive operation.
*/
static int pager_end_transaction(Pager *pPager, int hasMaster){
  PgHdr *pPg;
  int rc = SQLITE_OK;
  int rc2 = SQLITE_OK;
  assert( !MEMDB );
  if( pPager->state<PAGER_RESERVED ){
    return SQLITE_OK;
  }
  sqlite3PagerStmtCommit(pPager);
  if( pPager->stmtOpen && !pPager->exclusiveMode ){
    sqlite3OsClose(pPager->stfd);
    pPager->stmtOpen = 0;
  }
  if( pPager->journalOpen ){
    if( (pPager->exclusiveMode ||
         pPager->journalMode==PAGER_JOURNALMODE_PERSIST) 
       && (rc = zeroJournalHdr(pPager, hasMaster))==SQLITE_OK ){
      pPager->journalOff = 0;
      pPager->journalStarted = 0;
    }else{
      sqlite3OsClose(pPager->jfd);
      pPager->journalOpen = 0;
      if( rc==SQLITE_OK ){
        rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0);
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918

end_playback:
  if( rc==SQLITE_OK ){
    zMaster = pPager->pTmpSpace;
    rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
  }
  if( rc==SQLITE_OK ){
    rc = pager_end_transaction(pPager);
  }
  if( rc==SQLITE_OK && zMaster[0] ){
    /* If there was a master journal and this routine will return success,
    ** see if it is possible to delete the master journal.
    */
    rc = pager_delmaster(pPager, zMaster);
  }







|







1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943

end_playback:
  if( rc==SQLITE_OK ){
    zMaster = pPager->pTmpSpace;
    rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
  }
  if( rc==SQLITE_OK ){
    rc = pager_end_transaction(pPager, zMaster[0]!='\0');
  }
  if( rc==SQLITE_OK && zMaster[0] ){
    /* If there was a master journal and this routine will return success,
    ** see if it is possible to delete the master journal.
    */
    rc = pager_delmaster(pPager, zMaster);
  }
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970

  rc = writeJournalHdr(pPager);

  if( pPager->stmtAutoopen && rc==SQLITE_OK ){
    rc = sqlite3PagerStmtBegin(pPager);
  }
  if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && rc!=SQLITE_IOERR_NOMEM ){
    rc = pager_end_transaction(pPager);
    if( rc==SQLITE_OK ){
      rc = SQLITE_FULL;
    }
  }
  return rc;

failed_to_open_journal:







|







3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995

  rc = writeJournalHdr(pPager);

  if( pPager->stmtAutoopen && rc==SQLITE_OK ){
    rc = sqlite3PagerStmtBegin(pPager);
  }
  if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && rc!=SQLITE_IOERR_NOMEM ){
    rc = pager_end_transaction(pPager, 0);
    if( rc==SQLITE_OK ){
      rc = SQLITE_FULL;
    }
  }
  return rc;

failed_to_open_journal:
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
#endif
    pPager->pStmt = 0;
    pPager->state = PAGER_SHARED;
    pagerLeave(pPager);
    return SQLITE_OK;
  }
  assert( pPager->state==PAGER_SYNCED || !pPager->dirtyCache );
  rc = pager_end_transaction(pPager);
  rc = pager_error(pPager, rc);
  pagerLeave(pPager);
  return rc;
}

/*
** Rollback all changes.  The database falls back to PAGER_SHARED mode.







|







4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
#endif
    pPager->pStmt = 0;
    pPager->state = PAGER_SHARED;
    pagerLeave(pPager);
    return SQLITE_OK;
  }
  assert( pPager->state==PAGER_SYNCED || !pPager->dirtyCache );
  rc = pager_end_transaction(pPager, pPager->setMaster);
  rc = pager_error(pPager, rc);
  pagerLeave(pPager);
  return rc;
}

/*
** Rollback all changes.  The database falls back to PAGER_SHARED mode.
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
    pPager->stmtInUse = 0;
    pPager->state = PAGER_SHARED;
    return SQLITE_OK;
  }

  pagerEnter(pPager);
  if( !pPager->dirtyCache || !pPager->journalOpen ){
    rc = pager_end_transaction(pPager);
    pagerLeave(pPager);
    return rc;
  }

  if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    if( pPager->state>=PAGER_EXCLUSIVE ){
      pager_playback(pPager, 0);
    }
    pagerLeave(pPager);
    return pPager->errCode;
  }
  if( pPager->state==PAGER_RESERVED ){
    int rc2;
    rc = pager_playback(pPager, 0);
    rc2 = pager_end_transaction(pPager);
    if( rc==SQLITE_OK ){
      rc = rc2;
    }
  }else{
    rc = pager_playback(pPager, 0);
  }
  /* pager_reset(pPager); */







|














|







4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
    pPager->stmtInUse = 0;
    pPager->state = PAGER_SHARED;
    return SQLITE_OK;
  }

  pagerEnter(pPager);
  if( !pPager->dirtyCache || !pPager->journalOpen ){
    rc = pager_end_transaction(pPager, pPager->setMaster);
    pagerLeave(pPager);
    return rc;
  }

  if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){
    if( pPager->state>=PAGER_EXCLUSIVE ){
      pager_playback(pPager, 0);
    }
    pagerLeave(pPager);
    return pPager->errCode;
  }
  if( pPager->state==PAGER_RESERVED ){
    int rc2;
    rc = pager_playback(pPager, 0);
    rc2 = pager_end_transaction(pPager, pPager->setMaster);
    if( rc==SQLITE_OK ){
      rc = rc2;
    }
  }else{
    rc = pager_playback(pPager, 0);
  }
  /* pager_reset(pPager); */
Changes to test/jrnlmode.test.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 2008 April 17
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. The focus
# of these tests is the journal mode pragma.
#
# $Id: jrnlmode.test,v 1.1 2008/04/19 20:34:19 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable {!pager_pragmas} {
  finish_test
  return













|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 2008 April 17
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. The focus
# of these tests is the journal mode pragma.
#
# $Id: jrnlmode.test,v 1.2 2008/05/07 19:11:03 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable {!pager_pragmas} {
  finish_test
  return
147
148
149
150
151
152
153
154









155


































156
    execsql {
      DETACH aux1;
      DETACH aux2;
      DETACH aux3;
    }
  } {}
}













































finish_test








>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    execsql {
      DETACH aux1;
      DETACH aux2;
      DETACH aux3;
    }
  } {}
}

ifcapable attach {
  file delete -force test2.db
  do_test jrnlmode-2.1 {
    execsql {
      ATTACH 'test2.db' AS aux;
      PRAGMA main.journal_mode = persist;
      PRAGMA aux.journal_mode = persist;
      CREATE TABLE abc(a, b, c);
      CREATE TABLE aux.def(d, e, f);
    }
    execsql {
      BEGIN;
      INSERT INTO abc VALUES(1, 2, 3);
      INSERT INTO def VALUES(4, 5, 6);
      COMMIT;
    }
    list [file exists test.db-journal] [file exists test2.db-journal]
  } {1 1}

  do_test jrnlmode-2.2 {
    file size test.db-journal
  } {0}

  do_test jrnlmode-2.3 {
    execsql {
      SELECT * FROM abc;
    }
  } {1 2 3}

  do_test jrnlmode-2.4 {
    file size test.db-journal
  } {0}

  do_test jrnlmode-2.5 {
    execsql {
      SELECT * FROM def;
    }
  } {4 5 6}

}




finish_test