SQLite

Check-in [6806b9ecb5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Make sure that the database size cache in the pager is invalidated whenever the database is unlocked. A stale value in the database size cache can result in database corruption on a heavily loaded system running autovacuum. (CVS 3548)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 6806b9ecb5e3b90e793c5862404e76485df33b25
User & Date: drh 2007-01-03 15:34:30.000
Context
2007-01-03
23:36
Additional changes in the pager and os interface layers to fix problems that might be contributing to recently observed database corruption. (CVS 3549) (check-in: a593d5743e user: drh tags: trunk)
15:34
Make sure that the database size cache in the pager is invalidated whenever the database is unlocked. A stale value in the database size cache can result in database corruption on a heavily loaded system running autovacuum. (CVS 3548) (check-in: 6806b9ecb5 user: drh tags: trunk)
2007-01-02
18:41
Update the vtab_err test to check for -DSQLITE_MEMDEBUG=1 and skip the tests if missing. Pager is more careful to clear its file size cache. Remove an assert() in VDBE that might fail on a corrupt database file. (CVS 3547) (check-in: bf1afd016a user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/pager.c.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.278 2007/01/02 18:41:55 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.279 2007/01/03 15:34:30 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
845
846
847
848
849
850
851
















852
853
854
855
856
857
858
  if( pPager->aHash==0 ) return 0;
  p = pPager->aHash[pgno & (pPager->nHash-1)];
  while( p && p->pgno!=pgno ){
    p = p->pNextHash;
  }
  return p;
}

















/*
** Unlock the database and clear the in-memory cache.  This routine
** sets the state of the pager back to what it was when it was first
** opened.  Any outstanding pages are invalidated and subsequent attempts
** to access those pages will likely result in a coredump.
*/







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
  if( pPager->aHash==0 ) return 0;
  p = pPager->aHash[pgno & (pPager->nHash-1)];
  while( p && p->pgno!=pgno ){
    p = p->pNextHash;
  }
  return p;
}

/*
** Unlock the database file.
**
** Once all locks have been removed from the database file, other
** processes or threads might change the file.  So make sure all of
** our internal cache is invalidated.
*/
static void pager_unlock(Pager *pPager){
  if( !MEMDB ){
    sqlite3OsUnlock(pPager->fd, NO_LOCK);
    pPager->dbSize = -1;
  }
  pPager->state = PAGER_UNLOCK;
}


/*
** Unlock the database and clear the in-memory cache.  This routine
** sets the state of the pager back to what it was when it was first
** opened.  Any outstanding pages are invalidated and subsequent attempts
** to access those pages will likely result in a coredump.
*/
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
  pPager->nHash = 0;
  sqliteFree(pPager->aHash);
  pPager->nPage = 0;
  pPager->aHash = 0;
  if( pPager->state>=PAGER_RESERVED ){
    sqlite3pager_rollback(pPager);
  }
  sqlite3OsUnlock(pPager->fd, NO_LOCK);
  pPager->state = PAGER_UNLOCK;
  pPager->dbSize = -1;
  pPager->nRef = 0;
  assert( pPager->journalOpen==0 );
}

/*
** When this routine is called, the pager has the journal file open and
** a RESERVED or EXCLUSIVE lock on the database.  This routine releases







|
<
<







886
887
888
889
890
891
892
893


894
895
896
897
898
899
900
  pPager->nHash = 0;
  sqliteFree(pPager->aHash);
  pPager->nPage = 0;
  pPager->aHash = 0;
  if( pPager->state>=PAGER_RESERVED ){
    sqlite3pager_rollback(pPager);
  }
  pager_unlock(pPager);


  pPager->nRef = 0;
  assert( pPager->journalOpen==0 );
}

/*
** When this routine is called, the pager has the journal file open and
** a RESERVED or EXCLUSIVE lock on the database.  This routine releases
1421
1422
1423
1424
1425
1426
1427

1428
1429
1430
1431
1432
1433
1434
  }
  
  /* Truncate the database back to its original size.
  */
  if( pPager->state>=PAGER_EXCLUSIVE ){
    rc = pager_truncate(pPager, pPager->stmtSize);
  }

  pPager->dbSize = pPager->stmtSize;

  /* Figure out how many records are in the statement journal.
  */
  assert( pPager->stmtInUse && pPager->journalOpen );
  sqlite3OsSeek(pPager->stfd, 0);
  nRec = pPager->stmtNRec;







>







1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
  }
  
  /* Truncate the database back to its original size.
  */
  if( pPager->state>=PAGER_EXCLUSIVE ){
    rc = pager_truncate(pPager, pPager->stmtSize);
  }
  assert( pPager->state>=PAGER_SHARED );
  pPager->dbSize = pPager->stmtSize;

  /* Figure out how many records are in the statement journal.
  */
  assert( pPager->stmtInUse && pPager->journalOpen );
  sqlite3OsSeek(pPager->stfd, 0);
  nRec = pPager->stmtNRec;
1970
1971
1972
1973
1974
1975
1976


1977
1978
1979




1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997

1998
1999
2000
2001
2002
2003
2004
** false or until the lock succeeds.
**
** Return SQLITE_OK on success and an error code if we cannot obtain
** the lock.
*/
static int pager_wait_on_lock(Pager *pPager, int locktype){
  int rc;


  assert( PAGER_SHARED==SHARED_LOCK );
  assert( PAGER_RESERVED==RESERVED_LOCK );
  assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK );




  if( pPager->state>=locktype ){
    rc = SQLITE_OK;
  }else{
    do {
      rc = sqlite3OsLock(pPager->fd, locktype);
    }while( rc==SQLITE_BUSY && sqlite3InvokeBusyHandler(pPager->pBusyHandler) );
    if( rc==SQLITE_OK ){
      pPager->state = locktype;
    }
  }
  return rc;
}

/*
** Truncate the file to the number of pages specified.
*/
int sqlite3pager_truncate(Pager *pPager, Pgno nPage){
  int rc;

  sqlite3pager_pagecount(pPager);
  if( pPager->errCode ){
    rc = pPager->errCode;
    return rc;
  }
  if( nPage>=(unsigned)pPager->dbSize ){
    return SQLITE_OK;







>
>



>
>
>
>


















>







1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
** false or until the lock succeeds.
**
** Return SQLITE_OK on success and an error code if we cannot obtain
** the lock.
*/
static int pager_wait_on_lock(Pager *pPager, int locktype){
  int rc;

  /* The OS lock values must be the same as the Pager lock values */
  assert( PAGER_SHARED==SHARED_LOCK );
  assert( PAGER_RESERVED==RESERVED_LOCK );
  assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK );

  /* If the file is currently unlocked then the size must be unknown */
  assert( pPager->state>=PAGER_SHARED || pPager->dbSize<0 || MEMDB );

  if( pPager->state>=locktype ){
    rc = SQLITE_OK;
  }else{
    do {
      rc = sqlite3OsLock(pPager->fd, locktype);
    }while( rc==SQLITE_BUSY && sqlite3InvokeBusyHandler(pPager->pBusyHandler) );
    if( rc==SQLITE_OK ){
      pPager->state = locktype;
    }
  }
  return rc;
}

/*
** Truncate the file to the number of pages specified.
*/
int sqlite3pager_truncate(Pager *pPager, Pgno nPage){
  int rc;
  assert( pPager->state>=PAGER_SHARED || MEMDB );
  sqlite3pager_pagecount(pPager);
  if( pPager->errCode ){
    rc = pPager->errCode;
    return rc;
  }
  if( nPage>=(unsigned)pPager->dbSize ){
    return SQLITE_OK;
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
      /* We ignore any IO errors that occur during the rollback
      ** operation. So disable IO error simulation so that testing
      ** works more easily.
      */
      disable_simulated_io_errors();
      sqlite3pager_rollback(pPager);
      enable_simulated_io_errors();
      if( !MEMDB ){
        sqlite3OsUnlock(pPager->fd, NO_LOCK);
      }
      assert( pPager->errCode || pPager->journalOpen==0 );
      break;
    }
    case PAGER_SHARED: {
      if( !MEMDB ){
        sqlite3OsUnlock(pPager->fd, NO_LOCK);
      }
      break;
    }
    default: {
      /* Do nothing */
      break;
    }
  }







<
|
<




<
|
<







2082
2083
2084
2085
2086
2087
2088

2089

2090
2091
2092
2093

2094

2095
2096
2097
2098
2099
2100
2101
      /* We ignore any IO errors that occur during the rollback
      ** operation. So disable IO error simulation so that testing
      ** works more easily.
      */
      disable_simulated_io_errors();
      sqlite3pager_rollback(pPager);
      enable_simulated_io_errors();

      pager_unlock(pPager);

      assert( pPager->errCode || pPager->journalOpen==0 );
      break;
    }
    case PAGER_SHARED: {

      pager_unlock(pPager);

      break;
    }
    default: {
      /* Do nothing */
      break;
    }
  }
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
       ** 
       ** Because the intermediate RESERVED lock is not requested, the
       ** second process will get to this point in the code and fail to
       ** obtain it's own EXCLUSIVE lock on the database file.
       */
       rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK);
       if( rc!=SQLITE_OK ){
         sqlite3OsUnlock(pPager->fd, NO_LOCK);
         pPager->state = PAGER_UNLOCK;
         return pager_error(pPager, rc);
       }
       pPager->state = PAGER_EXCLUSIVE;

       /* Open the journal for reading only.  Return SQLITE_BUSY if
       ** we are unable to open the journal file. 
       **
       ** The journal file does not need to be locked itself.  The
       ** journal file is never open unless the main database file holds
       ** a write lock, so there is never any chance of two or more
       ** processes opening the journal at the same time.
       */
       rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd);
       if( rc!=SQLITE_OK ){
         sqlite3OsUnlock(pPager->fd, NO_LOCK);
         pPager->state = PAGER_UNLOCK;
         return SQLITE_BUSY;
       }
       pPager->journalOpen = 1;
       pPager->journalStarted = 0;
       pPager->journalOff = 0;
       pPager->setMaster = 0;
       pPager->journalHdr = 0;







|
<














|
<







2688
2689
2690
2691
2692
2693
2694
2695

2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710

2711
2712
2713
2714
2715
2716
2717
       ** 
       ** Because the intermediate RESERVED lock is not requested, the
       ** second process will get to this point in the code and fail to
       ** obtain it's own EXCLUSIVE lock on the database file.
       */
       rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK);
       if( rc!=SQLITE_OK ){
         pager_unlock(pPager);

         return pager_error(pPager, rc);
       }
       pPager->state = PAGER_EXCLUSIVE;

       /* Open the journal for reading only.  Return SQLITE_BUSY if
       ** we are unable to open the journal file. 
       **
       ** The journal file does not need to be locked itself.  The
       ** journal file is never open unless the main database file holds
       ** a write lock, so there is never any chance of two or more
       ** processes opening the journal at the same time.
       */
       rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd);
       if( rc!=SQLITE_OK ){
         pager_unlock(pPager);

         return SQLITE_BUSY;
       }
       pPager->journalOpen = 1;
       pPager->journalStarted = 0;
       pPager->journalOff = 0;
       pPager->setMaster = 0;
       pPager->journalHdr = 0;
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
    /* If this was a malloc() failure, then we will not be closing the pager
    ** file. So delete any journal file we may have just created. Otherwise,
    ** the system will get confused, we have a read-lock on the file and a
    ** mysterious journal has appeared in the filesystem.
    */
    sqlite3OsDelete(pPager->zJournal);
  }else{
    sqlite3OsUnlock(pPager->fd, NO_LOCK);
    pPager->state = PAGER_UNLOCK;
  }
  return rc;
}

/*
** Acquire a write-lock on the database.  The lock is removed when
** the any of the following happen:







|
<







2985
2986
2987
2988
2989
2990
2991
2992

2993
2994
2995
2996
2997
2998
2999
    /* If this was a malloc() failure, then we will not be closing the pager
    ** file. So delete any journal file we may have just created. Otherwise,
    ** the system will get confused, we have a read-lock on the file and a
    ** mysterious journal has appeared in the filesystem.
    */
    sqlite3OsDelete(pPager->zJournal);
  }else{
    pager_unlock(pPager);

  }
  return rc;
}

/*
** Acquire a write-lock on the database.  The lock is removed when
** the any of the following happen:
3229
3230
3231
3232
3233
3234
3235

3236
3237
3238
3239
3240
3241
3242
      }
      page_add_to_stmt_list(pPg);
    }
  }

  /* Update the database size and return.
  */

  if( pPager->dbSize<(int)pPg->pgno ){
    pPager->dbSize = pPg->pgno;
    if( !MEMDB && pPager->dbSize==PENDING_BYTE/pPager->pageSize ){
      pPager->dbSize++;
    }
  }
  return rc;







>







3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
      }
      page_add_to_stmt_list(pPg);
    }
  }

  /* Update the database size and return.
  */
  assert( pPager->state>=PAGER_SHARED );
  if( pPager->dbSize<(int)pPg->pgno ){
    pPager->dbSize = pPg->pgno;
    if( !MEMDB && pPager->dbSize==PENDING_BYTE/pPager->pageSize ){
      pPager->dbSize++;
    }
  }
  return rc;
3304
3305
3306
3307
3308
3309
3310

3311
3312
3313
3314
3315
3316
3317

  if( MEMDB ) return;

  pPg = pager_lookup(pPager, pgno);
  assert( pPg!=0 );  /* We never call _dont_write unless the page is in mem */
  pPg->alwaysRollback = 1;
  if( pPg->dirty && !pPager->stmtInUse ){

    if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSize<pPager->dbSize ){
      /* If this pages is the last page in the file and the file has grown
      ** during the current transaction, then do NOT mark the page as clean.
      ** When the database file grows, we must make sure that the last page
      ** gets written at least once so that the disk file will be the correct
      ** size. If you do not write this page and the size of the file
      ** on the disk ends up being too small, that can lead to database







>







3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334

  if( MEMDB ) return;

  pPg = pager_lookup(pPager, pgno);
  assert( pPg!=0 );  /* We never call _dont_write unless the page is in mem */
  pPg->alwaysRollback = 1;
  if( pPg->dirty && !pPager->stmtInUse ){
    assert( pPager->state>=PAGER_SHARED );
    if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSize<pPager->dbSize ){
      /* If this pages is the last page in the file and the file has grown
      ** during the current transaction, then do NOT mark the page as clean.
      ** When the database file grows, we must make sure that the last page
      ** gets written at least once so that the disk file will be the correct
      ** size. If you do not write this page and the size of the file
      ** on the disk ends up being too small, that can lead to database
3540
3541
3542
3543
3544
3545
3546

3547
3548
3549
3550
3551
3552
3553
** open.  A new statement journal is created that can be used to rollback
** changes of a single SQL command within a larger transaction.
*/
int sqlite3pager_stmt_begin(Pager *pPager){
  int rc;
  char zTemp[SQLITE_TEMPNAME_SIZE];
  assert( !pPager->stmtInUse );

  assert( pPager->dbSize>=0 );
  TRACE2("STMT-BEGIN %d\n", PAGERID(pPager));
  if( MEMDB ){
    pPager->stmtInUse = 1;
    pPager->stmtSize = pPager->dbSize;
    return SQLITE_OK;
  }







>







3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
** open.  A new statement journal is created that can be used to rollback
** changes of a single SQL command within a larger transaction.
*/
int sqlite3pager_stmt_begin(Pager *pPager){
  int rc;
  char zTemp[SQLITE_TEMPNAME_SIZE];
  assert( !pPager->stmtInUse );
  assert( pPager->state>=PAGER_SHARED );
  assert( pPager->dbSize>=0 );
  TRACE2("STMT-BEGIN %d\n", PAGERID(pPager));
  if( MEMDB ){
    pPager->stmtInUse = 1;
    pPager->stmtSize = pPager->dbSize;
    return SQLITE_OK;
  }