Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Make sure that the database size cache in the pager is invalidated whenever the database is unlocked. A stale value in the database size cache can result in database corruption on a heavily loaded system running autovacuum. (CVS 3548) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
6806b9ecb5e3b90e793c5862404e7648 |
User & Date: | drh 2007-01-03 15:34:30.000 |
Context
2007-01-03
| ||
23:36 | Additional changes in the pager and os interface layers to fix problems that might be contributing to recently observed database corruption. (CVS 3549) (check-in: a593d5743e user: drh tags: trunk) | |
15:34 | Make sure that the database size cache in the pager is invalidated whenever the database is unlocked. A stale value in the database size cache can result in database corruption on a heavily loaded system running autovacuum. (CVS 3548) (check-in: 6806b9ecb5 user: drh tags: trunk) | |
2007-01-02
| ||
18:41 | Update the vtab_err test to check for -DSQLITE_MEMDEBUG=1 and skip the tests if missing. Pager is more careful to clear its file size cache. Remove an assert() in VDBE that might fail on a corrupt database file. (CVS 3547) (check-in: bf1afd016a user: drh tags: trunk) | |
Changes
Changes to src/pager.c.
︙ | ︙ | |||
14 15 16 17 18 19 20 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** | | | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** ** @(#) $Id: pager.c,v 1.279 2007/01/03 15:34:30 drh Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" #include "os.h" #include "pager.h" #include <assert.h> #include <string.h> |
︙ | ︙ | |||
845 846 847 848 849 850 851 852 853 854 855 856 857 858 | if( pPager->aHash==0 ) return 0; p = pPager->aHash[pgno & (pPager->nHash-1)]; while( p && p->pgno!=pgno ){ p = p->pNextHash; } return p; } /* ** Unlock the database and clear the in-memory cache. This routine ** sets the state of the pager back to what it was when it was first ** opened. Any outstanding pages are invalidated and subsequent attempts ** to access those pages will likely result in a coredump. */ | > > > > > > > > > > > > > > > > | 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 | if( pPager->aHash==0 ) return 0; p = pPager->aHash[pgno & (pPager->nHash-1)]; while( p && p->pgno!=pgno ){ p = p->pNextHash; } return p; } /* ** Unlock the database file. ** ** Once all locks have been removed from the database file, other ** processes or threads might change the file. So make sure all of ** our internal cache is invalidated. */ static void pager_unlock(Pager *pPager){ if( !MEMDB ){ sqlite3OsUnlock(pPager->fd, NO_LOCK); pPager->dbSize = -1; } pPager->state = PAGER_UNLOCK; } /* ** Unlock the database and clear the in-memory cache. This routine ** sets the state of the pager back to what it was when it was first ** opened. Any outstanding pages are invalidated and subsequent attempts ** to access those pages will likely result in a coredump. */ |
︙ | ︙ | |||
870 871 872 873 874 875 876 | pPager->nHash = 0; sqliteFree(pPager->aHash); pPager->nPage = 0; pPager->aHash = 0; if( pPager->state>=PAGER_RESERVED ){ sqlite3pager_rollback(pPager); } | | < < | 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 | pPager->nHash = 0; sqliteFree(pPager->aHash); pPager->nPage = 0; pPager->aHash = 0; if( pPager->state>=PAGER_RESERVED ){ sqlite3pager_rollback(pPager); } pager_unlock(pPager); pPager->nRef = 0; assert( pPager->journalOpen==0 ); } /* ** When this routine is called, the pager has the journal file open and ** a RESERVED or EXCLUSIVE lock on the database. This routine releases |
︙ | ︙ | |||
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 | } /* Truncate the database back to its original size. */ if( pPager->state>=PAGER_EXCLUSIVE ){ rc = pager_truncate(pPager, pPager->stmtSize); } pPager->dbSize = pPager->stmtSize; /* Figure out how many records are in the statement journal. */ assert( pPager->stmtInUse && pPager->journalOpen ); sqlite3OsSeek(pPager->stfd, 0); nRec = pPager->stmtNRec; | > | 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 | } /* Truncate the database back to its original size. */ if( pPager->state>=PAGER_EXCLUSIVE ){ rc = pager_truncate(pPager, pPager->stmtSize); } assert( pPager->state>=PAGER_SHARED ); pPager->dbSize = pPager->stmtSize; /* Figure out how many records are in the statement journal. */ assert( pPager->stmtInUse && pPager->journalOpen ); sqlite3OsSeek(pPager->stfd, 0); nRec = pPager->stmtNRec; |
︙ | ︙ | |||
1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 | ** false or until the lock succeeds. ** ** Return SQLITE_OK on success and an error code if we cannot obtain ** the lock. */ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; assert( PAGER_SHARED==SHARED_LOCK ); assert( PAGER_RESERVED==RESERVED_LOCK ); assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); if( pPager->state>=locktype ){ rc = SQLITE_OK; }else{ do { rc = sqlite3OsLock(pPager->fd, locktype); }while( rc==SQLITE_BUSY && sqlite3InvokeBusyHandler(pPager->pBusyHandler) ); if( rc==SQLITE_OK ){ pPager->state = locktype; } } return rc; } /* ** Truncate the file to the number of pages specified. */ int sqlite3pager_truncate(Pager *pPager, Pgno nPage){ int rc; sqlite3pager_pagecount(pPager); if( pPager->errCode ){ rc = pPager->errCode; return rc; } if( nPage>=(unsigned)pPager->dbSize ){ return SQLITE_OK; | > > > > > > > | 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 | ** false or until the lock succeeds. ** ** Return SQLITE_OK on success and an error code if we cannot obtain ** the lock. */ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* The OS lock values must be the same as the Pager lock values */ assert( PAGER_SHARED==SHARED_LOCK ); assert( PAGER_RESERVED==RESERVED_LOCK ); assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); /* If the file is currently unlocked then the size must be unknown */ assert( pPager->state>=PAGER_SHARED || pPager->dbSize<0 || MEMDB ); if( pPager->state>=locktype ){ rc = SQLITE_OK; }else{ do { rc = sqlite3OsLock(pPager->fd, locktype); }while( rc==SQLITE_BUSY && sqlite3InvokeBusyHandler(pPager->pBusyHandler) ); if( rc==SQLITE_OK ){ pPager->state = locktype; } } return rc; } /* ** Truncate the file to the number of pages specified. */ int sqlite3pager_truncate(Pager *pPager, Pgno nPage){ int rc; assert( pPager->state>=PAGER_SHARED || MEMDB ); sqlite3pager_pagecount(pPager); if( pPager->errCode ){ rc = pPager->errCode; return rc; } if( nPage>=(unsigned)pPager->dbSize ){ return SQLITE_OK; |
︙ | ︙ | |||
2060 2061 2062 2063 2064 2065 2066 | /* We ignore any IO errors that occur during the rollback ** operation. So disable IO error simulation so that testing ** works more easily. */ disable_simulated_io_errors(); sqlite3pager_rollback(pPager); enable_simulated_io_errors(); | < | < < | < | 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 | /* We ignore any IO errors that occur during the rollback ** operation. So disable IO error simulation so that testing ** works more easily. */ disable_simulated_io_errors(); sqlite3pager_rollback(pPager); enable_simulated_io_errors(); pager_unlock(pPager); assert( pPager->errCode || pPager->journalOpen==0 ); break; } case PAGER_SHARED: { pager_unlock(pPager); break; } default: { /* Do nothing */ break; } } |
︙ | ︙ | |||
2670 2671 2672 2673 2674 2675 2676 | ** ** Because the intermediate RESERVED lock is not requested, the ** second process will get to this point in the code and fail to ** obtain it's own EXCLUSIVE lock on the database file. */ rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ | | < | < | 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 | ** ** Because the intermediate RESERVED lock is not requested, the ** second process will get to this point in the code and fail to ** obtain it's own EXCLUSIVE lock on the database file. */ rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ pager_unlock(pPager); return pager_error(pPager, rc); } pPager->state = PAGER_EXCLUSIVE; /* Open the journal for reading only. Return SQLITE_BUSY if ** we are unable to open the journal file. ** ** The journal file does not need to be locked itself. The ** journal file is never open unless the main database file holds ** a write lock, so there is never any chance of two or more ** processes opening the journal at the same time. */ rc = sqlite3OsOpenReadOnly(pPager->zJournal, &pPager->jfd); if( rc!=SQLITE_OK ){ pager_unlock(pPager); return SQLITE_BUSY; } pPager->journalOpen = 1; pPager->journalStarted = 0; pPager->journalOff = 0; pPager->setMaster = 0; pPager->journalHdr = 0; |
︙ | ︙ | |||
2969 2970 2971 2972 2973 2974 2975 | /* If this was a malloc() failure, then we will not be closing the pager ** file. So delete any journal file we may have just created. Otherwise, ** the system will get confused, we have a read-lock on the file and a ** mysterious journal has appeared in the filesystem. */ sqlite3OsDelete(pPager->zJournal); }else{ | | < | 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 | /* If this was a malloc() failure, then we will not be closing the pager ** file. So delete any journal file we may have just created. Otherwise, ** the system will get confused, we have a read-lock on the file and a ** mysterious journal has appeared in the filesystem. */ sqlite3OsDelete(pPager->zJournal); }else{ pager_unlock(pPager); } return rc; } /* ** Acquire a write-lock on the database. The lock is removed when ** the any of the following happen: |
︙ | ︙ | |||
3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 | } page_add_to_stmt_list(pPg); } } /* Update the database size and return. */ if( pPager->dbSize<(int)pPg->pgno ){ pPager->dbSize = pPg->pgno; if( !MEMDB && pPager->dbSize==PENDING_BYTE/pPager->pageSize ){ pPager->dbSize++; } } return rc; | > | 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 | } page_add_to_stmt_list(pPg); } } /* Update the database size and return. */ assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSize<(int)pPg->pgno ){ pPager->dbSize = pPg->pgno; if( !MEMDB && pPager->dbSize==PENDING_BYTE/pPager->pageSize ){ pPager->dbSize++; } } return rc; |
︙ | ︙ | |||
3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 | if( MEMDB ) return; pPg = pager_lookup(pPager, pgno); assert( pPg!=0 ); /* We never call _dont_write unless the page is in mem */ pPg->alwaysRollback = 1; if( pPg->dirty && !pPager->stmtInUse ){ if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSize<pPager->dbSize ){ /* If this pages is the last page in the file and the file has grown ** during the current transaction, then do NOT mark the page as clean. ** When the database file grows, we must make sure that the last page ** gets written at least once so that the disk file will be the correct ** size. If you do not write this page and the size of the file ** on the disk ends up being too small, that can lead to database | > | 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 | if( MEMDB ) return; pPg = pager_lookup(pPager, pgno); assert( pPg!=0 ); /* We never call _dont_write unless the page is in mem */ pPg->alwaysRollback = 1; if( pPg->dirty && !pPager->stmtInUse ){ assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSize<pPager->dbSize ){ /* If this pages is the last page in the file and the file has grown ** during the current transaction, then do NOT mark the page as clean. ** When the database file grows, we must make sure that the last page ** gets written at least once so that the disk file will be the correct ** size. If you do not write this page and the size of the file ** on the disk ends up being too small, that can lead to database |
︙ | ︙ | |||
3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 | ** open. A new statement journal is created that can be used to rollback ** changes of a single SQL command within a larger transaction. */ int sqlite3pager_stmt_begin(Pager *pPager){ int rc; char zTemp[SQLITE_TEMPNAME_SIZE]; assert( !pPager->stmtInUse ); assert( pPager->dbSize>=0 ); TRACE2("STMT-BEGIN %d\n", PAGERID(pPager)); if( MEMDB ){ pPager->stmtInUse = 1; pPager->stmtSize = pPager->dbSize; return SQLITE_OK; } | > | 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 | ** open. A new statement journal is created that can be used to rollback ** changes of a single SQL command within a larger transaction. */ int sqlite3pager_stmt_begin(Pager *pPager){ int rc; char zTemp[SQLITE_TEMPNAME_SIZE]; assert( !pPager->stmtInUse ); assert( pPager->state>=PAGER_SHARED ); assert( pPager->dbSize>=0 ); TRACE2("STMT-BEGIN %d\n", PAGERID(pPager)); if( MEMDB ){ pPager->stmtInUse = 1; pPager->stmtSize = pPager->dbSize; return SQLITE_OK; } |
︙ | ︙ |