Index: src/backup.c ================================================================== --- src/backup.c +++ src/backup.c @@ -460,11 +460,10 @@ } }else{ nDestTruncate = nSrcPage * (pgszSrc/pgszDest); } assert( nDestTruncate>0 ); - sqlite3PagerTruncateImage(pDestPager, nDestTruncate); if( pgszSrc= iSize || ( nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest )); - /* This call ensures that all data required to recreate the original + /* This block ensures that all data required to recreate the original ** database has been stored in the journal for pDestPager and the ** journal synced to disk. So at this point we may safely modify ** the database file in any way, knowing that if a power failure ** occurs, the original database will be reconstructed from the ** journal file. */ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } /* Write the extra pages and truncate the database file as required */ iEnd = MIN(PENDING_BYTE + pgszDest, iSize); for( iOff=PENDING_BYTE+pgszSrc; @@ -517,10 +531,11 @@ /* Sync the database file to disk. */ if( rc==SQLITE_OK ){ rc = sqlite3PagerSync(pDestPager); } }else{ + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); } /* Finish committing the transaction to the destination database. */ if( SQLITE_OK==rc Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -2593,10 +2593,11 @@ ** is requested, this is a no-op. */ if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ goto trans_begun; } + assert( pBt->bDoTruncate==0 ); /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ rc = SQLITE_READONLY; goto trans_begun; @@ -2907,30 +2908,32 @@ return rc; } /* Forward declaration required by incrVacuumStep(). */ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ /* -** Perform a single step of an incremental-vacuum. If successful, -** return SQLITE_OK. If there is no work to do (and therefore no -** point in calling this function again), return SQLITE_DONE. -** -** More specificly, this function attempts to re-organize the -** database so that the last page of the file currently in use -** is no longer in use. -** -** If the nFin parameter is non-zero, this function assumes -** that the caller will keep calling incrVacuumStep() until -** it returns SQLITE_DONE or an error, and that nFin is the -** number of pages the database file will contain after this -** process is complete. If nFin is zero, it is assumed that -** incrVacuumStep() will be called a finite amount of times -** which may or may not empty the freelist. A full autovacuum -** has nFin>0. A "PRAGMA incremental_vacuum" has nFin==0. -*/ -static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. +** +** More specificly, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. +** +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** operation, or false for an incremental vacuum. +*/ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ Pgno nFreeList; /* Number of pages still on the free-list */ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); assert( iLastPg>nFin ); @@ -2951,85 +2954,98 @@ if( eType==PTRMAP_ROOTPAGE ){ return SQLITE_CORRUPT_BKPT; } if( eType==PTRMAP_FREEPAGE ){ - if( nFin==0 ){ + if( bCommit==0 ){ /* Remove the page from the files free-list. This is not required - ** if nFin is non-zero. In that case, the free-list will be + ** if bCommit is non-zero. In that case, the free-list will be ** truncated to zero after this function returns, so it doesn't ** matter if it still contains some garbage entries. */ Pgno iFreePg; MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } assert( iFreePg==iLastPg ); releasePage(pFreePg); } } else { Pgno iFreePg; /* Index of free page to move pLastPg to */ MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); if( rc!=SQLITE_OK ){ return rc; } - /* If nFin is zero, this loop runs exactly once and page pLastPg + /* If bCommit is zero, this loop runs exactly once and page pLastPg ** is swapped with the first free page pulled off the free list. ** - ** On the other hand, if nFin is greater than zero, then keep + ** On the other hand, if bCommit is greater than zero, then keep ** looping until a free-page located within the first nFin pages ** of the file is found. */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } do { MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); if( rc!=SQLITE_OK ){ releasePage(pLastPg); return rc; } releasePage(pFreePg); - }while( nFin!=0 && iFreePg>nFin ); + }while( bCommit && iFreePg>nFin ); assert( iFreePgpDbPage); - if( rc==SQLITE_OK ){ - rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); - } + rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); releasePage(pLastPg); if( rc!=SQLITE_OK ){ return rc; } } } - if( nFin==0 ){ - iLastPg--; - while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ - if( PTRMAP_ISPAGE(pBt, iLastPg) ){ - MemPage *pPg; - rc = btreeGetPage(pBt, iLastPg, &pPg, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pPg->pDbPage); - releasePage(pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - } - iLastPg--; - } - sqlite3PagerTruncateImage(pBt->pPager, iLastPg); + if( bCommit==0 ){ + do { + iLastPg--; + }while( iLastPg==PENDING_BYTE_PAGE(pBt) || PTRMAP_ISPAGE(pBt, iLastPg) ); + pBt->bDoTruncate = 1; pBt->nPage = iLastPg; } return SQLITE_OK; } + +/* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFininTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); if( !pBt->autoVacuum ){ rc = SQLITE_DONE; }else{ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[28], pBt->nPage); + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig0 ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; } } sqlite3BtreeLeave(p); return rc; } @@ -3075,13 +3101,11 @@ invalidateAllOverflowCache(pBt); assert(pBt->autoVacuum); if( !pBt->incrVacuum ){ Pgno nFin; /* Number of pages in database after autovacuuming */ Pgno nFree; /* Number of pages on the freelist initially */ - Pgno nPtrmap; /* Number of PtrMap pages to be freed */ Pgno iFree; /* The next page to be freed */ - int nEntry; /* Number of entries on one ptrmap page */ Pgno nOrig; /* Database size before freeing */ nOrig = btreePagecount(pBt); if( PTRMAP_ISPAGE(pBt, nOrig) || nOrig==PENDING_BYTE_PAGE(pBt) ){ /* It is not possible to create a database for which the final page @@ -3090,30 +3114,22 @@ */ return SQLITE_CORRUPT_BKPT; } nFree = get4byte(&pBt->pPage1->aData[36]); - nEntry = pBt->usableSize/5; - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinnOrig ) return SQLITE_CORRUPT_BKPT; for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ - rc = incrVacuumStep(pBt, nFin, iFree); + rc = incrVacuumStep(pBt, nFin, iFree, 1); } if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[32], 0); put4byte(&pBt->pPage1->aData[36], 0); put4byte(&pBt->pPage1->aData[28], nFin); - sqlite3PagerTruncateImage(pBt->pPager, nFin); + pBt->bDoTruncate = 1; pBt->nPage = nFin; } if( rc!=SQLITE_OK ){ sqlite3PagerRollback(pPager); } @@ -3164,10 +3180,13 @@ if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); return rc; } } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } #endif rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); sqlite3BtreeLeave(p); } return rc; @@ -3179,10 +3198,13 @@ */ static void btreeEndTransaction(Btree *p){ BtShared *pBt = p->pBt; assert( sqlite3BtreeHoldsMutex(p) ); +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif btreeClearHasContent(pBt); if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ /* If there are other active statements that belong to this database ** handle, downgrade to a read-only transaction. The other statements ** may still be reading from the database. */ @@ -4865,11 +4887,11 @@ static int allocateBtreePage( BtShared *pBt, MemPage **ppPage, Pgno *pPgno, Pgno nearby, - u8 exact + u8 eMode ){ MemPage *pPage1; int rc; u32 n; /* Number of pages on the freelist */ u32 k; /* Number of leaves on the trunk of the freelist */ @@ -4893,20 +4915,23 @@ /* If the 'exact' parameter was true and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( exact && nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; - } - *pPgno = nearby; + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } + } + }else if( eMode==BTALLOC_LE ){ + searchList = 1; } #endif /* Decrement the free-list count by 1. Set iTrunk to the index of the ** first free-list trunk page. iPrevTrunk is initially 1. @@ -4957,15 +4982,17 @@ }else if( k>(u32)(pBt->usableSize/4 - 2) ){ /* Value of k is out of range. Database corruption */ rc = SQLITE_CORRUPT_BKPT; goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList && nearby==iTrunk ){ + }else if( searchList + && (nearby==iTrunk || (iTrunkpDbPage); if( rc ){ goto end_allocate_page; @@ -5024,18 +5051,28 @@ u32 closest; Pgno iPage; unsigned char *aData = pTrunk->aData; if( nearby>0 ){ u32 i; - int dist; closest = 0; - dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); - for(i=1; imxPage ){ rc = SQLITE_CORRUPT_BKPT; goto end_allocate_page; } testcase( iPage==mxPage ); - if( !searchList || iPage==nearby ){ + if( !searchList + || (iPage==nearby || (iPagepgno, n-1)); @@ -5072,12 +5111,30 @@ } releasePage(pPrevTrunk); pPrevTrunk = 0; }while( searchList ); }else{ - /* There are no pages on the freelist, so create a new page at the - ** end of the file */ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==pBt->bDoTruncate); + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; pBt->nPage++; if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; @@ -5088,11 +5145,11 @@ ** becomes a new pointer-map page, the second is used by the caller. */ MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); } if( rc ) return rc; @@ -5102,11 +5159,11 @@ #endif put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage); *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, 1); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); } @@ -7117,11 +7174,11 @@ /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens ** to reside at pgnoRoot). */ - rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); if( rc!=SQLITE_OK ){ return rc; } if( pgnoMove!=pgnoRoot ){ Index: src/btreeInt.h ================================================================== --- src/btreeInt.h +++ src/btreeInt.h @@ -409,10 +409,11 @@ MemPage *pPage1; /* First page of the database */ u8 openFlags; /* Flags to sqlite3BtreeOpen() */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -1836,10 +1836,12 @@ pPager->eState = PAGER_ERROR; } return rc; } +static int pager_truncate(Pager *pPager, Pgno nPage); + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called ** after rollback of a hot-journal, or if an error occurs while opening ** the journal file or writing the very first journal-header of a @@ -1889,11 +1891,11 @@ ** tries to unlock the database file if not in exclusive mode. If the ** unlock operation fails as well, then the first error code related ** to the first error encountered (the journal finalization one) is ** returned. */ -static int pager_end_transaction(Pager *pPager, int hasMaster){ +static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ /* Do nothing if the pager does not have an open write transaction ** or at least a RESERVED lock. This function may be called when there @@ -1975,11 +1977,21 @@ ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE ** lock held on the database file. */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); } + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; @@ -2014,11 +2026,11 @@ sqlite3BeginBenignMalloc(); sqlite3PagerRollback(pPager); sqlite3EndBenignMalloc(); }else if( !pPager->exclusiveMode ){ assert( pPager->eState==PAGER_READER ); - pager_end_transaction(pPager, 0); + pager_end_transaction(pPager, 0, 0); } } pager_unlock(pPager); } @@ -2789,11 +2801,11 @@ && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ rc = sqlite3PagerSync(pPager); } if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager, zMaster[0]!='\0'); + rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && zMaster[0] && res ){ /* If there was a master journal and this routine will return success, ** see if it is possible to delete the master journal. @@ -5883,40 +5895,10 @@ #else rc = pager_incr_changecounter(pPager, 0); #endif if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. - ** - ** Before reading the pages with page numbers larger than the - ** current value of Pager.dbSize, set dbSize back to the value - ** that it took at the start of the transaction. Otherwise, the - ** calls to sqlite3PagerGet() return zeroed pages instead of - ** reading data from the database file. - */ - if( pPager->dbSizedbOrigSize - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - ){ - Pgno i; /* Iterator variable */ - const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ - const Pgno dbSize = pPager->dbSize; /* Database image size */ - pPager->dbSize = pPager->dbOrigSize; - for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ - if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ - PgHdr *pPage; /* Page to journal */ - rc = sqlite3PagerGet(pPager, i, &pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - rc = sqlite3PagerWrite(pPage); - sqlite3PagerUnref(pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - } - pPager->dbSize = dbSize; - } - /* Write the master journal name into the journal file. If a master ** journal file name has already been written to the journal file, ** or if zMaster is NULL (no master journal), then this call is a no-op. */ rc = writeMasterJournal(pPager, zMaster); @@ -5940,15 +5922,18 @@ if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); goto commit_phase_one_exit; } sqlite3PcacheCleanAll(pPager->pPCache); - - /* If the file on disk is not the same size as the database image, - ** then use pager_truncate to grow or shrink the file here. - */ - if( pPager->dbSize!=pPager->dbFileSize ){ + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } @@ -6017,11 +6002,11 @@ pPager->eState = PAGER_READER; return SQLITE_OK; } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } /* ** If a write transaction is open, then all changes made within the @@ -6062,15 +6047,15 @@ if( pPager->eState<=PAGER_READER ) return SQLITE_OK; if( pagerUseWal(pPager) ){ int rc2; rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); - rc2 = pager_end_transaction(pPager, pPager->setMaster); + rc2 = pager_end_transaction(pPager, pPager->setMaster, 0); if( rc==SQLITE_OK ) rc = rc2; }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ int eState = pPager->eState; - rc = pager_end_transaction(pPager, 0); + rc = pager_end_transaction(pPager, 0, 0); if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ /* This can happen using journal_mode=off. Move the pager to the error ** state to indicate that the contents of the cache may not be trusted. ** Any active readers will get SQLITE_ABORT. */ Index: src/test_vfs.c ================================================================== --- src/test_vfs.c +++ src/test_vfs.c @@ -263,11 +263,12 @@ static void tvfsExecTcl( Testvfs *p, const char *zMethod, Tcl_Obj *arg1, Tcl_Obj *arg2, - Tcl_Obj *arg3 + Tcl_Obj *arg3, + Tcl_Obj *arg4 ){ int rc; /* Return code from Tcl_EvalObj() */ Tcl_Obj *pEval; assert( p->pScript ); @@ -280,10 +281,11 @@ Tcl_IncrRefCount(p->pScript); Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zMethod, -1)); if( arg1 ) Tcl_ListObjAppendElement(p->interp, pEval, arg1); if( arg2 ) Tcl_ListObjAppendElement(p->interp, pEval, arg2); if( arg3 ) Tcl_ListObjAppendElement(p->interp, pEval, arg3); + if( arg4 ) Tcl_ListObjAppendElement(p->interp, pEval, arg4); rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); if( rc!=TCL_OK ){ Tcl_BackgroundError(p->interp); Tcl_ResetResult(p->interp); @@ -300,11 +302,11 @@ TestvfsFd *pFd = pTestfile->pFd; Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_CLOSE_MASK ){ tvfsExecTcl(p, "xClose", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); } if( pFd->pShmId ){ Tcl_DecrRefCount(pFd->pShmId); @@ -331,11 +333,11 @@ int rc = SQLITE_OK; TestvfsFd *pFd = tvfsGetFd(pFile); Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_READ_MASK ){ tvfsExecTcl(p, "xRead", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } if( rc==SQLITE_OK && p->mask&TESTVFS_READ_MASK && tvfsInjectIoerr(p) ){ rc = SQLITE_IOERR; @@ -360,11 +362,11 @@ Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_WRITE_MASK ){ tvfsExecTcl(p, "xWrite", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, - Tcl_NewWideIntObj(iOfst) + Tcl_NewWideIntObj(iOfst), Tcl_NewIntObj(iAmt) ); tvfsResultCode(p, &rc); } if( rc==SQLITE_OK && tvfsInjectFullerr(p) ){ @@ -388,11 +390,11 @@ TestvfsFd *pFd = tvfsGetFd(pFile); Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_TRUNCATE_MASK ){ tvfsExecTcl(p, "xTruncate", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } if( rc==SQLITE_OK ){ @@ -429,11 +431,11 @@ assert(0); } tvfsExecTcl(p, "xSync", Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, - Tcl_NewStringObj(zFlags, -1) + Tcl_NewStringObj(zFlags, -1), 0 ); tvfsResultCode(p, &rc); } if( rc==SQLITE_OK && tvfsInjectFullerr(p) ) rc = SQLITE_FULL; @@ -576,11 +578,11 @@ z += strlen(z) + 1; Tcl_ListObjAppendElement(0, pArg, Tcl_NewStringObj(z, -1)); z += strlen(z) + 1; } } - tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0); + tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0, 0); Tcl_DecrRefCount(pArg); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; }else{ pId = Tcl_GetObjResult(p->interp); @@ -633,11 +635,11 @@ int rc = SQLITE_OK; Testvfs *p = (Testvfs *)pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_DELETE_MASK ){ tvfsExecTcl(p, "xDelete", - Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0 + Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0, 0 ); tvfsResultCode(p, &rc); } if( rc==SQLITE_OK ){ rc = sqlite3OsDelete(PARENTVFS(pVfs), zPath, dirSync); @@ -661,11 +663,11 @@ char *zArg = 0; if( flags==SQLITE_ACCESS_EXISTS ) zArg = "SQLITE_ACCESS_EXISTS"; if( flags==SQLITE_ACCESS_READWRITE ) zArg = "SQLITE_ACCESS_READWRITE"; if( flags==SQLITE_ACCESS_READ ) zArg = "SQLITE_ACCESS_READ"; tvfsExecTcl(p, "xAccess", - Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0 + Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0, 0 ); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; }else{ Tcl_Interp *interp = p->interp; @@ -689,11 +691,11 @@ char *zOut ){ Testvfs *p = (Testvfs *)pVfs->pAppData; if( p->pScript && p->mask&TESTVFS_FULLPATHNAME_MASK ){ int rc; - tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0); + tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0, 0); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; } } return sqlite3OsFullPathname(PARENTVFS(pVfs), zPath, nOut, zOut); @@ -769,11 +771,11 @@ ** ** SCRIPT xShmOpen FILENAME */ Tcl_ResetResult(p->interp); if( p->pScript && p->mask&TESTVFS_SHMOPEN_MASK ){ - tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0); + tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0, 0); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; } } @@ -839,11 +841,11 @@ Tcl_IncrRefCount(pArg); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(iPage)); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz)); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite)); tvfsExecTcl(p, "xShmMap", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg, 0 ); tvfsResultCode(p, &rc); Tcl_DecrRefCount(pArg); } if( rc==SQLITE_OK && p->mask&TESTVFS_SHMMAP_MASK && tvfsInjectIoerr(p) ){ @@ -889,11 +891,11 @@ }else{ strcpy(&zLock[nLock], " exclusive"); } tvfsExecTcl(p, "xShmLock", Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, - Tcl_NewStringObj(zLock, -1) + Tcl_NewStringObj(zLock, -1), 0 ); tvfsResultCode(p, &rc); } if( rc==SQLITE_OK && p->mask&TESTVFS_SHMLOCK_MASK && tvfsInjectIoerr(p) ){ @@ -935,11 +937,11 @@ return; } if( p->pScript && p->mask&TESTVFS_SHMBARRIER_MASK ){ tvfsExecTcl(p, "xShmBarrier", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0 ); } } static int tvfsShmUnmap( @@ -959,11 +961,11 @@ if( !pBuffer ) return SQLITE_OK; assert( pFd->pShmId && pFd->pShm ); if( p->pScript && p->mask&TESTVFS_SHMCLOSE_MASK ){ tvfsExecTcl(p, "xShmUnmap", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0 ); tvfsResultCode(p, &rc); } for(ppFd=&pBuffer->pFile; *ppFd!=pFd; ppFd=&((*ppFd)->pNext)); ADDED test/incrvacuum3.test Index: test/incrvacuum3.test ================================================================== --- /dev/null +++ test/incrvacuum3.test @@ -0,0 +1,137 @@ +# 2013 Feb 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the SQLite library, focusing +# on the incremental vacuum feature. +# +# The tests in this file were added at the same time as optimizations +# were made to: +# +# * Truncate the database after a rollback mode commit, and +# +# * Avoid moving pages to locations from which they may need to be moved +# a second time if an incremental-vacuum proccess is allowed to vacuum +# the entire database. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix incrvacuum3 + +# If this build of the library does not support auto-vacuum, omit this +# whole file. +ifcapable {!autovacuum || !pragma} { + finish_test + return +} + +proc check_on_disk {} { + + # Copy the files for database "test.db" to "test2.db". + forcedelete test2.db test2.db-journal test2.db-wal + forcecopy test.db test2.db + if {[file exists test.db-journal]} { + forcecopy test.db-journal test2.db-journal + } + if {[file exists test.db-wal]} { + forcecopy test.db-wal test2.db-wal + } + + # Open "test2.db" and check it is Ok. + sqlite3 dbcheck test2.db + set ret [dbcheck eval { PRAGMA integrity_check }] + dbcheck close + set ret +} + +# Run these tests once in rollback journal mode, and once in wal mode. +# +foreach {T jrnl_mode} { + 1 delete + 2 wal +} { + catch { db close } + forcedelete test.db test.db-journal test.db-wal + sqlite3 db test.db + db eval { + PRAGMA cache_size = 5; + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 2; + } + db eval "PRAGMA journal_mode = $jrnl_mode" + + foreach {tn sql} { + 1 { + CREATE TABLE t1(x UNIQUE); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 4 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 8 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 16 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 32 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 64 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 128 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 256 + } + + 2 { + DELETE FROM t1 WHERE rowid%8; + } + + 3 { + BEGIN; + PRAGMA incremental_vacuum = 100; + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 64 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 128 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 256 + ROLLBACK; + } + + 4 { + BEGIN; + SAVEPOINT one; + PRAGMA incremental_vacuum = 100; + SAVEPOINT two; + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 64 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 128 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 256 + } + + 5 { ROLLBACK to two } + + 6 { ROLLBACK to one } + + 7 { + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 64 + PRAGMA incremental_vacuum = 1000; + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 128 + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 256 + ROLLBACK; + } + + 8 { + BEGIN; + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 64 + PRAGMA incremental_vacuum = 1000; + INSERT INTO t1 SELECT randomblob(400) FROM t1; -- 128 + COMMIT; + } + } { + do_execsql_test $T.1.$tn.1 $sql + do_execsql_test $T.1.$tn.2 {PRAGMA integrity_check} ok + do_test $T.1.$tn.3 { check_on_disk } ok + } + + do_execsql_test $T.1.x.1 { PRAGMA freelist_count } 0 + do_execsql_test $T.1.x.2 { SELECT count(*) FROM t1 } 128 +} + +finish_test + Index: test/incrvacuum_ioerr.test ================================================================== --- test/incrvacuum_ioerr.test +++ test/incrvacuum_ioerr.test @@ -137,12 +137,12 @@ for {set iTest 1} {$::rc && $iTest<2000} {incr iTest} { # Figure out how big the database is and how many free pages it # has before running incremental-vacuum. # - set nPage [expr {[file size test.db]/1024}] set nFree [execsql {pragma freelist_count} db1] + set nPage [execsql {pragma page_count} db1] # Now run incremental-vacuum to vacuum 5 pages from the db file. # The iTest'th I/O call is set to fail. # set ::sqlite_io_error_pending $iTest @@ -156,11 +156,11 @@ set ::sqlite_io_error_persist 0 set ::sqlite_io_error_hit 0 set ::sqlite_io_error_hardhit 0 set nFree2 [execsql {pragma freelist_count} db1] - set nPage2 [expr {[file size test.db]/1024}] + set nPage2 [execsql {pragma page_count} db1] do_test incrvacuum-ioerr-4.$iTest.2 { set shrink [expr {$nPage-$nPage2}] expr {$shrink==0 || $shrink==5} } {1} Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -1115,10 +1115,29 @@ } } lappend r $msg } + +proc run_ioerr_prep {} { + set ::sqlite_io_error_pending 0 + catch {db close} + catch {db2 close} + catch {forcedelete test.db} + catch {forcedelete test.db-journal} + catch {forcedelete test2.db} + catch {forcedelete test2.db-journal} + set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db] + sqlite3_extended_result_codes $::DB $::ioerropts(-erc) + if {[info exists ::ioerropts(-tclprep)]} { + eval $::ioerropts(-tclprep) + } + if {[info exists ::ioerropts(-sqlprep)]} { + execsql $::ioerropts(-sqlprep) + } + expr 0 +} # Usage: do_ioerr_test # # This proc is used to implement test cases that check that IO errors # are correctly handled. The first argument, , is an integer @@ -1148,14 +1167,30 @@ array set ::ioerropts $args # TEMPORARY: For 3.5.9, disable testing of extended result codes. There are # a couple of obscure IO errors that do not return them. set ::ioerropts(-erc) 0 + + # Create a single TCL script from the TCL and SQL specified + # as the body of the test. + set ::ioerrorbody {} + if {[info exists ::ioerropts(-tclbody)]} { + append ::ioerrorbody "$::ioerropts(-tclbody)\n" + } + if {[info exists ::ioerropts(-sqlbody)]} { + append ::ioerrorbody "db eval {$::ioerropts(-sqlbody)}" + } + + save_prng_state + if {$::ioerropts(-cksum)} { + run_ioerr_prep + eval $::ioerrorbody + set ::goodcksum [cksum] + } set ::go 1 #reset_prng_state - save_prng_state for {set n $::ioerropts(-start)} {$::go} {incr n} { set ::TN $n incr ::ioerropts(-count) -1 if {$::ioerropts(-count)<0} break @@ -1168,52 +1203,27 @@ } # Delete the files test.db and test2.db, then execute the TCL and # SQL (in that order) to prepare for the test case. do_test $testname.$n.1 { - set ::sqlite_io_error_pending 0 - catch {db close} - catch {db2 close} - catch {forcedelete test.db} - catch {forcedelete test.db-journal} - catch {forcedelete test2.db} - catch {forcedelete test2.db-journal} - set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db] - sqlite3_extended_result_codes $::DB $::ioerropts(-erc) - if {[info exists ::ioerropts(-tclprep)]} { - eval $::ioerropts(-tclprep) - } - if {[info exists ::ioerropts(-sqlprep)]} { - execsql $::ioerropts(-sqlprep) - } - expr 0 + run_ioerr_prep } {0} # Read the 'checksum' of the database. if {$::ioerropts(-cksum)} { - set checksum [cksum] + set ::checksum [cksum] } # Set the Nth IO error to fail. do_test $testname.$n.2 [subst { set ::sqlite_io_error_persist $::ioerropts(-persist) set ::sqlite_io_error_pending $n }] $n - - # Create a single TCL script from the TCL and SQL specified - # as the body of the test. - set ::ioerrorbody {} - if {[info exists ::ioerropts(-tclbody)]} { - append ::ioerrorbody "$::ioerropts(-tclbody)\n" - } - if {[info exists ::ioerropts(-sqlbody)]} { - append ::ioerrorbody "db eval {$::ioerropts(-sqlbody)}" - } - - # Execute the TCL Script created in the above block. If - # there are at least N IO operations performed by SQLite as - # a result of the script, the Nth will fail. + + # Execute the TCL script created for the body of this test. If + # at least N IO operations performed by SQLite as a result of + # the script, the Nth will fail. do_test $testname.$n.3 { set ::sqlite_io_error_hit 0 set ::sqlite_io_error_hardhit 0 set r [catch $::ioerrorbody msg] set ::errseen $r @@ -1313,12 +1323,19 @@ if {$::go && $::sqlite_io_error_hardhit && $::ioerropts(-cksum)} { do_test $testname.$n.6 { catch {db close} catch {db2 close} set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db] - cksum - } $checksum + set nowcksum [cksum] + set res [expr {$nowcksum==$::checksum || $nowcksum==$::goodcksum}] + if {$res==0} { + puts "now=$nowcksum" + puts "the=$::checksum" + puts "fwd=$::goodcksum" + } + set res + } 1 } set ::sqlite_io_error_hardhit 0 set ::sqlite_io_error_pending 0 if {[info exists ::ioerropts(-cleanup)]} { Index: test/tkt3762.test ================================================================== --- test/tkt3762.test +++ test/tkt3762.test @@ -8,12 +8,12 @@ # May you share freely, never taking more than you give. # #*********************************************************************** # # Ticket #3762: Make sure that an incremental vacuum that reduces the -# size of the database file such that a pointer-map page is elemented -# can be correctly rolled back. +# size of the database file such that if a pointer-map page is eliminated +# it can be correctly rolled back. # # That ticket #3762 has been fixed has already been verified by the # savepoint6.test test script. But this script is simplier and a # redundant test never hurts. #