Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Auto-vacuum: Account for the page reserved for windows locking (PENDING_BYTE). (CVS 2076) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
d6335698696c7b651bbc436c5177d87e |
User & Date: | danielk1977 2004-11-08 07:13:14.000 |
Context
2004-11-08
| ||
09:26 | Test auto-vacuum mode for crash-proofness. Also fix a bug related to the same. (CVS 2077) (check-in: 839ad771a6 user: danielk1977 tags: trunk) | |
07:13 | Auto-vacuum: Account for the page reserved for windows locking (PENDING_BYTE). (CVS 2076) (check-in: d633569869 user: danielk1977 tags: trunk) | |
2004-11-07
| ||
13:01 | Reindex tests added and bugs fixed. (CVS 2075) (check-in: ad433ec2b6 user: drh tags: trunk) | |
Changes
Changes to src/btree.c.
1 2 3 4 5 6 7 8 9 10 11 | /* ** 2004 April 6 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | /* ** 2004 April 6 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** $Id: btree.c,v 1.214 2004/11/08 07:13:14 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to ** ** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: ** "Sorting And Searching", pages 473-480. Addison-Wesley ** Publishing Company, Reading, Massachusetts. |
︙ | ︙ | |||
390 391 392 393 394 395 396 | ** be defined locally, but now we use the varint routines in the util.c ** file. */ #define getVarint sqlite3GetVarint #define getVarint32 sqlite3GetVarint32 #define putVarint sqlite3PutVarint | | > > > > > | > < | 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 | ** be defined locally, but now we use the varint routines in the util.c ** file. */ #define getVarint sqlite3GetVarint #define getVarint32 sqlite3GetVarint32 #define putVarint sqlite3PutVarint /* The database page the PENDING_BYTE occupies. This page is never used. ** TODO: This macro is very similary to PAGER_MJ_PGNO() in pager.c. They ** should possibly be consolidated (presumably in pager.h). */ #define PENDING_BYTE_PAGE(pBt) ((PENDING_BYTE/(pBt)->pageSize)+1) #ifndef SQLITE_OMIT_AUTOVACUUM /* ** These two macros define the location of the pointer-map entry for a ** database page. The first argument to each is the page size used ** by the database (often 1024). The second is the page number to look ** up in the pointer map. ** ** PTRMAP_PAGENO returns the database page number of the pointer-map ** page that stores the required pointer. PTRMAP_PTROFFSET returns ** the offset of the requested map entry. ** ** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page, ** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be ** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements ** this test. */ #define PTRMAP_PAGENO(pgsz, pgno) (((pgno-2)/(pgsz/5+1))*(pgsz/5+1)+2) #define PTRMAP_PTROFFSET(pgsz, pgno) (((pgno-2)%(pgsz/5+1)-1)*5) #define PTRMAP_ISPAGE(pgsz, pgno) (PTRMAP_PAGENO(pgsz,pgno)==pgno) /* ** The pointer map is a lookup table that contains an entry for each database ** page in the file except for page 1. In this context 'database page' refers ** to any page that is not part of the pointer map itself. Each pointer map ** entry consists of a single byte 'type' and a 4 byte page number. The |
︙ | ︙ | |||
455 456 457 458 459 460 461 462 463 464 465 466 467 468 | */ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){ u8 *pPtrmap; /* The pointer map page */ Pgno iPtrmap; /* The pointer map page number */ int offset; /* Offset in pointer map page */ int rc; iPtrmap = PTRMAP_PAGENO(pBt->pageSize, key); rc = sqlite3pager_get(pBt->pPager, iPtrmap, (void **)&pPtrmap); if( rc!=SQLITE_OK ){ return rc; } offset = PTRMAP_PTROFFSET(pBt->pageSize, key); | > | 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 | */ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){ u8 *pPtrmap; /* The pointer map page */ Pgno iPtrmap; /* The pointer map page number */ int offset; /* Offset in pointer map page */ int rc; assert( key!=0 ); iPtrmap = PTRMAP_PAGENO(pBt->pageSize, key); rc = sqlite3pager_get(pBt->pPager, iPtrmap, (void **)&pPtrmap); if( rc!=SQLITE_OK ){ return rc; } offset = PTRMAP_PTROFFSET(pBt->pageSize, key); |
︙ | ︙ | |||
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 | *nTrunc = 0; return SQLITE_OK; } origSize = sqlite3pager_pagecount(pPager); nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pgsz, origSize)+pgsz/5)/(pgsz/5); finSize = origSize - nFreeList - nPtrMap; TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize)); /* Variable 'finSize' will be the size of the file in pages after ** the auto-vacuum has completed (the current file size minus the number ** of pages on the free list). Loop through the pages that lie beyond ** this mark, and if they are not already on the free list, move them ** to a free page earlier in the file (somewhere before finSize). */ for( iDbPage=finSize+1; iDbPage<=origSize; iDbPage++ ){ rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage); if( rc!=SQLITE_OK ) goto autovacuum_out; assert( eType!=PTRMAP_ROOTPAGE ); | > > > > > > > > > > > | < < | | 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 | *nTrunc = 0; return SQLITE_OK; } origSize = sqlite3pager_pagecount(pPager); nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pgsz, origSize)+pgsz/5)/(pgsz/5); finSize = origSize - nFreeList - nPtrMap; if( origSize>PENDING_BYTE_PAGE(pBt) && finSize<=PENDING_BYTE_PAGE(pBt) ){ finSize--; if( PTRMAP_ISPAGE(pBt->pageSize, finSize) ){ finSize--; } } TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize)); /* Variable 'finSize' will be the size of the file in pages after ** the auto-vacuum has completed (the current file size minus the number ** of pages on the free list). Loop through the pages that lie beyond ** this mark, and if they are not already on the free list, move them ** to a free page earlier in the file (somewhere before finSize). */ for( iDbPage=finSize+1; iDbPage<=origSize; iDbPage++ ){ /* If iDbPage is a pointer map page, or the pending-byte page, skip it. */ if( PTRMAP_ISPAGE(pgsz, iDbPage) || iDbPage==PENDING_BYTE_PAGE(pBt) ){ continue; } rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage); if( rc!=SQLITE_OK ) goto autovacuum_out; assert( eType!=PTRMAP_ROOTPAGE ); /* If iDbPage is free, do not swap it. */ if( eType==PTRMAP_FREEPAGE ){ continue; } rc = getPage(pBt, iDbPage, &pDbMemPage); if( rc!=SQLITE_OK ) goto autovacuum_out; /* Find the next page in the free-list that is not already at the end ** of the file. A page can be pulled off the free list using the |
︙ | ︙ | |||
3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 | #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt->pageSize, *pPgno) ){ /* If *pPgno refers to a pointer-map page, allocate two new pages ** at the end of the file instead of one. The first allocated page ** becomes a new pointer-map page, the second is used by the caller. */ TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno)); (*pPgno)++; } #endif rc = getPage(pBt, *pPgno, ppPage); if( rc ) return rc; rc = sqlite3pager_write((*ppPage)->aData); TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } return rc; } /* ** Add a page of the database file to the freelist. ** ** sqlite3pager_unref() is NOT called for pPage. | > > > > | 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 | #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt->pageSize, *pPgno) ){ /* If *pPgno refers to a pointer-map page, allocate two new pages ** at the end of the file instead of one. The first allocated page ** becomes a new pointer-map page, the second is used by the caller. */ TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno)); assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); (*pPgno)++; } #endif assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); rc = getPage(pBt, *pPgno, ppPage); if( rc ) return rc; rc = sqlite3pager_write((*ppPage)->aData); TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); return rc; } /* ** Add a page of the database file to the freelist. ** ** sqlite3pager_unref() is NOT called for pPage. |
︙ | ︙ | |||
4379 4380 4381 4382 4383 4384 4385 | ** root page of the new table should go. meta[3] is the largest root-page ** created so far, so the new root-page is (meta[3]+1). */ rc = sqlite3BtreeGetMeta(pBt, 4, &pgnoRoot); if( rc!=SQLITE_OK ) return rc; pgnoRoot++; | | > > | > | 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 | ** root page of the new table should go. meta[3] is the largest root-page ** created so far, so the new root-page is (meta[3]+1). */ rc = sqlite3BtreeGetMeta(pBt, 4, &pgnoRoot); if( rc!=SQLITE_OK ) return rc; pgnoRoot++; /* The new root-page may not be allocated on a pointer-map page, or the ** PENDING_BYTE page. */ if( pgnoRoot==PTRMAP_PAGENO(pBt->pageSize, pgnoRoot) || pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } assert( pgnoRoot>=3 ); /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens ** to reside at pgnoRoot). |
︙ | ︙ | |||
4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 | releasePage(pMove); if( rc!=SQLITE_OK ){ return rc; } *piMoved = maxRootPgno; } maxRootPgno--; if( maxRootPgno==PTRMAP_PAGENO(pBt->pageSize, maxRootPgno) ){ maxRootPgno--; } rc = sqlite3BtreeUpdateMeta(pBt, 4, maxRootPgno); }else{ rc = freePage(pPage); releasePage(pPage); } #endif }else{ | > > > > > > > > > > | 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 | releasePage(pMove); if( rc!=SQLITE_OK ){ return rc; } *piMoved = maxRootPgno; } /* Set the new 'max-root-page' value in the database header. This ** is the old value less one, less one more if that happens to ** be a root-page number, less one again if that is the ** PENDING_BYTE_PAGE. */ maxRootPgno--; if( maxRootPgno==PENDING_BYTE_PAGE(pBt) ){ maxRootPgno--; } if( maxRootPgno==PTRMAP_PAGENO(pBt->pageSize, maxRootPgno) ){ maxRootPgno--; } assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); rc = sqlite3BtreeUpdateMeta(pBt, 4, maxRootPgno); }else{ rc = freePage(pPage); releasePage(pPage); } #endif }else{ |
︙ | ︙ | |||
4651 4652 4653 4654 4655 4656 4657 | assert( idx>=0 && idx<=15 ); rc = sqlite3pager_get(pBt->pPager, 1, (void**)&pP1); if( rc ) return rc; *pMeta = get4byte(&pP1[36 + idx*4]); sqlite3pager_unref(pP1); | | | | | 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 | assert( idx>=0 && idx<=15 ); rc = sqlite3pager_get(pBt->pPager, 1, (void**)&pP1); if( rc ) return rc; *pMeta = get4byte(&pP1[36 + idx*4]); sqlite3pager_unref(pP1); /* If autovacuumed is disabled in this build but we are trying to ** access an autovacuumed database, then make the database readonly. */ #ifdef SQLITE_OMIT_AUTOVACUUM if( idx==4 && *pMeta>0 ) pBt->readOnly = 1; #endif return SQLITE_OK; } |
︙ | ︙ |
Changes to src/os.h.
︙ | ︙ | |||
158 159 160 161 162 163 164 | ** file format. Depending on how it is changed, you might not notice ** the incompatibility right away, even running a full regression test. ** The default location of PENDING_BYTE is the first byte past the ** 1GB boundary. ** */ #define PENDING_BYTE 0x40000000 /* First byte past the 1GB boundary */ | | | 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | ** file format. Depending on how it is changed, you might not notice ** the incompatibility right away, even running a full regression test. ** The default location of PENDING_BYTE is the first byte past the ** 1GB boundary. ** */ #define PENDING_BYTE 0x40000000 /* First byte past the 1GB boundary */ /* #define PENDING_BYTE 0x5400 // Page 22 - for testing */ #define RESERVED_BYTE (PENDING_BYTE+1) #define SHARED_FIRST (PENDING_BYTE+2) #define SHARED_SIZE 510 int sqlite3OsDelete(const char*); int sqlite3OsFileExists(const char*); |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
14 15 16 17 18 19 20 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** | | > > > > > > > > > > > > | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** ** @(#) $Id: pager.c,v 1.175 2004/11/08 07:13:14 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" #include "pager.h" #include <assert.h> #include <string.h> /* ** Macros for troubleshooting. Normally turned off */ #if 0 #define TRACE1(X) sqlite3DebugPrintf(X) #define TRACE2(X,Y) sqlite3DebugPrintf(X,Y) #define TRACE3(X,Y,Z) sqlite3DebugPrintf(X,Y,Z) #define TRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W) #define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W, V) #else #define TRACE1(X) #define TRACE2(X,Y) #define TRACE3(X,Y,Z) #define TRACE4(X,Y,Z,W) #define TRACE5(X,Y,Z,W,V) #endif /* ** The following two macros are used within the TRACEX() macros above ** to print out file-descriptors. They are required so that tracing ** can be turned on when using both the regular os_unix.c and os_test.c ** backends. ** ** PAGERID() takes a pointer to a Pager struct as it's argument. The ** associated file-descriptor is returned. FILEHANDLEID() takes an OsFile ** struct as it's argument. */ #ifdef OS_TEST #define PAGERID(p) (p->fd->fd.h) #define FILEHANDLEID(fd) (fd->fd.h) #else #define PAGERID(p) (p->fd.h) #define FILEHANDLEID(fd) (fd.h) #endif |
︙ | ︙ | |||
335 336 337 338 339 340 341 | /* ** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is ** reserved for working around a windows/posix incompatibility). It is ** used in the journal to signify that the remainder of the journal file ** is devoted to storing a master journal name - there are no more pages to ** roll back. See comments for function writeMasterJournal() for details. */ | | > | 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 | /* ** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is ** reserved for working around a windows/posix incompatibility). It is ** used in the journal to signify that the remainder of the journal file ** is devoted to storing a master journal name - there are no more pages to ** roll back. See comments for function writeMasterJournal() for details. */ /* #define PAGER_MJ_PGNO(x) (PENDING_BYTE/((x)->pageSize)) */ #define PAGER_MJ_PGNO(x) ((PENDING_BYTE/((x)->pageSize))+1) /* ** Enable reference count tracking (for debugging) here: */ #ifdef SQLITE_TEST int pager3_refinfo_enable = 0; static void pager_refinfo(PgHdr *p){ |
︙ | ︙ | |||
3267 3268 3269 3270 3271 3272 3273 | } sync_exit: return rc; } #ifndef SQLITE_OMIT_AUTOVACUUM | < | 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 | } sync_exit: return rc; } #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Move the page identified by pData to location pgno in the file. ** ** There must be no references to the current page pgno. If current page ** pgno is not already in the rollback journal, it is not written there by ** by this routine. The same applies to the page pData refers to on entry to ** this routine. |
︙ | ︙ | |||
3291 3292 3293 3294 3295 3296 3297 | PgHdr *pPg = DATA_TO_PGHDR(pData); PgHdr *pPgOld; int h; assert( !pPager->stmtInUse ); assert( pPg->nRef>0 ); | > | | > > | 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 | PgHdr *pPg = DATA_TO_PGHDR(pData); PgHdr *pPgOld; int h; assert( !pPager->stmtInUse ); assert( pPg->nRef>0 ); TRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", PAGERID(pPager), pPg->pgno, pPg->needSync, pgno); /* Unlink pPg from it's hash-chain */ unlinkHashChain(pPager, pPg); /* If the cache contains a page with page-number pgno, remove it ** from it's hash chain. Also, if the PgHdr.needSync was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ pPgOld = pager_lookup(pPager, pgno); if( pPgOld ){ assert( pPgOld->nRef==0 ); unlinkHashChain(pPager, pPgOld); pPgOld->dirty = 0; if( pPgOld->needSync ){ |
︙ | ︙ |