/ Check-in [d6335698]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Auto-vacuum: Account for the page reserved for windows locking (PENDING_BYTE). (CVS 2076)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: d6335698696c7b651bbc436c5177d87eb57a8934
User & Date: danielk1977 2004-11-08 07:13:14
Context
2004-11-08
09:26
Test auto-vacuum mode for crash-proofness. Also fix a bug related to the same. (CVS 2077) check-in: 839ad771 user: danielk1977 tags: trunk
07:13
Auto-vacuum: Account for the page reserved for windows locking (PENDING_BYTE). (CVS 2076) check-in: d6335698 user: danielk1977 tags: trunk
2004-11-07
13:01
Reindex tests added and bugs fixed. (CVS 2075) check-in: ad433ec2 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

     5      5   ** a legal notice, here is a blessing:
     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12         -** $Id: btree.c,v 1.213 2004/11/06 12:26:08 danielk1977 Exp $
           12  +** $Id: btree.c,v 1.214 2004/11/08 07:13:14 danielk1977 Exp $
    13     13   **
    14     14   ** This file implements a external (disk-based) database using BTrees.
    15     15   ** For a detailed discussion of BTrees, refer to
    16     16   **
    17     17   **     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
    18     18   **     "Sorting And Searching", pages 473-480. Addison-Wesley
    19     19   **     Publishing Company, Reading, Massachusetts.
................................................................................
   390    390   ** be defined locally, but now we use the varint routines in the util.c
   391    391   ** file.
   392    392   */
   393    393   #define getVarint    sqlite3GetVarint
   394    394   #define getVarint32  sqlite3GetVarint32
   395    395   #define putVarint    sqlite3PutVarint
   396    396   
          397  +/* The database page the PENDING_BYTE occupies. This page is never used.
          398  +** TODO: This macro is very similary to PAGER_MJ_PGNO() in pager.c. They
          399  +** should possibly be consolidated (presumably in pager.h).
          400  +*/
          401  +#define PENDING_BYTE_PAGE(pBt) ((PENDING_BYTE/(pBt)->pageSize)+1)
          402  +
   397    403   #ifndef SQLITE_OMIT_AUTOVACUUM
   398         -
   399    404   /*
   400    405   ** These two macros define the location of the pointer-map entry for a 
   401    406   ** database page. The first argument to each is the page size used 
   402    407   ** by the database (often 1024). The second is the page number to look
   403    408   ** up in the pointer map.
   404    409   **
   405    410   ** PTRMAP_PAGENO returns the database page number of the pointer-map
   406    411   ** page that stores the required pointer. PTRMAP_PTROFFSET returns
   407    412   ** the offset of the requested map entry.
   408    413   **
   409    414   ** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page,
   410    415   ** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be
   411         -** used to test if pgno is a pointer-map page.
          416  +** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements
          417  +** this test.
   412    418   */
   413    419   #define PTRMAP_PAGENO(pgsz, pgno) (((pgno-2)/(pgsz/5+1))*(pgsz/5+1)+2)
   414    420   #define PTRMAP_PTROFFSET(pgsz, pgno) (((pgno-2)%(pgsz/5+1)-1)*5)
   415         -
   416    421   #define PTRMAP_ISPAGE(pgsz, pgno) (PTRMAP_PAGENO(pgsz,pgno)==pgno)
   417    422   
   418    423   /*
   419    424   ** The pointer map is a lookup table that contains an entry for each database
   420    425   ** page in the file except for page 1. In this context 'database page' refers
   421    426   ** to any page that is not part of the pointer map itself.  Each pointer map
   422    427   ** entry consists of a single byte 'type' and a 4 byte page number. The
................................................................................
   455    460   */
   456    461   static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){
   457    462     u8 *pPtrmap;    /* The pointer map page */
   458    463     Pgno iPtrmap;   /* The pointer map page number */
   459    464     int offset;     /* Offset in pointer map page */
   460    465     int rc;
   461    466   
          467  +  assert( key!=0 );
   462    468     iPtrmap = PTRMAP_PAGENO(pBt->pageSize, key);
   463    469     rc = sqlite3pager_get(pBt->pPager, iPtrmap, (void **)&pPtrmap);
   464    470     if( rc!=SQLITE_OK ){
   465    471       return rc;
   466    472     }
   467    473     offset = PTRMAP_PTROFFSET(pBt->pageSize, key);
   468    474   
................................................................................
  1751   1757       *nTrunc = 0;
  1752   1758       return SQLITE_OK;
  1753   1759     }
  1754   1760   
  1755   1761     origSize = sqlite3pager_pagecount(pPager);
  1756   1762     nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pgsz, origSize)+pgsz/5)/(pgsz/5);
  1757   1763     finSize = origSize - nFreeList - nPtrMap;
         1764  +  if( origSize>PENDING_BYTE_PAGE(pBt) && finSize<=PENDING_BYTE_PAGE(pBt) ){
         1765  +    finSize--;
         1766  +    if( PTRMAP_ISPAGE(pBt->pageSize, finSize) ){
         1767  +      finSize--;
         1768  +    }
         1769  +  }
  1758   1770     TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize));
  1759   1771   
  1760   1772     /* Variable 'finSize' will be the size of the file in pages after
  1761   1773     ** the auto-vacuum has completed (the current file size minus the number
  1762   1774     ** of pages on the free list). Loop through the pages that lie beyond
  1763   1775     ** this mark, and if they are not already on the free list, move them
  1764   1776     ** to a free page earlier in the file (somewhere before finSize).
  1765   1777     */
  1766   1778     for( iDbPage=finSize+1; iDbPage<=origSize; iDbPage++ ){
         1779  +    /* If iDbPage is a pointer map page, or the pending-byte page, skip it. */
         1780  +    if( PTRMAP_ISPAGE(pgsz, iDbPage) || iDbPage==PENDING_BYTE_PAGE(pBt) ){
         1781  +      continue;
         1782  +    }
         1783  +
  1767   1784       rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage);
  1768   1785       if( rc!=SQLITE_OK ) goto autovacuum_out;
  1769   1786       assert( eType!=PTRMAP_ROOTPAGE );
  1770   1787   
  1771         -    /* If iDbPage is a free or pointer map page, do not swap it.
  1772         -    ** TODO: Instead, make sure the page is in the journal file.
  1773         -    */
  1774         -    if( eType==PTRMAP_FREEPAGE || PTRMAP_ISPAGE(pgsz, iDbPage) ){
         1788  +    /* If iDbPage is free, do not swap it.  */
         1789  +    if( eType==PTRMAP_FREEPAGE ){
  1775   1790         continue;
  1776   1791       }
  1777   1792       rc = getPage(pBt, iDbPage, &pDbMemPage);
  1778   1793       if( rc!=SQLITE_OK ) goto autovacuum_out;
  1779   1794   
  1780   1795       /* Find the next page in the free-list that is not already at the end 
  1781   1796       ** of the file. A page can be pulled off the free list using the 
................................................................................
  3048   3063   #ifndef SQLITE_OMIT_AUTOVACUUM
  3049   3064       if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt->pageSize, *pPgno) ){
  3050   3065         /* If *pPgno refers to a pointer-map page, allocate two new pages
  3051   3066         ** at the end of the file instead of one. The first allocated page
  3052   3067         ** becomes a new pointer-map page, the second is used by the caller.
  3053   3068         */
  3054   3069         TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno));
         3070  +      assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
  3055   3071         (*pPgno)++;
  3056   3072       }
  3057   3073   #endif
  3058   3074   
         3075  +    assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
  3059   3076       rc = getPage(pBt, *pPgno, ppPage);
  3060   3077       if( rc ) return rc;
  3061   3078       rc = sqlite3pager_write((*ppPage)->aData);
  3062   3079       TRACE(("ALLOCATE: %d from end of file\n", *pPgno));
  3063   3080     }
         3081  +
         3082  +  assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
  3064   3083     return rc;
  3065   3084   }
  3066   3085   
  3067   3086   /*
  3068   3087   ** Add a page of the database file to the freelist.
  3069   3088   **
  3070   3089   ** sqlite3pager_unref() is NOT called for pPage.
................................................................................
  4379   4398       ** root page of the new table should go. meta[3] is the largest root-page
  4380   4399       ** created so far, so the new root-page is (meta[3]+1).
  4381   4400       */
  4382   4401       rc = sqlite3BtreeGetMeta(pBt, 4, &pgnoRoot);
  4383   4402       if( rc!=SQLITE_OK ) return rc;
  4384   4403       pgnoRoot++;
  4385   4404   
  4386         -    /* The new root-page may not be allocated on a pointer-map page. */
  4387         -    if( pgnoRoot==PTRMAP_PAGENO(pBt->pageSize, pgnoRoot) ){
         4405  +    /* The new root-page may not be allocated on a pointer-map page, or the
         4406  +    ** PENDING_BYTE page.
         4407  +    */
         4408  +    if( pgnoRoot==PTRMAP_PAGENO(pBt->pageSize, pgnoRoot) ||
         4409  +        pgnoRoot==PENDING_BYTE_PAGE(pBt) ){
  4388   4410         pgnoRoot++;
  4389   4411       }
  4390   4412       assert( pgnoRoot>=3 );
  4391   4413   
  4392   4414       /* Allocate a page. The page that currently resides at pgnoRoot will
  4393   4415       ** be moved to the allocated page (unless the allocated page happens
  4394   4416       ** to reside at pgnoRoot).
................................................................................
  4612   4634           releasePage(pMove);
  4613   4635           if( rc!=SQLITE_OK ){
  4614   4636             return rc;
  4615   4637           }
  4616   4638           *piMoved = maxRootPgno;
  4617   4639         }
  4618   4640   
         4641  +      /* Set the new 'max-root-page' value in the database header. This
         4642  +      ** is the old value less one, less one more if that happens to
         4643  +      ** be a root-page number, less one again if that is the
         4644  +      ** PENDING_BYTE_PAGE.
         4645  +      */
  4619   4646         maxRootPgno--;
         4647  +      if( maxRootPgno==PENDING_BYTE_PAGE(pBt) ){
         4648  +        maxRootPgno--;
         4649  +      }
  4620   4650         if( maxRootPgno==PTRMAP_PAGENO(pBt->pageSize, maxRootPgno) ){
  4621   4651           maxRootPgno--;
  4622   4652         }
         4653  +      assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) );
         4654  +
  4623   4655         rc = sqlite3BtreeUpdateMeta(pBt, 4, maxRootPgno);
  4624   4656       }else{
  4625   4657         rc = freePage(pPage);
  4626   4658         releasePage(pPage);
  4627   4659       }
  4628   4660   #endif
  4629   4661     }else{
................................................................................
  4651   4683   
  4652   4684     assert( idx>=0 && idx<=15 );
  4653   4685     rc = sqlite3pager_get(pBt->pPager, 1, (void**)&pP1);
  4654   4686     if( rc ) return rc;
  4655   4687     *pMeta = get4byte(&pP1[36 + idx*4]);
  4656   4688     sqlite3pager_unref(pP1);
  4657   4689   
  4658         -  /* If autovacuumed is disabled in the implementation but we are
  4659         -  ** trying to access an autovacuumed database, then make the
  4660         -  ** database readonly. */
         4690  +  /* If autovacuumed is disabled in this build but we are trying to 
         4691  +  ** access an autovacuumed database, then make the database readonly. 
         4692  +  */
  4661   4693   #ifdef SQLITE_OMIT_AUTOVACUUM
  4662   4694     if( idx==4 && *pMeta>0 ) pBt->readOnly = 1;
  4663   4695   #endif
  4664   4696   
  4665   4697     return SQLITE_OK;
  4666   4698   }
  4667   4699   

Changes to src/os.h.

   158    158   ** file format.  Depending on how it is changed, you might not notice
   159    159   ** the incompatibility right away, even running a full regression test.
   160    160   ** The default location of PENDING_BYTE is the first byte past the
   161    161   ** 1GB boundary.
   162    162   **
   163    163   */
   164    164   #define PENDING_BYTE      0x40000000  /* First byte past the 1GB boundary */
   165         -/* #define PENDING_BYTE     0x5400   // Page 20 - for testing */
          165  +/* #define PENDING_BYTE     0x5400   // Page 22 - for testing */
   166    166   #define RESERVED_BYTE     (PENDING_BYTE+1)
   167    167   #define SHARED_FIRST      (PENDING_BYTE+2)
   168    168   #define SHARED_SIZE       510
   169    169   
   170    170   
   171    171   int sqlite3OsDelete(const char*);
   172    172   int sqlite3OsFileExists(const char*);

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.174 2004/11/06 12:26:08 danielk1977 Exp $
           21  +** @(#) $Id: pager.c,v 1.175 2004/11/08 07:13:14 danielk1977 Exp $
    22     22   */
    23     23   #include "sqliteInt.h"
    24     24   #include "os.h"
    25     25   #include "pager.h"
    26     26   #include <assert.h>
    27     27   #include <string.h>
    28     28   
................................................................................
    30     30   ** Macros for troubleshooting.  Normally turned off
    31     31   */
    32     32   #if 0
    33     33   #define TRACE1(X)       sqlite3DebugPrintf(X)
    34     34   #define TRACE2(X,Y)     sqlite3DebugPrintf(X,Y)
    35     35   #define TRACE3(X,Y,Z)   sqlite3DebugPrintf(X,Y,Z)
    36     36   #define TRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W)
           37  +#define TRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W, V)
    37     38   #else
    38     39   #define TRACE1(X)
    39     40   #define TRACE2(X,Y)
    40     41   #define TRACE3(X,Y,Z)
    41     42   #define TRACE4(X,Y,Z,W)
           43  +#define TRACE5(X,Y,Z,W,V)
    42     44   #endif
    43     45   
           46  +/*
           47  +** The following two macros are used within the TRACEX() macros above
           48  +** to print out file-descriptors. They are required so that tracing
           49  +** can be turned on when using both the regular os_unix.c and os_test.c
           50  +** backends.
           51  +**
           52  +** PAGERID() takes a pointer to a Pager struct as it's argument. The
           53  +** associated file-descriptor is returned. FILEHANDLEID() takes an OsFile
           54  +** struct as it's argument.
           55  +*/
    44     56   #ifdef OS_TEST
    45     57   #define PAGERID(p) (p->fd->fd.h)
    46     58   #define FILEHANDLEID(fd) (fd->fd.h)
    47     59   #else
    48     60   #define PAGERID(p) (p->fd.h)
    49     61   #define FILEHANDLEID(fd) (fd.h)
    50     62   #endif
................................................................................
   335    347   /*
   336    348   ** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is
   337    349   ** reserved for working around a windows/posix incompatibility). It is
   338    350   ** used in the journal to signify that the remainder of the journal file 
   339    351   ** is devoted to storing a master journal name - there are no more pages to
   340    352   ** roll back. See comments for function writeMasterJournal() for details.
   341    353   */
   342         -#define PAGER_MJ_PGNO(x) (PENDING_BYTE/((x)->pageSize))
          354  +/* #define PAGER_MJ_PGNO(x) (PENDING_BYTE/((x)->pageSize)) */
          355  +#define PAGER_MJ_PGNO(x) ((PENDING_BYTE/((x)->pageSize))+1)
   343    356   
   344    357   /*
   345    358   ** Enable reference count tracking (for debugging) here:
   346    359   */
   347    360   #ifdef SQLITE_TEST
   348    361     int pager3_refinfo_enable = 0;
   349    362     static void pager_refinfo(PgHdr *p){
................................................................................
  3267   3280     }
  3268   3281   
  3269   3282   sync_exit:
  3270   3283     return rc;
  3271   3284   }
  3272   3285   
  3273   3286   #ifndef SQLITE_OMIT_AUTOVACUUM
  3274         -
  3275   3287   /*
  3276   3288   ** Move the page identified by pData to location pgno in the file. 
  3277   3289   **
  3278   3290   ** There must be no references to the current page pgno. If current page
  3279   3291   ** pgno is not already in the rollback journal, it is not written there by
  3280   3292   ** by this routine. The same applies to the page pData refers to on entry to
  3281   3293   ** this routine.
................................................................................
  3291   3303     PgHdr *pPg = DATA_TO_PGHDR(pData);
  3292   3304     PgHdr *pPgOld; 
  3293   3305     int h;
  3294   3306   
  3295   3307     assert( !pPager->stmtInUse );
  3296   3308     assert( pPg->nRef>0 );
  3297   3309   
  3298         -  TRACE4("MOVE %d page %d moves to %d\n", PAGERID(pPager), pPg->pgno, pgno);
         3310  +  TRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", 
         3311  +      PAGERID(pPager), pPg->pgno, pPg->needSync, pgno);
  3299   3312   
  3300   3313     /* Unlink pPg from it's hash-chain */
  3301   3314     unlinkHashChain(pPager, pPg);
  3302   3315   
  3303   3316     /* If the cache contains a page with page-number pgno, remove it
  3304         -  ** from it's hash chain.
         3317  +  ** from it's hash chain. Also, if the PgHdr.needSync was set for 
         3318  +  ** page pgno before the 'move' operation, it needs to be retained 
         3319  +  ** for the page moved there.
  3305   3320     */
  3306   3321     pPgOld = pager_lookup(pPager, pgno);
  3307   3322     if( pPgOld ){
  3308   3323       assert( pPgOld->nRef==0 );
  3309   3324       unlinkHashChain(pPager, pPgOld);
  3310   3325       pPgOld->dirty = 0;
  3311   3326       if( pPgOld->needSync ){