** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.201 2004/11/03 03:52:37 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
**
** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page,
** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be
** used to test if pgno is a pointer-map page.
*/
#define PTRMAP_PAGENO(pgsz, pgno) (((pgno-2)/(pgsz/5+1))*(pgsz/5+1)+2)
#define PTRMAP_PTROFFSET(pgsz, pgno) (((pgno-2)%(pgsz/5+1)-1)*5)



/*
** The pointer map is a lookup table that contains an entry for each database
** page in the file except for page 1. In this context 'database page' refers
** to any page that is not part of the pointer map itself.  Each pointer map
** entry consists of a single byte 'type' and a 4 byte page number. The
** PTRMAP_XXX identifiers below are the valid types. The interpretation
................................................................................
/*
** This routine is called prior to sqlite3pager_commit when a transaction
** is commited for an auto-vacuum database.
*/
static int autoVacuumCommit(Btree *pBt){
  Pager *pPager = pBt->pPager;
  Pgno nFreeList;   /* Number of pages remaining on the free-list. */

  Pgno origDbSize;  /* Pages in the database file */
  Pgno finDbSize;   /* Pages in the database file after truncation */
  int i;            /* Counter variable */
  int rc;           /* Return code */
  u8 eType;

  Pgno iDbPage;              /* The database page to move */
  u8 *pDbPage = 0;           /* "" */
  MemPage *pDbMemPage = 0;   /* "" */
  Pgno iPtrPage;             /* The page that contains a pointer to iDbPage */
  MemPage *pPtrMemPage = 0;  /* "" */
  Pgno iFreePage;            /* The free-list page to move iDbPage to */
  MemPage *pFreeMemPage = 0; /* "" */

#ifndef NDEBUG
  int nRef = *sqlite3pager_stats(pPager);
#endif

  assert( pBt->autoVacuum );


  /* Figure out how many free-pages are in the database. If there are no
  ** free pages, then auto-vacuum is a no-op.
  */
  nFreeList = get4byte(&pBt->pPage1->aData[36]);
  if( nFreeList==0 ) return SQLITE_OK;


  /* TODO: This does not calculate finDbSize correctly for the case where
  ** pointer-map pages must be deallocated.
  */

  origDbSize = sqlite3pager_pagecount(pPager);

  finDbSize = origDbSize - nFreeList;

  TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origDbSize, finDbSize));

  /* Note: This is temporary code for use during development of auto-vacuum. 
  **
  ** Inspect the pointer map to make sure there are no root pages with a
  ** page number greater than finDbSize. If so, the auto-vacuum cannot
  ** proceed. This limitation will be fixed when root pages are automatically
  ** allocated at the start of the database file.
  */
  for( i=finDbSize+1; i<=origDbSize; i++ ){
    rc = ptrmapGet(pBt, i, &eType, 0);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    if( eType==PTRMAP_ROOTPAGE ){
      TRACE(("AUTOVACUUM: Cannot proceed due to root-page on page %d\n", i));
      return SQLITE_OK;
    }
  }

  /* Variable 'finDbSize' will be the size of the file in pages after
  ** the auto-vacuum has completed (the current file size minus the number
  ** of pages on the free list). Loop through the pages that lie beyond
  ** this mark, and if they are not already on the free list, move them
  ** to a free page earlier in the file (somewhere before finDbSize).
  */
  for( iDbPage=finDbSize+1; iDbPage<=origDbSize; iDbPage++ ){
    rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    assert( eType!=PTRMAP_ROOTPAGE );

    /* If iDbPage is a free or pointer map page, do not swap it. */
    if( eType==PTRMAP_FREEPAGE ||
        iDbPage==PTRMAP_PAGENO(pBt->pageSize, iDbPage) ){
      continue;
    }
    rc = getPage(pBt, iDbPage, &pDbMemPage);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    pDbPage = pDbMemPage->aData;

    /* Find the next page in the free-list that is not already at the end 
................................................................................
    do{
      if( pFreeMemPage ){
        releasePage(pFreeMemPage);
        pFreeMemPage = 0;
      }
      rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0);
      if( rc!=SQLITE_OK ) goto autovacuum_out;
      assert( iFreePage<=origDbSize );
    }while( iFreePage>finDbSize );

    /* Move page iDbPage from it's current location to page number iFreePage */
    TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", 
        iDbPage, iFreePage, iPtrPage, eType));
    releasePage(pFreeMemPage);
    pFreeMemPage = 0;
    rc = sqlite3pager_movepage(pPager, pDbPage, iFreePage);
................................................................................
    */
    if( eType==PTRMAP_BTREE ){
      rc = setChildPtrmaps(pDbMemPage);
      if( rc!=SQLITE_OK ) goto autovacuum_out;
    }else{
      Pgno nextOvfl = get4byte(pDbPage);
      if( nextOvfl!=0 ){
        assert( nextOvfl<=origDbSize );
        rc = ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage);
        if( rc!=SQLITE_OK ) goto autovacuum_out;
      }
    }
    releasePage(pDbMemPage);
    pDbMemPage = 0;

................................................................................
    modifyPagePointer(pPtrMemPage, iDbPage, iFreePage, eType);
    rc = ptrmapPut(pBt, iFreePage, eType, iPtrPage);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    releasePage(pPtrMemPage);
  }

  /* The entire free-list has been swapped to the end of the file. So
  ** truncate the database file to finDbSize pages and consider the
  ** free-list empty.
  */
  rc = sqlite3pager_write(pBt->pPage1->aData);
  if( rc!=SQLITE_OK ) goto autovacuum_out;
  put4byte(&pBt->pPage1->aData[32], 0);
  put4byte(&pBt->pPage1->aData[36], 0);
  rc = sqlite3pager_truncate(pBt->pPager, finDbSize);
  if( rc!=SQLITE_OK ) goto autovacuum_out;

autovacuum_out:
  /* TODO: A goto autovacuum_out; will fail to call releasePage() on 
  ** outstanding references. Fix.
  */
  assert( nRef==*sqlite3pager_stats(pPager) );
................................................................................
    }
  }else{
    /* There are no pages on the freelist, so create a new page at the
    ** end of the file */
    *pPgno = sqlite3pager_pagecount(pBt->pPager) + 1;

#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum && *pPgno==PTRMAP_PAGENO(pBt->pageSize, *pPgno) ){
      /* If *pPgno refers to a pointer-map page, allocate two new pages
      ** at the end of the file instead of one. The first allocated page
      ** becomes a new pointer-map page, the second is used by the caller.
      */
      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno));
      (*pPgno)++;
    }
................................................................................
  while( nPayload>0 ){
    if( spaceLeft==0 ){
#ifndef SQLITE_OMIT_AUTOVACUUM
      Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */
#endif
      rc = allocatePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl);
#ifndef SQLITE_OMIT_AUTOVACUUM
      /* If the database supports auto-vacuum, add an entry to the 
      ** pointer-map for the overflow page just allocated. If the page just 
      ** allocated was the first in the overflow list, then the balance() 
      ** routine may adjust the pointer-map entry later.
      */
      if( pBt->autoVacuum && rc==0 ){
        if( pgnoPtrmap!=0 ){
          rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW2, pgnoPtrmap);
        }else{
          rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno);
        }
      }
#endif
      if( rc ){
        releasePage(pToRelease);
        clearCell(pPage, pCell);
        return rc;
      }
................................................................................
      ptr[0] = ptr[-2];
      ptr[1] = ptr[-1];
    }
    put2byte(&data[ins], idx);
    put2byte(&data[hdr+3], pPage->nCell);
    pPage->idxShift = 1;
    pageIntegrity(pPage);
  }

#ifndef SQLITE_OMIT_AUTOVACUUM
  if( pPage->pBt->autoVacuum ){
    /* The cell may contain a pointer to an overflow page. If so, write
    ** the entry for the overflow page into the pointer map.
    */
    CellInfo info;
    parseCellPtr(pPage, pCell, &info);
    if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){
      Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
      int rc = ptrmapPut(pPage->pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno);
      if( rc!=SQLITE_OK ) return rc;
    }
  }
#endif


  return SQLITE_OK;
}

/*
** Add a list of cells to a page.  The page should be initially empty.
** The cells are guaranteed to fit on the page.
*/

** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.202 2004/11/03 11:37:08 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
**
** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page,
** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be
** used to test if pgno is a pointer-map page.
*/
#define PTRMAP_PAGENO(pgsz, pgno) (((pgno-2)/(pgsz/5+1))*(pgsz/5+1)+2)
#define PTRMAP_PTROFFSET(pgsz, pgno) (((pgno-2)%(pgsz/5+1)-1)*5)

#define PTRMAP_ISPAGE(pgsz, pgno) (PTRMAP_PAGENO(pgsz,pgno)==pgno)

/*
** The pointer map is a lookup table that contains an entry for each database
** page in the file except for page 1. In this context 'database page' refers
** to any page that is not part of the pointer map itself.  Each pointer map
** entry consists of a single byte 'type' and a 4 byte page number. The
** PTRMAP_XXX identifiers below are the valid types. The interpretation
................................................................................
/*
** This routine is called prior to sqlite3pager_commit when a transaction
** is commited for an auto-vacuum database.
*/
static int autoVacuumCommit(Btree *pBt){
  Pager *pPager = pBt->pPager;
  Pgno nFreeList;   /* Number of pages remaining on the free-list. */
  int nPtrMap;      /* Number of pointer-map pages deallocated */
  Pgno origSize;  /* Pages in the database file */
  Pgno finSize;   /* Pages in the database file after truncation */
  int i;            /* Counter variable */
  int rc;           /* Return code */
  u8 eType;
  int pgsz = pBt->pageSize;  /* Page size for this database */
  Pgno iDbPage;              /* The database page to move */
  u8 *pDbPage = 0;           /* "" */
  MemPage *pDbMemPage = 0;   /* "" */
  Pgno iPtrPage;             /* The page that contains a pointer to iDbPage */
  MemPage *pPtrMemPage = 0;  /* "" */
  Pgno iFreePage;            /* The free-list page to move iDbPage to */
  MemPage *pFreeMemPage = 0; /* "" */

#ifndef NDEBUG
  int nRef = *sqlite3pager_stats(pPager);
#endif

  assert( pBt->autoVacuum );
  assert( 0==PTRMAP_ISPAGE(pgsz, sqlite3pager_pagecount(pPager)) );

  /* Figure out how many free-pages are in the database. If there are no
  ** free pages, then auto-vacuum is a no-op.
  */
  nFreeList = get4byte(&pBt->pPage1->aData[36]);
  if( nFreeList==0 ){
    return SQLITE_OK;
  }




  origSize = sqlite3pager_pagecount(pPager);
  nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pgsz, origSize)+pgsz/5)/(pgsz/5);
  finSize = origSize - nFreeList - nPtrMap;

  TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize));

  /* Note: This is temporary code for use during development of auto-vacuum. 
  **
  ** Inspect the pointer map to make sure there are no root pages with a
  ** page number greater than finSize. If so, the auto-vacuum cannot
  ** proceed. This limitation will be fixed when root pages are automatically
  ** allocated at the start of the database file.
  */
  for( i=finSize+1; i<=origSize; i++ ){
    rc = ptrmapGet(pBt, i, &eType, 0);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    if( eType==PTRMAP_ROOTPAGE ){
      TRACE(("AUTOVACUUM: Cannot proceed due to root-page on page %d\n", i));
      return SQLITE_OK;
    }
  }

  /* Variable 'finSize' will be the size of the file in pages after
  ** the auto-vacuum has completed (the current file size minus the number
  ** of pages on the free list). Loop through the pages that lie beyond
  ** this mark, and if they are not already on the free list, move them
  ** to a free page earlier in the file (somewhere before finSize).
  */
  for( iDbPage=finSize+1; iDbPage<=origSize; iDbPage++ ){
    rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    assert( eType!=PTRMAP_ROOTPAGE );

    /* If iDbPage is a free or pointer map page, do not swap it. */
    if( eType==PTRMAP_FREEPAGE || PTRMAP_ISPAGE(pgsz, iDbPage) ){

      continue;
    }
    rc = getPage(pBt, iDbPage, &pDbMemPage);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    pDbPage = pDbMemPage->aData;

    /* Find the next page in the free-list that is not already at the end 
................................................................................
    do{
      if( pFreeMemPage ){
        releasePage(pFreeMemPage);
        pFreeMemPage = 0;
      }
      rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0);
      if( rc!=SQLITE_OK ) goto autovacuum_out;
      assert( iFreePage<=origSize );
    }while( iFreePage>finSize );

    /* Move page iDbPage from it's current location to page number iFreePage */
    TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", 
        iDbPage, iFreePage, iPtrPage, eType));
    releasePage(pFreeMemPage);
    pFreeMemPage = 0;
    rc = sqlite3pager_movepage(pPager, pDbPage, iFreePage);
................................................................................
    */
    if( eType==PTRMAP_BTREE ){
      rc = setChildPtrmaps(pDbMemPage);
      if( rc!=SQLITE_OK ) goto autovacuum_out;
    }else{
      Pgno nextOvfl = get4byte(pDbPage);
      if( nextOvfl!=0 ){
        assert( nextOvfl<=origSize );
        rc = ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage);
        if( rc!=SQLITE_OK ) goto autovacuum_out;
      }
    }
    releasePage(pDbMemPage);
    pDbMemPage = 0;

................................................................................
    modifyPagePointer(pPtrMemPage, iDbPage, iFreePage, eType);
    rc = ptrmapPut(pBt, iFreePage, eType, iPtrPage);
    if( rc!=SQLITE_OK ) goto autovacuum_out;
    releasePage(pPtrMemPage);
  }

  /* The entire free-list has been swapped to the end of the file. So
  ** truncate the database file to finSize pages and consider the
  ** free-list empty.
  */
  rc = sqlite3pager_write(pBt->pPage1->aData);
  if( rc!=SQLITE_OK ) goto autovacuum_out;
  put4byte(&pBt->pPage1->aData[32], 0);
  put4byte(&pBt->pPage1->aData[36], 0);
  rc = sqlite3pager_truncate(pBt->pPager, finSize);
  if( rc!=SQLITE_OK ) goto autovacuum_out;

autovacuum_out:
  /* TODO: A goto autovacuum_out; will fail to call releasePage() on 
  ** outstanding references. Fix.
  */
  assert( nRef==*sqlite3pager_stats(pPager) );
................................................................................
    }
  }else{
    /* There are no pages on the freelist, so create a new page at the
    ** end of the file */
    *pPgno = sqlite3pager_pagecount(pBt->pPager) + 1;

#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt->pageSize, *pPgno) ){
      /* If *pPgno refers to a pointer-map page, allocate two new pages
      ** at the end of the file instead of one. The first allocated page
      ** becomes a new pointer-map page, the second is used by the caller.
      */
      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno));
      (*pPgno)++;
    }
................................................................................
  while( nPayload>0 ){
    if( spaceLeft==0 ){
#ifndef SQLITE_OMIT_AUTOVACUUM
      Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */
#endif
      rc = allocatePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl);
#ifndef SQLITE_OMIT_AUTOVACUUM
      /* If the database supports auto-vacuum, and the second or subsequent
      ** overflow page is being allocated, add an entry to the pointer-map
      ** for that page now. The entry for the first overflow page will be
      ** added later, by the insertCell() routine.
      */
      if( pBt->autoVacuum && pgnoPtrmap!=0 && rc==SQLITE_OK ){

        rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW2, pgnoPtrmap);



      }
#endif
      if( rc ){
        releasePage(pToRelease);
        clearCell(pPage, pCell);
        return rc;
      }
................................................................................
      ptr[0] = ptr[-2];
      ptr[1] = ptr[-1];
    }
    put2byte(&data[ins], idx);
    put2byte(&data[hdr+3], pPage->nCell);
    pPage->idxShift = 1;
    pageIntegrity(pPage);


#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pPage->pBt->autoVacuum ){
      /* The cell may contain a pointer to an overflow page. If so, write
      ** the entry for the overflow page into the pointer map.
      */
      CellInfo info;
      parseCellPtr(pPage, pCell, &info);
      if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){
        Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]);
        int rc = ptrmapPut(pPage->pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno);
        if( rc!=SQLITE_OK ) return rc;
      }
    }
#endif
  }

  return SQLITE_OK;
}

/*
** Add a list of cells to a page.  The page should be initially empty.
** The cells are guaranteed to fit on the page.
*/