SQLite

/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.412 2007/08/24 11:52:29 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
#include "btreeInt.h"

    sqlite3PagerSetReiniter(pBt->pPager, pageReinit);
    pBt->pCursor = 0;
    pBt->pPage1 = 0;
    pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager);
    pBt->pageSize = get2byte(&zDbHeader[16]);
    if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE
         || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){
      pBt->pageSize = sqlite3PagerSetPagesize(pBt->pPager, 0);
      pBt->maxEmbedFrac = 64;   /* 25% */
      pBt->minEmbedFrac = 32;   /* 12.5% */
      pBt->minLeafFrac = 32;    /* 12.5% */
#ifndef SQLITE_OMIT_AUTOVACUUM
      /* If the magic name ":memory:" will create an in-memory database, then
      ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if
      ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.374 2007/08/24 11:52:29 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*

/*
** Create a new page cache and put a pointer to the page cache in *ppPager.
** The file to be cached need not exist.  The file is not locked until
** the first call to sqlite3PagerGet() and is only held open until the
** last page is released using sqlite3PagerUnref().
**
** If zFilename is NULL then a randomly-named temporary file is created
** and
** automatically when it is closed.
**
** If zFilename is ":memory:" then all information is held in cache.
** It is never written to disk.  This can be used to implement an
** in-memory database.
*/
int sqlite3PagerOpen(

  int i;
  int tempFile = 0;
  int memDb = 0;
  int readOnly = 0;
  int useJournal = (flags & PAGER_OMIT_JOURNAL)==0;
  int noReadlock = (flags & PAGER_NO_READLOCK)!=0;
  int journalFileSize = sqlite3JournalSize(pVfs);
  int nDefaultPage = SQLITE_DEFAULT_PAGE_SIZE;     /* Default page size. */

  /* The default return is a NULL pointer */
  *ppPager = 0;

  /* Allocate memory for the pager structure */
  pPager = sqlite3MallocZero(
    sizeof(*pPager) +           /* Pager structure */

          rc = SQLITE_CANTOPEN;
        }else{
          int oflag = 
              (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB);
          int fout = 0;
          rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, oflag, &fout);
          readOnly = (fout&SQLITE_OPEN_READONLY);
 
          /* If the file was successfully opened for read/write access,
          ** choose a default page size in case we have to create the
          ** database file. The default page size is the maximum of:
          **
          **    + SQLITE_DEFAULT_PAGE_SIZE,
          **    + The value returned by sqlite3OsSectorSize()
          **    + The largest page size that can be written atomically.
          */
          if( rc==SQLITE_OK && !readOnly ){
            int iSectorSize = sqlite3OsSectorSize(pPager->fd);
            if( nDefaultPage<iSectorSize ){
              nDefaultPage = iSectorSize;
            }
#ifdef SQLITE_ENABLE_ATOMIC_WRITE
            {
              int iDc = sqlite3OsDeviceCharacteristics(pPager->fd);
              int ii;
              assert(SQLITE_IOCAP_ATOMIC512==(512>>8));
              assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8));
              assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536);
              for(ii=nDefaultPage; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){
                if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ) nDefaultPage = ii;
              }
            }
#endif
            if( nDefaultPage>SQLITE_MAX_DEFAULT_PAGE_SIZE ){
              nDefaultPage = SQLITE_MAX_DEFAULT_PAGE_SIZE;
            }
          }
        }
      }
    }
  }else{
    /* If a temporary file is requested, it is not opened immediately.
    ** In this case we accept the default page size and delay actually
    ** opening the file until the first call to OsWrite().
    */ 
    tempFile = 1;
    pPager->state = PAGER_EXCLUSIVE;
  }

  if( pPager && rc==SQLITE_OK ){
    pPager->pTmpSpace = (char *)sqlite3_malloc(nDefaultPage);
  }

  /* If an error occured in either of the blocks above.
  ** Free the Pager structure and close the file.
  ** Since the pager is not allocated there is no need to set 
  ** any Pager.errMask variables.
  */

  /* pPager->journalOpen = 0; */
  pPager->useJournal = useJournal && !memDb;
  pPager->noReadlock = noReadlock && readOnly;
  /* pPager->stmtOpen = 0; */
  /* pPager->stmtInUse = 0; */
  /* pPager->nRef = 0; */
  pPager->dbSize = memDb-1;
  pPager->pageSize = nDefaultPage;
  /* pPager->stmtSize = 0; */
  /* pPager->stmtJSize = 0; */
  /* pPager->nPage = 0; */
  pPager->mxPage = 100;
  pPager->mxPgno = SQLITE_MAX_PAGE_COUNT;
  assert( PAGER_UNLOCK==0 );
  /* pPager->state = PAGER_UNLOCK; */

/*
** Set the page size.  Return the new size.  If the suggest new page
** size is inappropriate, then an alternative page size is selected
** and returned.
*/
int sqlite3PagerSetPagesize(Pager *pPager, int pageSize){
  assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) );
  if( pageSize && !pPager->memDb && pPager->nRef==0 ){
    pagerEnter(pPager);
    pager_reset(pPager);
    pPager->pageSize = pageSize;
    setSectorSize(pPager);
    pagerLeave(pPager);
    sqlite3_free(pPager->pTmpSpace);
    pPager->pTmpSpace = sqlite3_malloc(pageSize);
  }
  return pPager->pageSize;
}

      while( pPg && !pPg->dirty ){ pPg = pPg->pDirty; }
      pPager->pDirty = pPg;
      goto sync_exit;
    }
    pPager->pDirty = 0;

    /* Sync the database file. */
    if( !pPager->noSync ){ 
      rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
    }
    IOTRACE(("DBSYNC %p\n", pPager))

    pPager->state = PAGER_SYNCED;
  }else if( MEMDB && nTrunc!=0 ){
    rc = sqlite3PagerTruncate(pPager, nTrunc);

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** 
** This file defines various limits of what SQLite can process.
**
** @(#) $Id: sqliteLimit.h,v 1.2 2007/08/24 11:52:29 danielk1977 Exp $
*/

/*
** The maximum length of a TEXT or BLOB in bytes.   This also
** limits the size of a row in a table or index.
**
** The hard limit is the ability of a 32-bit signed integer

/*
** The default size of a database page.
*/
#ifndef SQLITE_DEFAULT_PAGE_SIZE
# define SQLITE_DEFAULT_PAGE_SIZE 1024
#endif

/*
** Ordinarily, if no value is explicitly provided, SQLite creates databases
** with page size SQLITE_DEFAULT_PAGE_SIZE. However, based on certain
** device characteristics (sector-size and atomic write() support),
** SQLite may choose a larger value. This constant is the maximum value
** SQLite will choose on it's own.
*/
#ifndef SQLITE_MAX_DEFAULT_PAGE_SIZE
# define SQLITE_MAX_DEFAULT_PAGE_SIZE 8192
#endif

/* Maximum page size.  The upper bound on this value is 32768.  This a limit
** imposed by the necessity of storing the value in a 2-byte unsigned integer
** and the fact that the page size must be a power of 2.
*/
#ifndef SQLITE_MAX_PAGE_SIZE
# define SQLITE_MAX_PAGE_SIZE 32768
#endif

          sqlite3OsDelete(pVfs, zMaster, 0);
          sqlite3_free(zMaster);
          return rc;
        }
      }
    }


    /* Sync the master journal file. If the IOCAP_SEQUENTIAL device
    ** flag is set this is not required.
    */
    zMainFile = sqlite3BtreeGetDirname(db->aDb[0].pBt);
    if( (needSync 
     && (0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL))
     && (rc=sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL))!=SQLITE_OK) ){
      sqlite3OsCloseFree(pMaster);
      sqlite3OsDelete(pVfs, zMaster, 0);
      sqlite3_free(zMaster);
      return rc;
    }

#
# The focus of this file is testing the ability of the database to
# uses its rollback journal to recover intact (no database corruption)
# from a power failure during the middle of a COMMIT. Even more
# specifically, the tests in this file verify this functionality
# for storage mediums with various sector sizes.
#
# $Id: crash2.test,v 1.5 2007/08/24 11:52:29 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !crashtest {
  finish_test
  return
}

db close

# This test is designed to check that the crash-test infrastructure
# can create files that do not consist of an integer number of
# simulated disk blocks (i.e. 3KB file using 2KB disk blocks).
#
do_test crash2-1.1 {
  crashsql -delay 500 -file test.db -blocksize 2048 {
    PRAGMA auto_vacuum=OFF;
    PRAGMA page_size=1024;
    BEGIN;
    CREATE TABLE abc AS SELECT 1 AS a, 2 AS b, 3 AS c;
    CREATE TABLE def AS SELECT 1 AS d, 2 AS e, 3 AS f;
    COMMIT;
  }
  file size test.db
} {3072}

for {set ii 0} {$ii < 5} {incr ii} {

  # Simple test using the database created above: Create a new
  # table so that page 1 and page 4 are modified. Using a
  # block-size of 2048 and page-size of 1024, this means
  # pages 2 and 3 must also be saved in the journal to avoid
  # risking corruption.
  #
  # The loop is so that this test can be run with a couple
  # of different seeds for the random number generator.
  #
  do_test crash2-1.2.$ii {
    crashsql -file test.db -blocksize 2048 [subst {
      [string repeat {SELECT random();} $ii]
      CREATE TABLE hij(h, i, j);

    }]
    sqlite3 db test.db
    db eval {PRAGMA integrity_check}
  } {ok}
}

proc signature {} {
  return [db eval {SELECT count(*), md5sum(a), md5sum(b), md5sum(c) FROM abc}]
}

  }
  execsql COMMIT
  expr ([file size test.db] / 1024) > 450
} {1}
for {set i 1} {$i < 30} {incr i} {
  set sig [signature]
  set sector [expr 1024 * 1<<($i%4)]
  db close
  do_test crash2-2.$i.1 {
     crashsql -blocksize $sector -delay [expr $i%5 + 1] -file test.db-journal "
       BEGIN;
       SELECT random() FROM abc LIMIT $i;
       INSERT INTO abc SELECT randstr(10,10), 0, 0 FROM abc WHERE random()%2==0;
       DELETE FROM abc WHERE random()%2!=0;
       COMMIT;
     "
  } {1 {child process exited abnormally}}
  do_test crash2-2.$i.2 {
    sqlite3 db test.db
    signature
  } $sig
} 


# Test case for crashing during database sync with simulated
# sector-size values from 1024 to 8192.
#
for {set i 1} {$i < 10} {incr i} {
  set sig [signature]
  set sector [expr 1024 * 1<<($i%4)]
  db close
  do_test crash2-3.$i.1 {
     crashsql -blocksize $sector -file test.db "
       BEGIN;
       SELECT random() FROM abc LIMIT $i;
       INSERT INTO abc SELECT randstr(10,10), 0, 0 FROM abc WHERE random()%2==0;
       DELETE FROM abc WHERE random()%2!=0;
       COMMIT;
     "
  } {1 {child process exited abnormally}}
  do_test crash2-3.$i.2 {
    sqlite3 db test.db
    signature
  } $sig
} 

finish_test

#
#***********************************************************************
#
# This file contains tests that verify that SQLite can correctly rollback
# databases after crashes when using the special IO modes triggered 
# by device IOCAP flags.
#
# $Id: crash3.test,v 1.3 2007/08/24 11:52:29 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !crashtest {
  finish_test
  return

  for {set ii 0} {$ii < 10} {incr ii} {

    db close
    file delete -force test.db test.db-journal
    sqlite3 db test.db
    do_test crash3-1.$tn.1 {
      execsql {
        PRAGMA page_size = 1024;
        BEGIN;
        CREATE TABLE abc(a, b, c);
        INSERT INTO abc VALUES(1, 2, 3);
        COMMIT;
      }
    } {}
    db close

#
#***********************************************************************
#
# The focus of this file is testing some specific characteristics of the 
# IO traffic generated by SQLite (making sure SQLite is not writing out
# more database pages than it has to, stuff like that).
#
# $Id: io.test,v 1.6 2007/08/24 11:52:29 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Test summary:
#
# io-1.* -  Test that quick-balance does not journal pages unnecessarily.
#
# io-2.* -  Test the "atomic-write optimization".
#
# io-3.* -  Test the IO traffic enhancements triggered when the 
#           IOCAP_SEQUENTIAL device capability flag is set (no 
#           fsync() calls on the journal file).
#
# io-4.* -  Test the IO traffic enhancements triggered when the 
#           IOCAP_SAFE_APPEND device capability flag is set (fewer 
#           fsync() calls on the journal file, no need to set nRec
#           field in the single journal header).
#
# io-5.* -  Test that the default page size is selected and used 
#           correctly.
#           

set ::nWrite 0
proc nWrite {db} {
  set bt [btree_from_db $db]
  db_enter $db
  array set stats [btree_pager_stats $bt]

# is created during execution of the COMMIT statement, so we have to
# use the same technique to check that it is created as in the above 
# block.
file delete -force test2.db test2.db-journal
do_test io-2.7.1 {
  execsql {
    ATTACH 'test2.db' AS aux;
    PRAGMA aux.page_size = 1024;
    CREATE TABLE aux.abc2(a, b);
    BEGIN;
    INSERT INTO abc VALUES(9, 10);
  }
  file exists test.db-journal
} {0}
do_test io-2.7.2 {

  #    <jrnl file size> = <jrnl header size> + nPage * (<page-size> + 8)
  #
  # If the journal file contains additional headers, this formula
  # will not predict the size of the journal file.
  #
  file size test.db-journal
} [expr 1024 + (1024+8)*41]

#----------------------------------------------------------------------
# Test cases io-5.* test that the default page size is selected and
# used correctly.
#
set tn 0
foreach {char                 sectorsize pgsize} {
         {}                     512      1024
         {}                    1024      1024
         {}                    2048      2048
         {}                    8192      8192
         {}                   16384      8192
         {atomic}               512      8192
         {atomic512}            512      1024
         {atomic2K}             512      2048
         {atomic2K}            4096      4096
         {atomic2K atomic}      512      8192
         {atomic64K}            512      1024
} {
  incr tn
  db close
  file delete -force test.db test.db-journal
  sqlite3_simulate_device -char $char -sectorsize $sectorsize
  sqlite3 db test.db
  do_test io-5.$tn {
    execsql {
      CREATE TABLE abc(a, b, c);
    }
    expr {[file size test.db]/2}
  } $pgsize
}

sqlite3_simulate_device -char {} -sectorsize 0
finish_test