SQLite

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.226 2005/12/19 14:18:11 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

  pPg = pPager->pFirstSynced;

  /* If we could not find a page that does not require an fsync()
  ** on the journal file then fsync the journal file.  This is a
  ** very slow operation, so we work hard to avoid it.  But sometimes
  ** it can't be helped.
  */
  if( pPg==0 && pPager->pFirst && syncOk ){
    int rc = syncJournal(pPager);
    if( rc!=0 ){
      sqlite3pager_rollback(pPager);
      return SQLITE_IOERR;
    }
    if( pPager->fullSync ){
      /* If in full-sync mode, write a new journal header into the

/*
** This function is called to free superfluous dynamically allocated memory
** held by the pager system. Memory in use by any SQLite pager allocated
** by the current thread may be sqliteFree()ed.
**
** nReq is the number of bytes of memory required. Once this much has
** been released, the function returns. A negative value for nReq means
** free as much memory as possible. The return value is the total number 
** of bytes of memory released.
*/
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
int sqlite3pager_release_memory(int nReq){
  SqliteTsd *pTsd = sqlite3Tsd();
  Pager *p;
  int nReleased = 0;
  int i;

  /* Outermost loop runs for at most two iterations. First iteration we
  ** try to find memory that can be released without calling fsync(). Second
  ** iteration (which only runs if the first failed to free nReq bytes of
  ** memory) is permitted to call fsync(). This is of course much more 
  ** expensive.
  */
  for(i=0; i==0 || i==1; i++){

    /* Loop through all the SQLite pagers opened by the current thread. */
    for(p=pTsd->pPager; p && (nReq<0 || nReleased<nReq); p=p->pNext){
      PgHdr *pPg;
      int rc;

      /* For each pager, try to free as many pages as possible (without 
      ** calling fsync() if this is the first iteration of the outermost 
      ** loop).
      */
      while( SQLITE_OK==(rc = pager_recycle(p, i, &pPg)) && pPg) {
	/* We've found a page to free. At this point the page has been 
        ** removed from the page hash-table, free-list and synced-list 
	** (pFirstSynced). It is still in the all pages (pAll) list. 
        ** Remove it from this list before freeing.
        **
        ** Todo: Check the Pager.pStmt list to make sure this is Ok. It 
        ** probably is though.
        */
        PgHdr *pTmp;
        assert( pPg );
        if( pPg==p->pAll ){
           p->pAll = pPg->pNextAll;
        }else{
          for( pTmp=p->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll );
          pTmp->pNextAll = pPg->pNextAll;
        }
        nReleased += sqliteAllocSize(pPg);
        sqliteFree(pPg);
      }

      if( rc!=SQLITE_OK ){

        assert(0);
      }
    }
  }
  
  return nReleased;
}
#endif

/*
** Acquire a page.
**
** A read lock on the disk file is obtained when the first page is acquired. 
** This read lock is dropped when the last page is released.
**

**    May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem.  The page cache subsystem reads and writes a file a page
** at a time and provides a journal for rollback.
**
** @(#) $Id: pager.h,v 1.48 2005/12/19 14:18:11 danielk1977 Exp $
*/

#ifndef _PAGER_H_
#define _PAGER_H_

/*
** The default size of a database page.

const char *sqlite3pager_dirname(Pager*);
const char *sqlite3pager_journalname(Pager*);
int sqlite3pager_nosync(Pager*);
int sqlite3pager_rename(Pager*, const char *zNewName);
void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*);
int sqlite3pager_movepage(Pager*,void*,Pgno);
int sqlite3pager_reset(Pager*);
int sqlite3pager_release_memory(int);

#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
int sqlite3pager_lockstate(Pager*);
#endif

#ifdef SQLITE_TEST
void sqlite3pager_refdump(Pager*);
int pager3_refinfo_enable;
#endif

#endif /* _PAGER_H_ */

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the SQLite library
** presents to client programs.
**
** @(#) $Id: sqlite.h.in,v 1.150 2005/12/19 14:18:11 danielk1977 Exp $
*/
#ifndef _SQLITE3_H_
#define _SQLITE3_H_
#include <stdarg.h>     /* Needed for the definition of va_list */

/*
** Make sure we can call this stuff from C++.

** For the purposes of this API, a transaction is said to have been 
** rolled back if an explicit "ROLLBACK" statement is executed, or
** an error or constraint causes an implicit rollback to occur. The 
** callback is not invoked if a transaction is automatically rolled
** back because the database connection is closed.
*/
void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);

int sqlite3_release_memory(int);

/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.
*/
#ifdef SQLITE_OMIT_FLOATING_POINT
# undef double
#endif

#ifdef __cplusplus
}  /* End of the 'extern "C"' block */
#endif
#endif

/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Internal interface definitions for SQLite.
**
** @(#) $Id: sqliteInt.h,v 1.441 2005/12/19 14:18:11 danielk1977 Exp $
*/
#ifndef _SQLITEINT_H_
#define _SQLITEINT_H_

/*
** Many people are failing to set -DNDEBUG=1 when compiling SQLite.
** Setting NDEBUG makes the code smaller and run faster.  So the following

/*
** An instance of this structure is allocated for each thread that uses SQLite.
*/
typedef struct SqliteTsd SqliteTsd;
struct SqliteTsd {
  int isInit;                     /* True if structure has been initialised */
  int mallocFailed;               /* True after a malloc() has failed */

#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
  unsigned int nSoftHeapLimit;    /* (uint)-1 for unlimited */
  unsigned int nAlloc;            /* Number of bytes currently allocated */
  Pager *pPager;                  /* Linked list of all pagers in this thread */
#endif

#ifndef NDEBUG

/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** A TCL Interface to SQLite
**
** $Id: tclsqlite.c,v 1.141 2005/12/19 14:18:11 danielk1977 Exp $
*/
#ifndef NO_TCL     /* Omit this whole file if TCL is unavailable */

#include "sqliteInt.h"
#include "hash.h"
#include "tcl.h"
#include <stdlib.h>

    "authorizer",         "busy",              "cache",
    "changes",            "close",             "collate",
    "collation_needed",   "commit_hook",       "complete",
    "copy",               "errorcode",         "eval",
    "exists",             "function",          "last_insert_rowid",
    "nullvalue",          "onecolumn",         "profile",
    "progress",           "rekey",             "rollback_hook",
    "release_memory",     "soft_heap_limit",   "timeout",
    "total_changes",      "trace",             "transaction",
    "update_hook",        "version",           0                    
  };
  enum DB_enum {
    DB_AUTHORIZER,        DB_BUSY,             DB_CACHE,
    DB_CHANGES,           DB_CLOSE,            DB_COLLATE,
    DB_COLLATION_NEEDED,  DB_COMMIT_HOOK,      DB_COMPLETE,
    DB_COPY,              DB_ERRORCODE,        DB_EVAL,
    DB_EXISTS,            DB_FUNCTION,         DB_LAST_INSERT_ROWID,
    DB_NULLVALUE,         DB_ONECOLUMN,        DB_PROFILE,
    DB_PROGRESS,          DB_REKEY,            DB_ROLLBACK_HOOK,
    DB_RELEASE_MEMORY,    DB_SOFT_HEAP_LIMIT,  DB_TIMEOUT,
    DB_TOTAL_CHANGES,     DB_TRACE,            DB_TRANSACTION,
    DB_UPDATE_HOOK,       DB_VERSION
  };
  /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
    return TCL_ERROR;
  }

    break;
  }

  /*
  **     $db soft_heap_limit N
  **
  ** Set the soft-heap-limit for this thread. Note that the limit is 
  ** per-thread, not per-database. An empty string is returned.
  */
  case DB_SOFT_HEAP_LIMIT: {
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
    int n;
    if( objc!=3 ){
      Tcl_WrongNumArgs(interp, 2, objv, "BYTES");
      return TCL_ERROR;
    }
    if( Tcl_GetIntFromObj(interp, objv[2], &n) ){
      return TCL_ERROR;
    }
    sqlite3_soft_heap_limit(n);
    Tcl_ResetResult(interp);
#endif
    break;
  }

  /*
  **     $db release_memory ?N?
  **
  ** Try to release memory currently held (but not really required) by 
  ** SQLite database connections opened by the current thread. If an
  ** integer argument is supplied, then SQLite stops trying to free memory
  ** after N bytes have been freed.
  **
  ** The value returned is the number of bytes actually freed.
  **/
  case DB_RELEASE_MEMORY: {
    int nRelease = 0;
    int N = -1;
    if( objc!=2 && objc!=3 ){
      Tcl_WrongNumArgs(interp, 2, objv, "?N?");
      return TCL_ERROR;
    }
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
    if( objc==3 && TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){
      return TCL_ERROR;
    }
    nRelease = sqlite3_release_memory(N);
#endif
    Tcl_SetObjResult(interp, Tcl_NewIntObj(nRelease));
    break;
  }
  
  /*
  **     $db total_changes
  **
  ** Return the number of rows that were modified, inserted, or deleted 

**
*************************************************************************
** Utility functions used throughout sqlite.
**
** This file contains functions for allocating memory, comparing
** strings, and stuff like that.
**
** $Id: util.c,v 1.156 2005/12/19 14:18:11 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <stdarg.h>
#include <ctype.h>

/*
** MALLOC WRAPPER ARCHITECTURE
**
** The sqlite code accesses dynamic memory allocation/deallocation by invoking
** the following four APIs (which may be implemented as macros).
**
**     sqlite3Malloc()
**     sqlite3MallocRaw()
**     sqlite3Realloc()
**     sqlite3ReallocOrFree()
**     sqlite3Free()
**     sqlite3AllocSize()
**
** The function sqlite3FreeX performs the same task as sqlite3Free and is
** guaranteed to be a real function. The same holds for sqlite3MallocX
**
** The above APIs are implemented in terms of the functions provided at the Os
** level (not in this file). The Os level interface is never accessed directly
** by code outside of this file.
**
**     sqlite3Os.xMalloc()
**     sqlite3Os.xRealloc()
**     sqlite3Os.xFree()
**     sqlite3Os.xAllocationSize()
**
** Functions sqlite3MallocRaw() and sqlite3Realloc() may invoke 
** sqlite3_release_memory() if a call to sqlite3Os.xMalloc() or
** sqlite3Os.xRealloc() fails (or if the soft-heap-limit for the thread is
** exceeded). Function sqlite3Malloc() usually invokes
** sqlite3MallocRaw().
**
** MALLOC TEST WRAPPER ARCHITECTURE
**
** The test wrapper provides extra test facilities to ensure the library 
** does not leak memory and handles the failure of the underlying (Os level)
** allocation system correctly. It is only present if the library is 
** compiled with the SQLITE_MEMDEBUG macro set.
**
**     * Guardposts to detect overwrites.
**     * Ability to cause a specific Malloc() or Realloc() to fail.
**     * Audit outstanding memory allocations (i.e check for leaks).
*/






#ifdef SQLITE_MEMDEBUG
/*--------------------------------------------------------------------------
** Begin code for memory allocation system test layer.
**
** Memory debugging is turned on by defining the SQLITE_MEMDEBUG macro.
**
** SQLITE_MEMDEBUG==1    -> Fence-posting only (thread safe) 

  sizeof(void *)*2 +                   /* pPrev and pNext pointers */   \
  TESTALLOC_NGUARD*sizeof(u32)*2 +              /* Guard words */       \
  sizeof(u32) + TESTALLOC_FILESIZE +   /* File and line number */       \
  TESTALLOC_USERSIZE +                 /* User string */                \
  TESTALLOC_STACKSIZE                  /* backtrace() stack */          \
)


#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
/*
** Set the soft heap-size limit for the current thread.
*/
void sqlite3_soft_heap_limit(int n){
  unsigned int N;
  if( n<0 ){
    /* No limit */
    N = 0xFFFFFFFF;
  }else{
    N = n;
  }
  sqlite3Tsd()->nSoftHeapLimit = N;
}

/*
** Release memory held by SQLite instances created by the current thread.
*/
int sqlite3_release_memory(int n){
  return sqlite3pager_release_memory(n);
}
#else
/* If SQLITE_OMIT_MEMORY_MANAGEMENT is defined, then define a version
** of sqlite3_release_memory() to be used by other code in this file.
** This is done for no better reason than to reduce the number of 
** pre-processor #ifndef statements.
*/
#define sqlite3_release_memory(x) 0    /* 0 == no memory freed */
#endif

/*
** For keeping track of the number of mallocs and frees.   This
** is used to check for memory leaks.  The iMallocFail and iMallocReset
** values are used to simulate malloc() failures during testing in 
** order to verify that the library correctly handles an out-of-memory
** condition.
*/

void OSMALLOC_FAILED(){
  sqlite3Tsd()->isFail = 0;
}

int OSSIZEOF(void *p){
  if( p ){
    u32 *pOs = (u32 *)getOsPointer(p);
    return sqlite3Os.xAllocationSize(pOs) - TESTALLOC_OVERHEAD;
  }
  return 0;
}

#else
/* Define macros to call the sqlite3Os.xXXX interface directly if 
** the SQLITE_MEMDEBUG macro is not defined.

** Clear the "mallocFailed" flag. This should be invoked before exiting any
** entry points that may have called sqliteMalloc().
*/
void sqlite3MallocClearFailed(){
  sqlite3Tsd()->mallocFailed = 0;
}

















#ifndef NDEBUG
/*
** This function sets a flag in the thread-specific-data structure that will
** cause an assert to fail if sqliteMalloc() or sqliteRealloc() is called.
*/
void sqlite3MallocDisallow(){

# 2005 November 30
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: malloc5.test,v 1.1 2005/12/19 14:18:12 danielk1977 Exp $

#---------------------------------------------------------------------------
# NOTES ON EXPECTED BEHAVIOUR
#
#---------------------------------------------------------------------------

set testdir [file dirname $argv0]
source $testdir/tester.tcl

do_test malloc5-1.1 {
  # Simplest possible test. Call [db release_memory] when there is exactly
  # one unused page in a single pager cache. This test case set's the 
  # value of the ::pgalloc variable, which is used in subsequent tests.
  #
  # Note: Even though executing this statement on an empty database 
  # modifies 2 pages (the root of sqlite_master and the new root page), 
  # the sqlite_master root (page 1) is never freed because the btree layer
  # retains a reference to it for the entire transaction. 
  execsql {
    BEGIN;
    CREATE TABLE abc(a, b, c);
  }
  set ::pgalloc [db release_memory]
  expr $::pgalloc > 0
} {1}
do_test malloc5-1.2 {
  # Test that the transaction started in the above test is still active.
  # Because the page freed had been written to, freeing it required a
  # journal sync and exclusive lock on the database file. Test the file
  # appears to be locked.
  sqlite3 db2 test.db
  catchsql {
    SELECT * FROM abc;
  } db2
} {1 {database is locked}}
do_test malloc5-1.3 {
  # Again call [db release_memory] when there is exactly one unused page 
  # in the cache. The same amount of memory is required, but no journal-sync
  # or exclusive lock should be established.
  execsql {
    COMMIT;
    BEGIN;
    SELECT * FROM abc;
  }
  db release_memory
} $::pgalloc
do_test malloc5-1.4 {
  # Database should not be locked this time.
  catchsql {
    SELECT * FROM abc;
  } db2
} {0 {}}
do_test malloc5-1.5 {
  # Manipulate the cache so that it contains two unused pages. One requires 
  # a journal-sync to free, the other does not.
  execsql {
    SELECT * FROM abc;
    CREATE TABLE def(d, e, f);
  }
  db release_memory 500
} $::pgalloc
do_test malloc5-1.6 {
  # Database should not be locked this time. The above test case only
  # requested 500 bytes of memory, which can be obtained by freeing the page
  # that does not require an fsync().
  catchsql {
    SELECT * FROM abc;
  } db2
} {0 {}}
do_test malloc5-1.7 {
  # Release another 500 bytes of memory. This time we require a sync(), 
  # so the database file will be locked afterwards.
  db release_memory 500
} $::pgalloc
do_test malloc5-1.8 {
  catchsql {
    SELECT * FROM abc;
  } db2
} {1 {database is locked}}
do_test malloc5-1.9 {
  execsql {
    COMMIT;
  }
} {}


do_test malloc5-2.1 {
  # Put some data in tables abc and def. Both tables are still wholly 
  # contained within their root pages.
  execsql {
    INSERT INTO abc VALUES(1, 2, 3);
    INSERT INTO abc VALUES(4, 5, 6);
    INSERT INTO def VALUES(7, 8, 9);
    INSERT INTO def VALUES(10,11,12);
  }
} {}
do_test malloc5-2.2 {
  # Load the root-page for table def into the cache. Then query table abc. 
  # Halfway through the query call sqlite3_release_memory(). The goal of this
  # test is to make sure we don't free pages that are in use (specifically, 
  # the root of table abc).
  set nRelease 0
  execsql { 
    BEGIN;
    SELECT * FROM def;
  }
  db eval {SELECT * FROM abc} {
    incr nRelease [db release_memory]
    lappend data $a $b $c
  }
  list $nRelease $data
} [list $pgalloc [list 1 2 3 4 5 6]]

finish_test