SQLite

** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.228 2005/12/20 09:19:37 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

    sqlite3Os.xDelete(pPager->zJournal);
    return 0;
  }else{
    return 1;
  }
}

/*
** Try to find a page in the cache that can be recycled.
*/
static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){
  PgHdr *pPg;
  *ppPg = 0;

  /* Find a page to recycle.  Try to locate a page that does not
  ** require us to do an fsync() on the journal.
  */
  pPg = pPager->pFirstSynced;

  /* If we could not find a page that does not require an fsync()
  ** on the journal file then fsync the journal file.  This is a
  ** very slow operation, so we work hard to avoid it.  But sometimes
  ** it can't be helped.
  */
  if( pPg==0 && pPager->pFirst && syncOk && !MEMDB){
    int rc = syncJournal(pPager);
    if( rc!=0 ){
      sqlite3pager_rollback(pPager);
      return SQLITE_IOERR;
    }
    if( pPager->fullSync ){
      /* If in full-sync mode, write a new journal header into the

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the SQLite library
** presents to client programs.
**
** @(#) $Id: sqlite.h.in,v 1.151 2005/12/20 09:19:37 danielk1977 Exp $
*/
#ifndef _SQLITE3_H_
#define _SQLITE3_H_
#include <stdarg.h>     /* Needed for the definition of va_list */

/*
** Make sure we can call this stuff from C++.

** The limit is called "soft", because if sqlite3_release_memory() cannot free
** sufficient memory to prevent the limit from being exceeded, the memory is
** allocated anyway and the current operation proceeds.
**
** This function is only available if the library was compiled without the 
** SQLITE_OMIT_MEMORY_MANAGEMENT option set.
*/
void sqlite3_soft_heap_limit(sqlite_int64);

/*
** Register a callback function with the database connection identified by the 
** first argument to be invoked whenever a row is updated, inserted or deleted.
** Any callback set by a previous call to this function for the same 
** database connection is overridden.
**

/*
** 2001 September 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** Internal interface definitions for SQLite.
**
** @(#) $Id: sqliteInt.h,v 1.442 2005/12/20 09:19:37 danielk1977 Exp $
*/
#ifndef _SQLITEINT_H_
#define _SQLITEINT_H_

/*
** Many people are failing to set -DNDEBUG=1 when compiling SQLite.
** Setting NDEBUG makes the code smaller and run faster.  So the following

#define sqliteFree(x)          sqlite3FreeX(x)
#define sqliteAllocSize(x)     sqlite3AllocSize(x)

/*
** An instance of this structure is allocated for each thread that uses SQLite.
*/

struct SqliteTsd {
  u8 isInit;               /* True if structure has been initialised */
  u8 mallocFailed;         /* True after a malloc() has failed */

#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
  i64 nSoftHeapLimit;      /* -ve for unlimited */
  i64 nAlloc;              /* Number of bytes currently allocated */
  Pager *pPager;           /* Linked list of all pagers in this thread */
#endif




#ifdef SQLITE_MEMDEBUG
  i64 nMaxAlloc;           /* High water mark of SqliteTsd.nAlloc */
  int mallocAllowed;       /* assert() in sqlite3Malloc() if not set */
  int isFail;              /* True if all malloc() calls should fail */
  const char *zFile;       /* Filename to associate debugging info with */
  int iLine;               /* Line number to associate debugging info with */
  void *pFirst;            /* Pointer to linked list of allocations */
#endif
};

typedef struct Index Index;
typedef struct KeyClass KeyClass;
typedef struct KeyInfo KeyInfo;
typedef struct NameContext NameContext;
typedef struct Parse Parse;
typedef struct Select Select;
typedef struct SrcList SrcList;
typedef struct SqliteTsd SqliteTsd;
typedef struct Table Table;
typedef struct Token Token;
typedef struct TriggerStack TriggerStack;
typedef struct TriggerStep TriggerStep;
typedef struct Trigger Trigger;
typedef struct WhereInfo WhereInfo;
typedef struct WhereLevel WhereLevel;

**    May you share freely, never taking more than you give.
**
*************************************************************************
** Code for testing the printf() interface to SQLite.  This code
** is not included in the SQLite library.  It is used for automated
** testing of the SQLite library.
**
** $Id: test1.c,v 1.177 2005/12/20 09:19:37 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "tcl.h"
#include "os.h"
#include <stdlib.h>
#include <string.h>

static int sqlite_malloc_stat(
  void *NotUsed,
  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
  int argc,              /* Number of arguments */
  char **argv            /* Text of each argument */
){
  char zBuf[200];
  sprintf(zBuf, "%d %d %d", sqlite3_nMalloc,sqlite3_nFree,sqlite3_iMallocFail);
  Tcl_AppendResult(interp, zBuf, 0);
  return TCL_OK;
}

/*
** This function implements a Tcl command that may be invoked using any of
** the four forms enumerated below.
**
** sqlite_malloc_outstanding
**     Return a summary of all unfreed blocks of memory allocated by the
**     current thread. See comments above function sqlite3OutstandingMallocs() 
**     in util.c for a description of the returned value.
**
** sqlite_malloc_outstanding -bytes
**     Return the total amount of unfreed memory (in bytes) allocated by 
**     this thread.
**
** sqlite_malloc_outstanding -maxbytes
**     Return the maximum amount of dynamic memory in use at one time 
**     by this thread.
**
** sqlite_malloc_outstanding -clearmaxbytes
**     Set the value returned by [sqlite_malloc_outstanding -maxbytes]
**     to the current value of [sqlite_malloc_outstanding -bytes]. 
*/
static int sqlite_malloc_outstanding(

  ClientData clientData,
  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
  int objc,              /* Number of arguments */
  Tcl_Obj *CONST objv[]  /* Command arguments */
){
  extern int sqlite3OutstandingMallocs(Tcl_Interp *interp);

  if( objc!=1 && objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "?-bytes?");
    return TCL_ERROR;
  }

  if( objc==2 ){
    const char *zArg = Tcl_GetString(objv[1]);
    if( 0==strcmp(zArg, "-bytes") ){
      Tcl_SetObjResult(interp, Tcl_NewIntObj(sqlite3Tsd()->nAlloc));
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
    }else if( 0==strcmp(zArg, "-maxbytes") ){
      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(sqlite3Tsd()->nMaxAlloc));
    }else if( 0==strcmp(zArg, "-clearmaxbytes") ){
      sqlite3Tsd()->nMaxAlloc = sqlite3Tsd()->nAlloc;
#endif
    }else{
      Tcl_AppendResult(interp, "bad option \"", zArg, 
        "\": must be -bytes, -maxbytes or -clearmaxbytes", 0
      );
      return TCL_ERROR;
    }

    return TCL_OK;
  }

  return sqlite3OutstandingMallocs(interp);
}
#endif

/*
** Usage:  sqlite_abort
**

     { "sqlite3_create_function",       (Tcl_CmdProc*)test_create_function  },
     { "sqlite3_create_aggregate",      (Tcl_CmdProc*)test_create_aggregate },
     { "sqlite_register_test_function", (Tcl_CmdProc*)test_register_func    },
     { "sqlite_abort",                  (Tcl_CmdProc*)sqlite_abort          },
#ifdef SQLITE_MEMDEBUG
     { "sqlite_malloc_fail",            (Tcl_CmdProc*)sqlite_malloc_fail    },
     { "sqlite_malloc_stat",            (Tcl_CmdProc*)sqlite_malloc_stat    },

#endif
     { "sqlite_bind",                   (Tcl_CmdProc*)test_bind             },
     { "breakpoint",                    (Tcl_CmdProc*)test_breakpoint       },
     { "sqlite3_key",                   (Tcl_CmdProc*)test_key              },
     { "sqlite3_rekey",                 (Tcl_CmdProc*)test_rekey            },
     { "sqlite_set_magic",              (Tcl_CmdProc*)sqlite_set_magic      },
     { "sqlite3_interrupt",             (Tcl_CmdProc*)test_interrupt        },

     { "sqlite3OsUnlock",         test_sqlite3OsUnlock, 0    },
#endif
#ifndef SQLITE_OMIT_UTF16
     { "add_test_collate",        test_collate, 0            },
     { "add_test_collate_needed", test_collate_needed, 0     },
     { "add_test_function",       test_function, 0           },
#endif
#ifdef SQLITE_MEMDEBUG
     { "sqlite_malloc_outstanding", sqlite_malloc_outstanding, 0},
#endif
     { "sqlite3_test_errstr",     test_errstr, 0             },
     { "tcl_variable_type",       tcl_variable_type, 0       },
  };
  static int bitmask_size = sizeof(Bitmask)*8;
  int i;
  extern int sqlite3_os_trace;
  extern int sqlite3_where_trace;

**
*************************************************************************
** Utility functions used throughout sqlite.
**
** This file contains functions for allocating memory, comparing
** strings, and stuff like that.
**
** $Id: util.c,v 1.157 2005/12/20 09:19:37 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <stdarg.h>
#include <ctype.h>

/*

** allocation system correctly. It is only present if the library is 
** compiled with the SQLITE_MEMDEBUG macro set.
**
**     * Guardposts to detect overwrites.
**     * Ability to cause a specific Malloc() or Realloc() to fail.
**     * Audit outstanding memory allocations (i.e check for leaks).
*/

#define MAX(x,y) ((x)>(y)?(x):(y))

#ifdef SQLITE_MEMDEBUG
/*--------------------------------------------------------------------------
** Begin code for memory allocation system test layer.
**
** Memory debugging is turned on by defining the SQLITE_MEMDEBUG macro.
**

  TESTALLOC_USERSIZE +                 /* User string */                \
  TESTALLOC_STACKSIZE                  /* backtrace() stack */          \
)


#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
/*
** Set the soft heap-size limit for the current thread. Passing a negative
** value indicates no limit.
*/
void sqlite3_soft_heap_limit(sqlite_int64 n){







  sqlite3Tsd()->nSoftHeapLimit = n;
}

/*
** Release memory held by SQLite instances created by the current thread.
*/
int sqlite3_release_memory(int n){
  return sqlite3pager_release_memory(n);

}
#endif

/*
** This is the test layer's wrapper around sqlite3Os.xMalloc().
*/
static void * OSMALLOC(int n){
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
  SqliteTsd *pTsd = sqlite3Tsd();
  pTsd->nMaxAlloc = MAX(pTsd->nMaxAlloc, pTsd->nAlloc);
#endif
  if( !failMalloc() ){
    u32 *p;
    p = (u32 *)sqlite3Os.xMalloc(n + TESTALLOC_OVERHEAD);
    assert(p);
    sqlite3_nMalloc++;
    applyGuards(p);
    linkAlloc(p);

  sqlite3_nFree++;
}

/*
** This is the test layer's wrapper around sqlite3Os.xRealloc().
*/
void * OSREALLOC(void *pRealloc, int n){
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
  SqliteTsd *pTsd = sqlite3Tsd();
  pTsd->nMaxAlloc = MAX(pTsd->nMaxAlloc, pTsd->nAlloc);
#endif
  if( !failMalloc() ){
    u32 *p = (u32 *)getOsPointer(pRealloc);
    checkGuards(p);
    p = sqlite3Os.xRealloc(p, n + TESTALLOC_OVERHEAD);
    applyGuards(p);
    relinkAlloc(p);
    return (void *)(&p[TESTALLOC_NGUARD + 2*sizeof(void *)/sizeof(u32)]);

** successful is returned to the caller.
**
** If SQLITE_OMIT_MEMORY_MANAGEMENT is defined, this function is a no-op.
*/
#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
static void handleSoftLimit(int n){
  SqliteTsd *pTsd = sqlite3Tsd();
  pTsd->nAlloc += (i64)n;
  if( n>0 && pTsd->nSoftHeapLimit>0 ){
    while( pTsd->nAlloc>pTsd->nSoftHeapLimit && sqlite3_release_memory(n) );
  }
}
#else
#define handleSoftLimit()
#endif

/*
** Allocate and return N bytes of uninitialised memory by calling
** sqlite3Os.xMalloc(). If the Malloc() call fails, attempt to free memory 
** by calling sqlite3_release_memory().
*/
void *sqlite3MallocRaw(int n){
  SqliteTsd *pTsd = sqlite3Tsd();
  void *p = 0;
  if( n>0 && !pTsd->mallocFailed ){
    handleSoftLimit(n);
    while( !(p = OSMALLOC(n)) && sqlite3_release_memory(n) );
    if( !p ){
      /* If the allocation failed, call handleSoftLimit() again, this time
      ** with the additive inverse of the argument passed to 
      ** handleSoftLimit() above. This is so the SqliteTsd.nAlloc variable is
      ** still correct after a malloc() failure. 
      */
      handleSoftLimit(n * -1);
      sqlite3Tsd()->mallocFailed = 1;
      OSMALLOC_FAILED();
    }
  }
  return p;
}

  if( !p ){
    return sqlite3Malloc(n);
  }else{
    void *np = 0;
    handleSoftLimit(n - OSSIZEOF(p));
    while( !(np = OSREALLOC(p, n)) && sqlite3_release_memory(n) );
    if( !np ){
      /* If the allocation failed, call handleSoftLimit() again, this time
      ** with the additive inverse of the argument passed to 
      ** handleSoftLimit() above. This is so the SqliteTsd.nAlloc variable is
      ** still correct after a malloc() failure. 
      */
      handleSoftLimit(OSSIZEOF(p) - n);
      pTsd->mallocFailed = 1;
      OSMALLOC_FAILED();
    }
    return np;
  }
}

/*
** Free the memory pointed to by p. p must be either a NULL pointer or a 
** value returned by a previous call to sqlite3Malloc() or sqlite3Realloc().
*/
void sqlite3FreeX(void *p){
  handleSoftLimit(0 - OSSIZEOF(p));
  if( p ){
    OSFREE(p);
  }
}

/*
** A version of sqliteMalloc() that is always a function, not a macro.

/*
** Return a pointer to the SqliteTsd associated with the calling thread.
*/
SqliteTsd *sqlite3Tsd(){
  SqliteTsd *pTsd = sqlite3Os.xThreadSpecificData(sizeof(SqliteTsd));
  if( pTsd && !pTsd->isInit ){
    pTsd->nSoftHeapLimit = -1;
#ifndef NDEBUG
    pTsd->mallocAllowed = 1;
#endif
    pTsd->isInit = 1;
  }
  return pTsd;
}

# 2005 November 30
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: malloc5.test,v 1.2 2005/12/20 09:19:38 danielk1977 Exp $

#---------------------------------------------------------------------------
# NOTES ON EXPECTED BEHAVIOUR
#
#---------------------------------------------------------------------------

set testdir [file dirname $argv0]

} {1 {database is locked}}
do_test malloc5-1.9 {
  execsql {
    COMMIT;
  }
} {}


do_test malloc5-2.1 {
  # Put some data in tables abc and def. Both tables are still wholly 
  # contained within their root pages.
  execsql {
    INSERT INTO abc VALUES(1, 2, 3);
    INSERT INTO abc VALUES(4, 5, 6);
    INSERT INTO def VALUES(7, 8, 9);
    INSERT INTO def VALUES(10,11,12);
  }
} {}
do_test malloc5-2.2 {
  # Load the root-page for table def into the cache. Then query table abc. 
  # Halfway through the query call sqlite3_release_memory(). The goal of this
  # test is to make sure we don't free pages that are in use (specifically, 
  # the root of table abc).
  set nRelease 0
  execsql { 
    BEGIN;
    SELECT * FROM def;
  }
  set data [list]
  db eval {SELECT * FROM abc} {
    incr nRelease [db release_memory]
    lappend data $a $b $c
  }
  execsql {
    COMMIT;
  }
  list $nRelease $data
} [list $pgalloc [list 1 2 3 4 5 6]]

do_test malloc5-3.1 {
  # Simple test to show that if two pagers are opened from within this
  # thread, memory is freed from both when sqlite3_release_memory() is
  # called.
  execsql {
    BEGIN;
    SELECT * FROM abc;
  }
  execsql {
    SELECT * FROM sqlite_master;
    BEGIN;
    SELECT * FROM def;
  } db2
  db release_memory
} [expr $::pgalloc * 2]
do_test malloc5-3.2 {
  concat \
    [execsql {SELECT * FROM abc; COMMIT}] \
    [execsql {SELECT * FROM def; COMMIT} db2]
} {1 2 3 4 5 6 7 8 9 10 11 12}

db2 close
sqlite_malloc_outstanding -clearmaxbytes

# The following two test cases each execute a transaction in which 
# 10000 rows are inserted into table abc. The first test case is used
# to ensure that more than 1MB of dynamic memory is used to perform
# the transaction. 
#
# The second test case sets the "soft-heap-limit" to 100,000 bytes (0.1 MB)
# and tests to see that this limit is not exceeded at any point during 
# transaction execution.
#
do_test malloc5-4.1 {
  execsql {BEGIN;}
  execsql {DELETE FROM abc;}
  for {set i 0} {$i < 10000} {incr i} {
    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
  }
  execsql {COMMIT;}
  set ::nMaxBytes [sqlite_malloc_outstanding -maxbytes]
  expr $::nMaxBytes > 1000000
} {1}
do_test malloc5-4.2 {
  db release_memory
  sqlite_malloc_outstanding -clearmaxbytes
  db soft_heap_limit 100000
  execsql {BEGIN;}
  for {set i 0} {$i < 10000} {incr i} {
    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
  }
  execsql {COMMIT;}
  set ::nMaxBytes [sqlite_malloc_outstanding -maxbytes]
  expr $::nMaxBytes <= 100000
} {1}
do_test malloc5-4.3 {
  # Check that the content of table abc is at least roughly as expected.
  execsql {
    SELECT count(*), sum(a), sum(b) FROM abc;
  }
} [list 20000 [expr int(20000.0 * 4999.5)] [expr int(20000.0 * 4999.5)]]

# Unset the soft-heap-limit value.
db soft_heap_limit -1

finish_test

# This file implements regression tests for TCL interface to the
# SQLite library. 
#
# Actually, all tests are based on the TCL interface, so the main
# interface is pretty well tested.  This file contains some addition
# tests for fringe issues that the main test suite does not cover.
#
# $Id: tclsqlite.test,v 1.48 2005/12/20 09:19:38 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Check the error messages generated by tclsqlite
#
if {[sqlite3 -has-codec]} {
  set r "sqlite_orig HANDLE FILENAME ?-key CODEC-KEY?"
} else {
  set r "sqlite3 HANDLE FILENAME ?MODE?"
}
do_test tcl-1.1 {
  set v [catch {sqlite3 bogus} msg]
  lappend v $msg
} [list 1 "wrong # args: should be \"$r\""]
do_test tcl-1.2 {
  set v [catch {db bogus} msg]
  lappend v $msg
} {1 {bad option "bogus": must be authorizer, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, errorcode, eval, exists, function, last_insert_rowid, nullvalue, onecolumn, profile, progress, rekey, rollback_hook, release_memory, soft_heap_limit, timeout, total_changes, trace, transaction, update_hook, or version}}
do_test tcl-1.3 {
  execsql {CREATE TABLE t1(a int, b int)}
  execsql {INSERT INTO t1 VALUES(10,20)}
  set v [catch {
    db eval {SELECT * FROM t1} data {
      error "The error message"
    }