# 2005 November 30
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains test cases focused on the two memory-management APIs,
# sqlite3_soft_heap_limit() and sqlite3_release_memory().
#
# $Id: malloc5.test,v 1.18 2008/02/18 22:24:58 drh Exp $
#---------------------------------------------------------------------------
# NOTES ON EXPECTED BEHAVIOUR
#
#---------------------------------------------------------------------------
set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
db close
# Only run these tests if memory debugging is turned on.
#
if {!$MEMDEBUG} {
puts "Skipping malloc5 tests: not compiled with -DSQLITE_MEMDEBUG..."
finish_test
return
}
# Skip these tests if OMIT_MEMORY_MANAGEMENT was defined at compile time.
ifcapable !memorymanage {
finish_test
return
}
sqlite3_soft_heap_limit 0
sqlite3 db test.db
do_test malloc5-1.1 {
# Simplest possible test. Call sqlite3_release_memory when there is exactly
# one unused page in a single pager cache. This test case set's the
# value of the ::pgalloc variable, which is used in subsequent tests.
#
# Note: Even though executing this statement on an empty database
# modifies 2 pages (the root of sqlite_master and the new root page),
# the sqlite_master root (page 1) is never freed because the btree layer
# retains a reference to it for the entire transaction.
execsql {
PRAGMA auto_vacuum=OFF;
BEGIN;
CREATE TABLE abc(a, b, c);
}
set ::pgalloc [sqlite3_release_memory]
expr $::pgalloc > 0
} {1}
do_test malloc5-1.2 {
# Test that the transaction started in the above test is still active.
# Because the page freed had been written to, freeing it required a
# journal sync and exclusive lock on the database file. Test the file
# appears to be locked.
sqlite3 db2 test.db
catchsql {
SELECT * FROM abc;
} db2
} {1 {database is locked}}
do_test malloc5-1.3 {
# Again call [sqlite3_release_memory] when there is exactly one unused page
# in the cache. The same amount of memory is required, but no journal-sync
# or exclusive lock should be established.
execsql {
COMMIT;
BEGIN;
SELECT * FROM abc;
}
sqlite3_release_memory
} $::pgalloc
do_test malloc5-1.4 {
# Database should not be locked this time.
catchsql {
SELECT * FROM abc;
} db2
} {0 {}}
do_test malloc5-1.5 {
# Manipulate the cache so that it contains two unused pages. One requires
# a journal-sync to free, the other does not.
db2 close
execsql {
SELECT * FROM abc;
CREATE TABLE def(d, e, f);
}
sqlite3_release_memory 500
} $::pgalloc
do_test malloc5-1.6 {
# Database should not be locked this time. The above test case only
# requested 500 bytes of memory, which can be obtained by freeing the page
# that does not require an fsync().
sqlite3 db2 test.db
catchsql {
SELECT * FROM abc;
} db2
} {0 {}}
do_test malloc5-1.7 {
# Release another 500 bytes of memory. This time we require a sync(),
# so the database file will be locked afterwards.
db2 close
sqlite3_release_memory 500
} $::pgalloc
do_test malloc5-1.8 {
sqlite3 db2 test.db
catchsql {
SELECT * FROM abc;
} db2
} {1 {database is locked}}
do_test malloc5-1.9 {
execsql {
COMMIT;
}
} {}
do_test malloc5-2.1 {
# Put some data in tables abc and def. Both tables are still wholly
# contained within their root pages.
execsql {
INSERT INTO abc VALUES(1, 2, 3);
INSERT INTO abc VALUES(4, 5, 6);
INSERT INTO def VALUES(7, 8, 9);
INSERT INTO def VALUES(10,11,12);
}
} {}
do_test malloc5-2.2 {
# Load the root-page for table def into the cache. Then query table abc.
# Halfway through the query call sqlite3_release_memory(). The goal of this
# test is to make sure we don't free pages that are in use (specifically,
# the root of table abc).
set nRelease 0
execsql {
BEGIN;
SELECT * FROM def;
}
set data [list]
db eval {SELECT * FROM abc} {
incr nRelease [sqlite3_release_memory]
lappend data $a $b $c
}
execsql {
COMMIT;
}
list $nRelease $data
} [list $pgalloc [list 1 2 3 4 5 6]]
do_test malloc5-3.1 {
# Simple test to show that if two pagers are opened from within this
# thread, memory is freed from both when sqlite3_release_memory() is
# called.
execsql {
BEGIN;
SELECT * FROM abc;
}
execsql {
SELECT * FROM sqlite_master;
BEGIN;
SELECT * FROM def;
} db2
sqlite3_release_memory
} [expr $::pgalloc * 2]
do_test malloc5-3.2 {
concat \
[execsql {SELECT * FROM abc; COMMIT}] \
[execsql {SELECT * FROM def; COMMIT} db2]
} {1 2 3 4 5 6 7 8 9 10 11 12}
db2 close
puts "Highwater mark: [sqlite3_memory_highwater]"
# The following two test cases each execute a transaction in which
# 10000 rows are inserted into table abc. The first test case is used
# to ensure that more than 1MB of dynamic memory is used to perform
# the transaction.
#
# The second test case sets the "soft-heap-limit" to 100,000 bytes (0.1 MB)
# and tests to see that this limit is not exceeded at any point during
# transaction execution.
#
# Before executing malloc5-4.* we save the value of the current soft heap
# limit in variable ::soft_limit. The original value is restored after
# running the tests.
#
set ::soft_limit [sqlite3_soft_heap_limit -1]
execsql {PRAGMA cache_size=2000}
do_test malloc5-4.1 {
execsql {BEGIN;}
execsql {DELETE FROM abc;}
for {set i 0} {$i < 10000} {incr i} {
execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
}
execsql {COMMIT;}
set nMaxBytes [sqlite3_memory_highwater 1]
puts -nonewline " (Highwater mark: $nMaxBytes) "
expr $nMaxBytes > 1000000
} {1}
do_test malloc5-4.2 {
sqlite3_release_memory
sqlite3_soft_heap_limit 100000
sqlite3_memory_highwater 1
execsql {BEGIN;}
for {set i 0} {$i < 10000} {incr i} {
execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
}
execsql {COMMIT;}
set nMaxBytes [sqlite3_memory_highwater 1]
puts -nonewline " (Highwater mark: $nMaxBytes) "
# We used to test ($nMaxBytes<100000), because the soft-heap-limit is
# 100000 bytes. But if an allocation that will exceed the
# soft-heap-limit is requested from within the only pager instance in
# the system, then there is no way to free memory and the limit has to
# be exceeded. An exception is memory allocated to store actual page
# data (the code contains a special case for this).
#
# This is not a problem because all allocations apart from those
# used to store cached page data are both small and transient.
#
# Summary: the actual high-water mark for memory usage may be slightly
# higher than the soft-heap-limit. The specific allocations that cause
# the problem are the calls to sqlite3_malloc() inserted into selected
# sqlite3OsXXX() functions in test builds.
#
expr $nMaxBytes <= 100100
} {1}
do_test malloc5-4.3 {
# Check that the content of table abc is at least roughly as expected.
execsql {
SELECT count(*), sum(a), sum(b) FROM abc;
}
} [list 20000 [expr int(20000.0 * 4999.5)] [expr int(20000.0 * 4999.5)]]
# Restore the soft heap limit.
sqlite3_soft_heap_limit $::soft_limit
# Test that there are no problems calling sqlite3_release_memory when
# there are open in-memory databases.
#
# At one point these tests would cause a seg-fault.
#
do_test malloc5-5.1 {
db close
sqlite3 db :memory:
execsql {
BEGIN;
CREATE TABLE abc(a, b, c);
INSERT INTO abc VALUES('abcdefghi', 1234567890, NULL);
INSERT INTO abc SELECT * FROM abc;
INSERT INTO abc SELECT * FROM abc;
INSERT INTO abc SELECT * FROM abc;
INSERT INTO abc SELECT * FROM abc;
INSERT INTO abc SELECT * FROM abc;
INSERT INTO abc SELECT * FROM abc;
INSERT INTO abc SELECT * FROM abc;
}
sqlite3_release_memory
} 0
do_test malloc5-5.2 {
sqlite3_soft_heap_limit 5000
execsql {
COMMIT;
PRAGMA temp_store = memory;
SELECT * FROM abc ORDER BY a;
}
expr 1
} {1}
sqlite3_soft_heap_limit $::soft_limit
#-------------------------------------------------------------------------
# The following test cases (malloc5-6.*) test the new global LRU list
# used to determine the pages to recycle when sqlite3_release_memory is
# called and there is more than one pager open.
#
proc nPage {db} {
set bt [btree_from_db $db]
array set stats [btree_pager_stats $bt]
set stats(page)
}
db close
file delete -force test.db test.db-journal test2.db test2.db-journal
# This block of test-cases (malloc5-6.1.*) prepares two database files
# for the subsequent tests.
do_test malloc5-6.1.1 {
sqlite3 db test.db
execsql {
PRAGMA page_size=1024;
PRAGMA default_cache_size=10;
BEGIN;
CREATE TABLE abc(a PRIMARY KEY, b, c);
INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
INSERT INTO abc
SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
COMMIT;
}
copy_file test.db test2.db
sqlite3 db2 test2.db
list \
[expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]
} {1 1}
do_test malloc5-6.1.2 {
list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
} {10 10}
do_test malloc5-6.2.1 {
execsql { SELECT * FROM abc } db2
execsql {SELECT * FROM abc} db
list [nPage db] [nPage db2]
} {10 10}
do_test malloc5-6.2.2 {
# If we now try to reclaim some memory, it should come from the db2 cache.
sqlite3_release_memory 3000
list [nPage db] [nPage db2]
} {10 7}
do_test malloc5-6.2.3 {
# Access the db2 cache again, so that all the db2 pages have been used
# more recently than all the db pages. Then try to reclaim 3000 bytes.
# This time, 3 pages should be pulled from the db cache.
execsql { SELECT * FROM abc } db2
sqlite3_release_memory 3000
list [nPage db] [nPage db2]
} {7 10}
do_test malloc5-6.3.1 {
# Now open a transaction and update 2 pages in the db2 cache. Then
# do a SELECT on the db cache so that all the db pages are more recently
# used than the db2 pages. When we try to free memory, SQLite should
# free the non-dirty db2 pages, then the db pages, then finally use
# sync() to free up the dirty db2 pages. The only page that cannot be
# freed is page1 of db2. Because there is an open transaction, the
# btree layer holds a reference to page 1 in the db2 cache.
execsql {
BEGIN;
UPDATE abc SET c = randstr(100,100)
WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);
} db2
execsql { SELECT * FROM abc } db
list [nPage db] [nPage db2]
} {10 10}
do_test malloc5-6.3.2 {
# Try to release 7700 bytes. This should release all the
# non-dirty pages held by db2.
sqlite3_release_memory [expr 7*1100]
list [nPage db] [nPage db2]
} {10 3}
do_test malloc5-6.3.3 {
# Try to release another 1000 bytes. This should come fromt the db
# cache, since all three pages held by db2 are either in-use or diry.
sqlite3_release_memory 1000
list [nPage db] [nPage db2]
} {9 3}
do_test malloc5-6.3.4 {
# Now release 9900 more (about 9 pages worth). This should expunge
# the rest of the db cache. But the db2 cache remains intact, because
# SQLite tries to avoid calling sync().
sqlite3_release_memory 9900
list [nPage db] [nPage db2]
} {0 3}
do_test malloc5-6.3.5 {
# But if we are really insistent, SQLite will consent to call sync()
# if there is no other option.
sqlite3_release_memory 1000
list [nPage db] [nPage db2]
} {0 2}
do_test malloc5-6.3.6 {
# The referenced page (page 1 of the db2 cache) will not be freed no
# matter how much memory we ask for:
sqlite3_release_memory 31459
list [nPage db] [nPage db2]
} {0 1}
db2 close
sqlite3_soft_heap_limit $::soft_limit
finish_test
catch {db close}