/ Check-in [b01c65b0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Modifications to malloc5.test to account for the fact that sqlite3_release_memory() no longer reclaims dirty pages. (CVS 5625)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:b01c65b065c62e3dd71e88866a953668b5e2f25f
User & Date: danielk1977 2008-08-27 16:38:57
Context
2008-08-27
18:03
Do not mark pages as clean when doing a statement journal rollback. (CVS 5626) check-in: 9d7722f4 user: drh tags: trunk
16:38
Modifications to malloc5.test to account for the fact that sqlite3_release_memory() no longer reclaims dirty pages. (CVS 5625) check-in: b01c65b0 user: danielk1977 tags: trunk
16:14
Clear the $result variable prior to using it in corrupt2.test. (CVS 5624) check-in: 12f2d24f user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pcache.c.

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
...
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements that page cache.
**
** @(#) $Id: pcache.c,v 1.17 2008/08/27 15:16:34 danielk1977 Exp $
*/
#include "sqliteInt.h"

/*
** A complete page cache is an instance of this structure.
*/
struct PCache {
................................................................................
        return rc;
      }
      pcacheEnterGlobal();
    }
  }

  /* If the global page limit has been reached, try to recycle a page. */
  if( pcache.nCurrentPage>=pcache.nMaxPage ){
    p = pcacheRecyclePage();
  }

  /* If a page has been recycled but it is the wrong size, free it. */
  if( p && (p->pCache->szPage!=szPage || p->pCache->szExtra!=szExtra) ){
    pcachePageFree(p);
    p = 0;
................................................................................
  p->szPage = szPage;
  p->szExtra = szExtra;
  p->bPurgeable = bPurgeable;
  p->xDestroy = xDestroy;
  p->xStress = xStress;
  p->pStress = pStress;
  p->nMax = 100;
  p->nMin = 20;

  pcacheEnterGlobal();
  if( bPurgeable ){
    pcache.nMaxPage += p->nMax;
    pcache.nMinPage += p->nMin;
  }








|







 







|







 







|







7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
...
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements that page cache.
**
** @(#) $Id: pcache.c,v 1.18 2008/08/27 16:38:57 danielk1977 Exp $
*/
#include "sqliteInt.h"

/*
** A complete page cache is an instance of this structure.
*/
struct PCache {
................................................................................
        return rc;
      }
      pcacheEnterGlobal();
    }
  }

  /* If the global page limit has been reached, try to recycle a page. */
  if( pCache->bPurgeable && pcache.nCurrentPage>=pcache.nMaxPage ){
    p = pcacheRecyclePage();
  }

  /* If a page has been recycled but it is the wrong size, free it. */
  if( p && (p->pCache->szPage!=szPage || p->pCache->szExtra!=szExtra) ){
    pcachePageFree(p);
    p = 0;
................................................................................
  p->szPage = szPage;
  p->szExtra = szExtra;
  p->bPurgeable = bPurgeable;
  p->xDestroy = xDestroy;
  p->xStress = xStress;
  p->pStress = pStress;
  p->nMax = 100;
  p->nMin = 10;

  pcacheEnterGlobal();
  if( bPurgeable ){
    pcache.nMaxPage += p->nMax;
    pcache.nMinPage += p->nMin;
  }

Changes to test/malloc5.test.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
..
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

62
63
64
65
66


67
68
69
70
71
72
73


74
75







76
77
78
79
80
81
82

83
84
85
86
87

88



89
90
91
92
93

94
95
96
97
98

99
100
101
102
103
104
105
106
107
108
109
110


111
112
113

114


115
116
117
118
119

120
121
122
123
124
125
126
...
134
135
136
137
138
139
140

141
142
143
144
145
146
147
...
196
197
198
199
200
201
202



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
...
322
323
324
325
326
327
328

329
330
331
332
333
334
335
336
...
378
379
380
381
382
383
384
385

386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains test cases focused on the two memory-management APIs, 
# sqlite3_soft_heap_limit() and sqlite3_release_memory().
#
# $Id: malloc5.test,v 1.19 2008/08/21 15:54:01 danielk1977 Exp $

#---------------------------------------------------------------------------
# NOTES ON EXPECTED BEHAVIOUR
#
#---------------------------------------------------------------------------


set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
db close

# Only run these tests if memory debugging is turned on.
................................................................................
}

sqlite3_soft_heap_limit 0
sqlite3 db test.db

do_test malloc5-1.1 {
  # Simplest possible test. Call sqlite3_release_memory when there is exactly
  # one unused page in a single pager cache. This test case set's the 
  # value of the ::pgalloc variable, which is used in subsequent tests.
  #
  # Note: Even though executing this statement on an empty database 
  # modifies 2 pages (the root of sqlite_master and the new root page), 
  # the sqlite_master root (page 1) is never freed because the btree layer
  # retains a reference to it for the entire transaction. 
  execsql {
    PRAGMA auto_vacuum=OFF;
    BEGIN;
    CREATE TABLE abc(a, b, c);
  }
  set ::pgalloc [sqlite3_release_memory]
  expr $::pgalloc > 0
} {1}

do_test malloc5-1.2 {
  # Test that the transaction started in the above test is still active.
  # Because the page freed had been written to, freeing it required a
  # journal sync and exclusive lock on the database file. Test the file
  # appears to be locked.


  sqlite3 db2 test.db
  catchsql {
    SELECT * FROM abc;
  } db2
} {1 {database is locked}}
do_test malloc5-1.3 {
  # Again call [sqlite3_release_memory] when there is exactly one unused page 


  # in the cache. The same amount of memory is required, but no journal-sync
  # or exclusive lock should be established.







  execsql {
    COMMIT;
    BEGIN;
    SELECT * FROM abc;
  }
  sqlite3_release_memory
} $::pgalloc

do_test malloc5-1.4 {
  # Database should not be locked this time.
  catchsql {
    SELECT * FROM abc;
  } db2

} {0 {}}



do_test malloc5-1.5 {
  # Manipulate the cache so that it contains two unused pages. One requires 
  # a journal-sync to free, the other does not.
  db2 close
  execsql {

    SELECT * FROM abc;
    CREATE TABLE def(d, e, f);
  }
  sqlite3_release_memory 500
} $::pgalloc

do_test malloc5-1.6 {
  # Database should not be locked this time. The above test case only
  # requested 500 bytes of memory, which can be obtained by freeing the page
  # that does not require an fsync().
  sqlite3 db2 test.db
  catchsql {
    SELECT * FROM abc;
  } db2
} {0 {}}
do_test malloc5-1.7 {
  # Release another 500 bytes of memory. This time we require a sync(), 
  # so the database file will be locked afterwards.


  db2 close
  sqlite3_release_memory 500
} $::pgalloc

do_test malloc5-1.8 {


  sqlite3 db2 test.db
  catchsql {
    SELECT * FROM abc;
  } db2
} {1 {database is locked}}

do_test malloc5-1.9 {
  execsql {
    COMMIT;
  }
} {}

do_test malloc5-2.1 {
................................................................................
  }
} {}
do_test malloc5-2.2 {
  # Load the root-page for table def into the cache. Then query table abc. 
  # Halfway through the query call sqlite3_release_memory(). The goal of this
  # test is to make sure we don't free pages that are in use (specifically, 
  # the root of table abc).

  set nRelease 0
  execsql { 
    BEGIN;
    SELECT * FROM def;
  }
  set data [list]
  db eval {SELECT * FROM abc} {
................................................................................
do_test malloc5-4.1 {
  execsql {BEGIN;}
  execsql {DELETE FROM abc;}
  for {set i 0} {$i < 10000} {incr i} {
    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
  }
  execsql {COMMIT;}



  set nMaxBytes [sqlite3_memory_highwater 1]
  puts -nonewline " (Highwater mark: $nMaxBytes) "
  expr $nMaxBytes > 1000000
} {1}
do_test malloc5-4.2 {
  sqlite3_release_memory
  sqlite3_soft_heap_limit 110000
  sqlite3_memory_highwater 1
  execsql {BEGIN;}
  for {set i 0} {$i < 10000} {incr i} {
    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
  }
  execsql {COMMIT;}
  set nMaxBytes [sqlite3_memory_highwater 1]
  puts -nonewline " (Highwater mark: $nMaxBytes) "

  # We used to test ($nMaxBytes<100000), because the soft-heap-limit is
  # 100000 bytes. But if an allocation that will exceed the 
  # soft-heap-limit is requested from within the only pager instance in 
  # the system, then there is no way to free memory and the limit has to 
  # be exceeded. An exception is memory allocated to store actual page
  # data (the code contains a special case for this).
  #
  # This is not a problem because all allocations apart from those
  # used to store cached page data are both small and transient.
  #
  # Summary: the actual high-water mark for memory usage may be slightly 
  # higher than the soft-heap-limit. The specific allocations that cause
  # the problem are the calls to sqlite3_malloc() inserted into selected
  # sqlite3OsXXX() functions in test builds.
  #
  expr $nMaxBytes <= 110100
} {1}
do_test malloc5-4.3 {
  # Check that the content of table abc is at least roughly as expected.
  execsql {
    SELECT count(*), sum(a), sum(b) FROM abc;
  }
} [list 20000 [expr int(20000.0 * 4999.5)] [expr int(20000.0 * 4999.5)]]

# Restore the soft heap limit.
sqlite3_soft_heap_limit $::soft_limit

# Test that there are no problems calling sqlite3_release_memory when
# there are open in-memory databases.
#
................................................................................
    [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]
} {1 1}
do_test malloc5-6.1.2 {
  list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
} {10 10}

do_test malloc5-6.2.1 {

  execsql { SELECT * FROM abc } db2
  execsql {SELECT * FROM abc} db
  expr [nPage db] + [nPage db2]
} {20}

do_test malloc5-6.2.2 {
  # If we now try to reclaim some memory, it should come from the db2 cache.
  sqlite3_release_memory 3000
................................................................................
  # the rest of the db cache. But the db2 cache remains intact, because
  # SQLite tries to avoid calling sync().
  sqlite3_release_memory 9900
  list [nPage db] [nPage db2]
} {0 3}
do_test malloc5-6.3.5 {
  # But if we are really insistent, SQLite will consent to call sync()
  # if there is no other option.

  sqlite3_release_memory 1000
  list [nPage db] [nPage db2]
} {0 2}
do_test malloc5-6.3.6 {
  # The referenced page (page 1 of the db2 cache) will not be freed no
  # matter how much memory we ask for:
  sqlite3_release_memory 31459
  list [nPage db] [nPage db2]
} {0 1}

db2 close

sqlite3_soft_heap_limit $::soft_limit
finish_test
catch {db close}







|
|
|
|
|
|
|







 







|
|

<
<
<
<





|
<
|
>


<
|
<
>
>

<
|
|
<

|
>
>
|
|
>
>
>
>
>
>
>







>
|
<
|
|
<
>
|
>
>
>
|




>





>
|
|
<
<

<
|
<

|
<
<
>
>


<
>

>
>

<
|
<
<
>







 







>







 







>
>
>






|

|
<
<
<
<


<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|






|







 







>
|







 







|
>


|





|






8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
..
40
41
42
43
44
45
46
47
48
49




50
51
52
53
54
55

56
57
58
59

60

61
62
63

64
65

66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

88
89

90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108


109

110

111
112


113
114
115
116

117
118
119
120
121

122


123
124
125
126
127
128
129
130
...
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
...
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219




220
221
















222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
...
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
...
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains test cases focused on the two memory-management APIs, 
# sqlite3_soft_heap_limit() and sqlite3_release_memory().
#
# Prior to version 3.6.2, calling sqlite3_release_memory() or exceeding
# the configured soft heap limit could cause sqlite to upgrade database 
# locks and flush dirty pages to the file system. As of 3.6.2, this is
# no longer the case. In version 3.6.2, sqlite3_release_memory() only
# reclaims clean pages. This test file has been updated accordingly.
#
# $Id: malloc5.test,v 1.20 2008/08/27 16:38:57 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
db close

# Only run these tests if memory debugging is turned on.
................................................................................
}

sqlite3_soft_heap_limit 0
sqlite3 db test.db

do_test malloc5-1.1 {
  # Simplest possible test. Call sqlite3_release_memory when there is exactly
  # one unused page in a single pager cache. The page cannot be freed, as
  # it is dirty. So sqlite3_release_memory() returns 0.
  #




  execsql {
    PRAGMA auto_vacuum=OFF;
    BEGIN;
    CREATE TABLE abc(a, b, c);
  }
  sqlite3_release_memory

} {0}

do_test malloc5-1.2 {
  # Test that the transaction started in the above test is still active.

  # The lock on the database file should not have been upgraded (this was

  # not the case before version 3.6.2).
  #
  sqlite3 db2 test.db

  execsql { SELECT * FROM sqlite_master } db2
} {}

do_test malloc5-1.3 {
  # Call [sqlite3_release_memory] when there is exactly one unused page 
  # in the cache belonging to db2.
  #
  set ::pgalloc [sqlite3_release_memory]
  expr $::pgalloc > 0
} {1}

do_test malloc5-1.4 {
  # Commit the transaction and open a new one. Read 1 page into the cache.
  # Because the page is not dirty, it is eligible for collection even
  # before the transaction is concluded.
  #
  execsql {
    COMMIT;
    BEGIN;
    SELECT * FROM abc;
  }
  sqlite3_release_memory
} $::pgalloc

do_test malloc5-1.5 {

  # Conclude the transaction opened in the previous [do_test] block. This
  # causes another page (page 1) to become eligible for recycling.

  #
  execsql { COMMIT }
  sqlite3_release_memory
} $::pgalloc

do_test malloc5-1.6 {
  # Manipulate the cache so that it contains two unused pages. One requires 
  # a journal-sync to free, the other does not.
  db2 close
  execsql {
    BEGIN;
    SELECT * FROM abc;
    CREATE TABLE def(d, e, f);
  }
  sqlite3_release_memory 500
} $::pgalloc

do_test malloc5-1.7 {
  # Database should not be locked this time. 


  sqlite3 db2 test.db

  catchsql { SELECT * FROM abc } db2

} {0 {}}
do_test malloc5-1.8 {


  # Try to release another block of memory. This will fail as the only
  # pages currently in the cache are dirty (page 3) or pinned (page 1).
  db2 close
  sqlite3_release_memory 500

} 0
do_test malloc5-1.8 {
  # Database is still not locked.
  #
  sqlite3 db2 test.db

  catchsql { SELECT * FROM abc } db2


} {0 {}}
do_test malloc5-1.9 {
  execsql {
    COMMIT;
  }
} {}

do_test malloc5-2.1 {
................................................................................
  }
} {}
do_test malloc5-2.2 {
  # Load the root-page for table def into the cache. Then query table abc. 
  # Halfway through the query call sqlite3_release_memory(). The goal of this
  # test is to make sure we don't free pages that are in use (specifically, 
  # the root of table abc).
  sqlite3_release_memory
  set nRelease 0
  execsql { 
    BEGIN;
    SELECT * FROM def;
  }
  set data [list]
  db eval {SELECT * FROM abc} {
................................................................................
do_test malloc5-4.1 {
  execsql {BEGIN;}
  execsql {DELETE FROM abc;}
  for {set i 0} {$i < 10000} {incr i} {
    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
  }
  execsql {COMMIT;}
  sqlite3_release_memory
  sqlite3_memory_highwater 1
  execsql {SELECT * FROM abc}
  set nMaxBytes [sqlite3_memory_highwater 1]
  puts -nonewline " (Highwater mark: $nMaxBytes) "
  expr $nMaxBytes > 1000000
} {1}
do_test malloc5-4.2 {
  sqlite3_release_memory
  sqlite3_soft_heap_limit 100000
  sqlite3_memory_highwater 1
  execsql {SELECT * FROM abc}




  set nMaxBytes [sqlite3_memory_highwater 1]
  puts -nonewline " (Highwater mark: $nMaxBytes) "
















  expr $nMaxBytes <= 100000
} {1}
do_test malloc5-4.3 {
  # Check that the content of table abc is at least roughly as expected.
  execsql {
    SELECT count(*), sum(a), sum(b) FROM abc;
  }
} [list 10000 [expr int(10000.0 * 4999.5)] [expr int(10000.0 * 4999.5)]]

# Restore the soft heap limit.
sqlite3_soft_heap_limit $::soft_limit

# Test that there are no problems calling sqlite3_release_memory when
# there are open in-memory databases.
#
................................................................................
    [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20]
} {1 1}
do_test malloc5-6.1.2 {
  list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
} {10 10}

do_test malloc5-6.2.1 {
breakpoint
  execsql {SELECT * FROM abc} db2
  execsql {SELECT * FROM abc} db
  expr [nPage db] + [nPage db2]
} {20}

do_test malloc5-6.2.2 {
  # If we now try to reclaim some memory, it should come from the db2 cache.
  sqlite3_release_memory 3000
................................................................................
  # the rest of the db cache. But the db2 cache remains intact, because
  # SQLite tries to avoid calling sync().
  sqlite3_release_memory 9900
  list [nPage db] [nPage db2]
} {0 3}
do_test malloc5-6.3.5 {
  # But if we are really insistent, SQLite will consent to call sync()
  # if there is no other option. UPDATE: As of 3.6.2, SQLite will not
  # call sync() in this scenario. So no further memory can be reclaimed.
  sqlite3_release_memory 1000
  list [nPage db] [nPage db2]
} {0 3}
do_test malloc5-6.3.6 {
  # The referenced page (page 1 of the db2 cache) will not be freed no
  # matter how much memory we ask for:
  sqlite3_release_memory 31459
  list [nPage db] [nPage db2]
} {0 3}

db2 close

sqlite3_soft_heap_limit $::soft_limit
finish_test
catch {db close}