Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add some comments and test-cases for the global lru page list (used by sqlite3_release_memory()). (CVS 4308) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
0b80168895993af5774369f839f28471 |
User & Date: | danielk1977 2007-08-28 08:00:18.000 |
Context
2007-08-28
| ||
08:19 | Use "affected" instead of "effected". Ticket #2589. No code changes. (CVS 4309) (check-in: 4c5631ce34 user: danielk1977 tags: trunk) | |
08:00 | Add some comments and test-cases for the global lru page list (used by sqlite3_release_memory()). (CVS 4308) (check-in: 0b80168895 user: danielk1977 tags: trunk) | |
02:27 | Work toward correct btree locking in a multithreaded environment. (CVS 4307) (check-in: b8cc493b47 user: drh tags: trunk) | |
Changes
Changes to src/pager.c.
︙ | ︙ | |||
14 15 16 17 18 19 20 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** | | | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** ** @(#) $Id: pager.c,v 1.378 2007/08/28 08:00:18 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" #include <assert.h> #include <string.h> /* |
︙ | ︙ | |||
128 129 130 131 132 133 134 135 136 137 138 139 140 141 | /* ** This macro rounds values up so that if the value is an address it ** is guaranteed to be an address that is aligned to an 8-byte boundary. */ #define FORCE_ALIGNMENT(X) (((X)+7)&~7) typedef struct PgHdr PgHdr; typedef struct PagerLruLink PagerLruLink; struct PagerLruLink { PgHdr *pNext; PgHdr *pPrev; }; /* | > > > > > > > > > > > > > > > > > > > > > > > > > > > | 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | /* ** This macro rounds values up so that if the value is an address it ** is guaranteed to be an address that is aligned to an 8-byte boundary. */ #define FORCE_ALIGNMENT(X) (((X)+7)&~7) typedef struct PgHdr PgHdr; /* ** Each pager stores all currently unreferenced pages in a list sorted ** in least-recently-used (LRU) order (i.e. the first item on the list has ** not been referenced in a long time, the last item has been recently ** used). An instance of this structure is included as part of each ** pager structure for this purpose (variable Pager.lru). ** ** Additionally, if memory-management is enabled, all unreferenced pages ** are stored in a global LRU list (global variable sqlite3LruPageList). ** ** In both cases, the PagerLruList.pFirstSynced variable points to ** the first page in the corresponding list that does not require an ** fsync() operation before it's memory can be reclaimed. If no such ** page exists, PagerLruList.pFirstSynced is set to NULL. */ typedef struct PagerLruList PagerLruList; struct PagerLruList { PgHdr *pFirst; /* First page in LRU list */ PgHdr *pLast; /* Last page in LRU list (the most recently used) */ PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */ }; /* ** The following structure contains the next and previous pointers used ** to link a PgHdr structure into a PagerLruList linked list. */ typedef struct PagerLruLink PagerLruLink; struct PagerLruLink { PgHdr *pNext; PgHdr *pPrev; }; /* |
︙ | ︙ | |||
288 289 290 291 292 293 294 | */ #define PGHDR_TO_DATA(P) ((void*)(&(P)[1])) #define DATA_TO_PGHDR(D) (&((PgHdr*)(D))[-1]) #define PGHDR_TO_EXTRA(G,P) ((void*)&((char*)(&(G)[1]))[(P)->pageSize]) #define PGHDR_TO_HIST(P,PGR) \ ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra]) | < < < < < < < | 315 316 317 318 319 320 321 322 323 324 325 326 327 328 | */ #define PGHDR_TO_DATA(P) ((void*)(&(P)[1])) #define DATA_TO_PGHDR(D) (&((PgHdr*)(D))[-1]) #define PGHDR_TO_EXTRA(G,P) ((void*)&((char*)(&(G)[1]))[(P)->pageSize]) #define PGHDR_TO_HIST(P,PGR) \ ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra]) /* ** A open page cache is an instance of the following structure. ** ** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or ** or SQLITE_FULL. Once one of the first three errors occurs, it persists ** and is returned as the result of every major pager API call. The ** SQLITE_FULL return code is slightly different. It persists only until the |
︙ | ︙ | |||
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 | cnt++; /* Something to set a breakpoint on */ } # define REFINFO(X) pager_refinfo(X) #else # define REFINFO(X) #endif static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ pLink->pNext = 0; pLink->pPrev = pList->pLast; if( pList->pLast ){ int iOff = (char *)pLink - (char *)pPg; PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]); pLastLink->pNext = pPg; }else{ assert(!pList->pFirst); pList->pFirst = pPg; } pList->pLast = pPg; if( !pList->pFirstSynced && pPg->needSync==0 ){ pList->pFirstSynced = pPg; } } static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ int iOff = (char *)pLink - (char *)pPg; if( pPg==pList->pFirst ){ pList->pFirst = pLink->pNext; } if( pPg==pList->pLast ){ pList->pLast = pLink->pPrev; } | > > > > > > > > > > > > > > > > > > > > > > > | 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 | cnt++; /* Something to set a breakpoint on */ } # define REFINFO(X) pager_refinfo(X) #else # define REFINFO(X) #endif /* ** Add page pPg to the end of the linked list managed by structure ** pList (pPg becomes the last entry in the list - the most recently ** used). Argument pLink should point to either pPg->free or pPg->gfree, ** depending on whether pPg is being added to the pager-specific or ** global LRU list. */ static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ pLink->pNext = 0; pLink->pPrev = pList->pLast; #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT assert(pLink==&pPg->free || pLink==&pPg->gfree); assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList); #endif if( pList->pLast ){ int iOff = (char *)pLink - (char *)pPg; PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]); pLastLink->pNext = pPg; }else{ assert(!pList->pFirst); pList->pFirst = pPg; } pList->pLast = pPg; if( !pList->pFirstSynced && pPg->needSync==0 ){ pList->pFirstSynced = pPg; } } /* ** Remove pPg from the list managed by the structure pointed to by pList. ** ** Argument pLink should point to either pPg->free or pPg->gfree, depending ** on whether pPg is being added to the pager-specific or global LRU list. */ static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ int iOff = (char *)pLink - (char *)pPg; #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT assert(pLink==&pPg->free || pLink==&pPg->gfree); assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList); #endif if( pPg==pList->pFirst ){ pList->pFirst = pLink->pNext; } if( pPg==pList->pLast ){ pList->pLast = pLink->pPrev; } |
︙ | ︙ | |||
576 577 578 579 580 581 582 | pList->pFirstSynced = p; } pLink->pNext = pLink->pPrev = 0; } /* | | > > | > > > > | > > | 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 | pList->pFirstSynced = p; } pLink->pNext = pLink->pPrev = 0; } /* ** Add page pPg to the list of free pages for the pager. If ** memory-management is enabled, also add the page to the global ** list of free pages. */ static void lruListAdd(PgHdr *pPg){ listAdd(&pPg->pPager->lru, &pPg->free, pPg); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( !pPg->pPager->memDb ){ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); listAdd(&sqlite3LruPageList, &pPg->gfree, pPg); sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); } #endif } /* ** Remove page pPg from the list of free pages for the associated pager. ** If memory-management is enabled, also remove pPg from the global list ** of free pages. */ static void lruListRemove(PgHdr *pPg){ listRemove(&pPg->pPager->lru, &pPg->free, pPg); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( !pPg->pPager->memDb ){ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); listRemove(&sqlite3LruPageList, &pPg->gfree, pPg); sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); } #endif } /* ** This function is called just after the needSync flag has been cleared ** from all pages managed by pPager (usually because the journal file ** has just been synced). It updates the pPager->lru.pFirstSynced variable ** and, if memory-management is enabled, the sqlite3LruPageList.pFirstSynced ** variable also. */ static void lruListSetFirstSynced(Pager *pPager){ pPager->lru.pFirstSynced = pPager->lru.pFirst; #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( !pPager->memDb ){ PgHdr *p; sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); |
︙ | ︙ | |||
3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 | pPg = sqlite3LruPageList.pFirst; while( pPg && pPg->pPager->iInUseDB ){ pPg = pPg->gfree.pNext; } } sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); if( !pPg ) break; pPager = pPg->pPager; assert(!pPg->needSync || pPg==pPager->lru.pFirst); assert(pPg->needSync || pPg==pPager->lru.pFirstSynced); rc = pager_recycle(pPager, 1, &pRecycled); | > > > > | 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 | pPg = sqlite3LruPageList.pFirst; while( pPg && pPg->pPager->iInUseDB ){ pPg = pPg->gfree.pNext; } } sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU)); /* If pPg==0, then the block above has failed to find a page to ** recycle. In this case return early - no further memory will ** be released. */ if( !pPg ) break; pPager = pPg->pPager; assert(!pPg->needSync || pPg==pPager->lru.pFirst); assert(pPg->needSync || pPg==pPager->lru.pFirstSynced); rc = pager_recycle(pPager, 1, &pRecycled); |
︙ | ︙ | |||
3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 | sizeof(*pPg) + pPager->pageSize + sizeof(u32) + pPager->nExtra + MEMDB*sizeof(PgHistory) ); IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno)); PAGER_INCR(sqlite3_pager_pgfree_count); sqlite3_free(pPg); }else{ /* An error occured whilst writing to the database file or ** journal in pager_recycle(). The error is not returned to the ** caller of this function. Instead, set the Pager.errCode variable. ** The error will be returned to the user (or users, in the case ** of a shared pager cache) of the pager for which the error occured. */ | > | 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 | sizeof(*pPg) + pPager->pageSize + sizeof(u32) + pPager->nExtra + MEMDB*sizeof(PgHistory) ); IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno)); PAGER_INCR(sqlite3_pager_pgfree_count); sqlite3_free(pPg); pPager->nPage--; }else{ /* An error occured whilst writing to the database file or ** journal in pager_recycle(). The error is not returned to the ** caller of this function. Instead, set the Pager.errCode variable. ** The error will be returned to the user (or users, in the case ** of a shared pager cache) of the pager for which the error occured. */ |
︙ | ︙ |
Changes to test/malloc5.test.
︙ | ︙ | |||
8 9 10 11 12 13 14 | # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains test cases focused on the two memory-management APIs, # sqlite3_soft_heap_limit() and sqlite3_release_memory(). # | | | 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains test cases focused on the two memory-management APIs, # sqlite3_soft_heap_limit() and sqlite3_release_memory(). # # $Id: malloc5.test,v 1.14 2007/08/28 08:00:18 danielk1977 Exp $ #--------------------------------------------------------------------------- # NOTES ON EXPECTED BEHAVIOUR # #--------------------------------------------------------------------------- |
︙ | ︙ | |||
244 245 246 247 248 249 250 | INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; } sqlite3_release_memory } 0 | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 | INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; } sqlite3_release_memory } 0 do_test malloc5-5.2 { sqlite3_soft_heap_limit 5000 execsql { COMMIT; PRAGMA temp_store = memory; SELECT * FROM abc ORDER BY a; } expr 1 } {1} sqlite3_soft_heap_limit $::soft_limit #------------------------------------------------------------------------- # The following test cases (malloc5-6.*) test the new global LRU list # used to determine the pages to recycle when sqlite3_release_memory is # called and there is more than one pager open. # proc nPage {db} { set bt [btree_from_db $db] array set stats [btree_pager_stats $bt] set stats(page) } db close file delete -force test.db test.db-journal test2.db test2.db-journal # This block of test-cases (malloc5-6.1.*) prepares two database files # for the subsequent tests. do_test malloc5-6.1.1 { sqlite3 db test.db execsql { PRAGMA page_size=1024; PRAGMA default_cache_size=10; BEGIN; CREATE TABLE abc(a PRIMARY KEY, b, c); INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100)); INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; INSERT INTO abc SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc; COMMIT; } copy_file test.db test2.db sqlite3 db2 test2.db list [expr [file size test.db]/1024] [expr [file size test2.db]/1024] } {23 23} do_test malloc5-6.1.2 { list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2] } {10 10} do_test malloc5-6.2.1 { execsql { SELECT * FROM abc } db2 execsql {SELECT * FROM abc} db list [nPage db] [nPage db2] } {10 10} do_test malloc5-6.2.2 { # If we now try to reclaim some memory, it should come from the db2 cache. sqlite3_release_memory 3000 list [nPage db] [nPage db2] } {10 7} do_test malloc5-6.2.3 { # Access the db2 cache again, so that all the db2 pages have been used # more recently than all the db pages. Then try to reclaim 3000 bytes. # This time, 3 pages should be pulled from the db cache. execsql { SELECT * FROM abc } db2 sqlite3_release_memory 3000 list [nPage db] [nPage db2] } {7 10} do_test malloc5-6.3.1 { # Now open a transaction and update 2 pages in the db2 cache. Then # do a SELECT on the db cache so that all the db pages are more recently # used than the db2 pages. When we try to free memory, SQLite should # free the non-dirty db2 pages, then the db pages, then finally use # sync() to free up the dirty db2 pages. The only page that cannot be # freed is page1 of db2. Because there is an open transaction, the # btree layer holds a reference to page 1 in the db2 cache. execsql { BEGIN; UPDATE abc SET c = randstr(100,100) WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc); } db2 execsql { SELECT * FROM abc } db list [nPage db] [nPage db2] } {10 10} do_test malloc5-6.3.2 { # Try to release 7700 bytes. This should release all the # non-dirty pages held by db2. sqlite3_release_memory [expr 7*1100] list [nPage db] [nPage db2] } {10 3} do_test malloc5-6.3.3 { # Try to release another 1000 bytes. This should come fromt the db # cache, since all three pages held by db2 are either in-use or diry. sqlite3_release_memory 1000 list [nPage db] [nPage db2] } {9 3} do_test malloc5-6.3.4 { # Now release 9900 more (about 9 pages worth). This should expunge # the rest of the db cache. But the db2 cache remains intact, because # SQLite tries to avoid calling sync(). sqlite3_release_memory 9900 list [nPage db] [nPage db2] } {0 3} do_test malloc5-6.3.5 { # But if we are really insistent, SQLite will consent to call sync() # if there is no other option. sqlite3_release_memory 1000 list [nPage db] [nPage db2] } {0 2} do_test malloc5-6.3.6 { # The referenced page (page 1 of the db2 cache) will not be freed no # matter how much memory we ask for: sqlite3_release_memory 31459 list [nPage db] [nPage db2] } {0 1} db2 close sqlite3_soft_heap_limit $::soft_limit finish_test catch {db close} |