Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Refactor the implementation of the scratch memory allocator. Add the SQLITE_TESTCTRL_SCRATCHMALLOC interface to facilitate testing. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
a3475ddfbe4526e6e0b334fd1376ee7c |
User & Date: | drh 2010-08-27 17:16:45.000 |
Context
2010-08-27
| ||
17:48 | Add the sqlite3_create_function_v2() API, a version of create_function that allows a destructor to be specified. (check-in: 9a724dfbe8 user: dan tags: trunk) | |
17:16 | Refactor the implementation of the scratch memory allocator. Add the SQLITE_TESTCTRL_SCRATCHMALLOC interface to facilitate testing. (check-in: a3475ddfbe user: drh tags: trunk) | |
12:21 | Remove unnecessary code from malloc.c. Enhance pcache1.c so that is tries to reuse existing pages, rather than create new pages, when SQLite is under memory pressure. "Memory pressure" means that SQLITE_CONFIG_PAGECACHE memory is nearly exhausted or sqlite3_soft_heap_limit() has been reached. (check-in: 51049479a8 user: drh tags: trunk) | |
Changes
Changes to src/main.c.
︙ | ︙ | |||
2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 | ** ** Return the size of a pcache header in bytes. */ case SQLITE_TESTCTRL_PGHDRSZ: { rc = sizeof(PgHdr); break; } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; } | > > > > > > > > > > > > > > > > | 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 | ** ** Return the size of a pcache header in bytes. */ case SQLITE_TESTCTRL_PGHDRSZ: { rc = sizeof(PgHdr); break; } /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); ** ** Pass pFree into sqlite3ScratchFree(). ** If sz>0 then allocate a scratch buffer into pNew. */ case SQLITE_TESTCTRL_SCRATCHMALLOC: { void *pFree, **ppNew; int sz; sz = va_arg(ap, int); ppNew = va_arg(ap, void**); pFree = va_arg(ap, void*); if( sz ) *ppNew = sqlite3ScratchMalloc(sz); sqlite3ScratchFree(pFree); break; } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; } |
Changes to src/malloc.c.
︙ | ︙ | |||
67 68 69 70 71 72 73 74 75 76 77 | return nRet; #else UNUSED_PARAMETER(n); return SQLITE_OK; #endif } /* ** State information local to the memory allocation subsystem. */ static SQLITE_WSD struct Mem0Global { | > > > > > > > > < < < < | | > | > > | | | > > | | | | | > > > > > > > > | 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | return nRet; #else UNUSED_PARAMETER(n); return SQLITE_OK; #endif } /* ** An instance of the following object records the location of ** each unused scratch buffer. */ typedef struct ScratchFreeslot { struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ } ScratchFreeslot; /* ** State information local to the memory allocation subsystem. */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ /* ** The alarm callback and its arguments. The mem0.mutex lock will ** be held while the callback is running. Recursive calls into ** the memory subsystem are allowed, but no new callbacks will be ** issued. */ sqlite3_int64 alarmThreshold; void (*alarmCallback)(void*, sqlite3_int64,int); void *alarmArg; /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory ** (so that a range test can be used to determine if an allocation ** being freed came from pScratch) and a pointer to the list of ** unused scratch allocations. */ void *pScratchEnd; ScratchFreeslot *pScratchFree; u32 nScratchFree; /* ** True if heap is nearly "full" where "full" is defined by the ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; } mem0 = { 0, 0, 0, 0, 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) /* ** Initialize the memory allocation subsystem. */ int sqlite3MallocInit(void){ if( sqlite3GlobalConfig.m.xMalloc==0 ){ sqlite3MemSetDefault(); } memset(&mem0, 0, sizeof(mem0)); if( sqlite3GlobalConfig.bCoreMutex ){ mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); } if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 && sqlite3GlobalConfig.nScratch>=0 ){ int i, n, sz; ScratchFreeslot *pSlot; sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); sqlite3GlobalConfig.szScratch = sz; pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; n = sqlite3GlobalConfig.nScratch; mem0.pScratchFree = pSlot; mem0.nScratchFree = n; for(i=0; i<n-1; i++){ pSlot->pNext = (ScratchFreeslot*)(sz+(char*)pSlot); pSlot = pSlot->pNext; } pSlot->pNext = 0; mem0.pScratchEnd = (void*)&pSlot[1]; }else{ mem0.pScratchEnd = 0; sqlite3GlobalConfig.pScratch = 0; sqlite3GlobalConfig.szScratch = 0; sqlite3GlobalConfig.nScratch = 0; } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 || sqlite3GlobalConfig.nPage<1 ){ sqlite3GlobalConfig.pPage = 0; sqlite3GlobalConfig.szPage = 0; sqlite3GlobalConfig.nPage = 0; } |
︙ | ︙ | |||
323 324 325 326 327 328 329 330 | ** structures that would not normally fit on the stack of an ** embedded processor. */ void *sqlite3ScratchMalloc(int n){ void *p; assert( n>0 ); #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | > | > | < > | | < | < < | | | < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < > < < < < < < < < < < < < < < < < < < < < | 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 | ** structures that would not normally fit on the stack of an ** embedded processor. */ void *sqlite3ScratchMalloc(int n){ void *p; assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; mem0.nScratchFree--; sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); }else{ if( sqlite3GlobalConfig.bMemstat ){ sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); n = mallocWithAlarm(n, &p); if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); }else{ p = sqlite3GlobalConfig.m.xMalloc(n); } sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); } sqlite3_mutex_leave(mem0.mutex); #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) /* Verify that no more than two scratch allocations per thread ** are outstanding at one time. (This is only checked in the ** single-threaded case since checking in the multi-threaded case ** would be much more complicated.) */ assert( scratchAllocOut<=1 ); if( p ) scratchAllocOut++; #endif return p; } void sqlite3ScratchFree(void *p){ if( p ){ #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) /* Verify that no more than two scratch allocation per thread ** is outstanding at one time. (This is only checked in the ** single-threaded case since checking in the multi-threaded case ** would be much more complicated.) */ assert( scratchAllocOut>=1 && scratchAllocOut<=2 ); scratchAllocOut--; #endif if( p>=sqlite3GlobalConfig.pScratch && p<mem0.pScratchEnd ){ /* Release memory from the SQLITE_CONFIG_SCRATCH allocation */ ScratchFreeslot *pSlot; pSlot = (ScratchFreeslot*)p; sqlite3_mutex_enter(mem0.mutex); pSlot->pNext = mem0.pScratchFree; mem0.pScratchFree = pSlot; mem0.nScratchFree++; assert( mem0.nScratchFree<=sqlite3GlobalConfig.nScratch ); sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); sqlite3_mutex_leave(mem0.mutex); }else{ /* Release memory back to the heap */ assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); if( sqlite3GlobalConfig.bMemstat ){ int iSize = sqlite3MallocSize(p); sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ sqlite3GlobalConfig.m.xFree(p); } } } } /* ** TRUE if p is a lookaside memory allocation from db */ |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
1253 1254 1255 1256 1257 1258 1259 | ** ** <dt>SQLITE_CONFIG_SCRATCH</dt> ** <dd> ^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz | | < | | | | | < | | 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 | ** ** <dt>SQLITE_CONFIG_SCRATCH</dt> ** <dd> ^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz ** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. ** ^SQLite will use no more than two scratch buffers per thread. So ** N should be set to twice the expected maximum number of threads. ** ^SQLite will never require a scratch buffer that is more than 6 ** times the database page size. ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.</dd> ** ** <dt>SQLITE_CONFIG_PAGECACHE</dt> ** <dd> ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implemenation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. ** There are three arguments to this option: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus a little extra for each ** page header. ^The page header size is 20 to 40 bytes depending on ** the host architecture. ^It is harmless, apart from the wasted memory, ** to make sz a little too large. The first ** argument should point to an allocation of at least sz*N bytes of memory. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.</dd> ** ** <dt>SQLITE_CONFIG_HEAP</dt> ** <dd> ^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. |
︙ | ︙ | |||
5096 5097 5098 5099 5100 5101 5102 | #define SQLITE_TESTCTRL_PENDING_BYTE 11 #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 | > | | 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 | #define SQLITE_TESTCTRL_PENDING_BYTE 11 #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 #define SQLITE_TESTCTRL_SCRATCHMALLOC 18 #define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for |
︙ | ︙ |
Changes to test/memsubsys1.test.
1 2 3 4 5 6 7 8 9 10 11 12 13 | # 2008 June 18 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains tests of the memory allocation subsystem # | < | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | # 2008 June 18 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains tests of the memory allocation subsystem # set testdir [file dirname $argv0] source $testdir/tester.tcl sqlite3_reset_auto_extension # This test assumes that no page-cache or scratch buffers are installed # by default when a new database connection is opened. As a result, it |
︙ | ︙ |