Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -232,10 +232,12 @@ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ + $(TOP)/src/lookaside.c \ + $(TOP)/src/lookaside.h \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/mem0.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ @@ -523,10 +525,11 @@ $(TOP)/src/btree.h \ $(TOP)/src/btreeInt.h \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ keywordhash.h \ + $(TOP)/src/lookaside.h \ $(TOP)/src/msvc.h \ $(TOP)/src/mutex.h \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ @@ -867,10 +870,13 @@ $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c loadext.lo: $(TOP)/src/loadext.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/loadext.c +lookaside.lo: $(TOP)/src/lookaside.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/lookaside.c + main.lo: $(TOP)/src/main.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/main.c malloc.lo: $(TOP)/src/malloc.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/malloc.c Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1288,10 +1288,11 @@ $(TOP)\src\global.c \ $(TOP)\src\hash.c \ $(TOP)\src\insert.c \ $(TOP)\src\legacy.c \ $(TOP)\src\loadext.c \ + $(TOP)\src\lookaside.c \ $(TOP)\src\main.c \ $(TOP)\src\malloc.c \ $(TOP)\src\mem0.c \ $(TOP)\src\mem1.c \ $(TOP)\src\mem2.c \ @@ -1358,10 +1359,11 @@ SRC04 = \ $(TOP)\src\btree.h \ $(TOP)\src\btreeInt.h \ $(TOP)\src\hash.h \ $(TOP)\src\hwtime.h \ + $(TOP)\src\lookaside.h \ $(TOP)\src\msvc.h \ $(TOP)\src\mutex.h \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ $(TOP)\src\os_setup.h \ @@ -1578,10 +1580,11 @@ HDR = \ $(TOP)\src\btree.h \ $(TOP)\src\btreeInt.h \ $(TOP)\src\hash.h \ $(TOP)\src\hwtime.h \ + $(TOP)\src\lookaside.h \ keywordhash.h \ $(TOP)\src\msvc.h \ $(TOP)\src\mutex.h \ opcodes.h \ $(TOP)\src\os.h \ @@ -1959,10 +1962,13 @@ $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c loadext.lo: $(TOP)\src\loadext.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\loadext.c +lookaside.lo: $(TOP)\src\lookaside.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\lookaside.c + main.lo: $(TOP)\src\main.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\main.c malloc.lo: $(TOP)\src\malloc.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\malloc.c Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -109,10 +109,12 @@ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ + $(TOP)/src/lookaside.c \ + $(TOP)/src/lookaside.h \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/mem0.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ @@ -441,10 +443,11 @@ $(TOP)/src/btree.h \ $(TOP)/src/btreeInt.h \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ keywordhash.h \ + $(TOP)/src/lookaside.h \ $(TOP)/src/msvc.h \ $(TOP)/src/mutex.h \ opcodes.h \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ Index: src/analyze.c ================================================================== --- src/analyze.c +++ src/analyze.c @@ -139,10 +139,11 @@ ** all contain just a single integer which is the same as the first ** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE #include "sqliteInt.h" +#include "lookaside.h" #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 #else # define IsStat4 0 @@ -1660,11 +1661,11 @@ sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ char *zSql; /* Text of the SQL statement */ Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ - assert( db->lookaside.bDisable ); + assert( sqlite3LookasideDisabled(&db->lookaside) ); zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); @@ -1771,11 +1772,11 @@ ** the Index.aSample[] arrays of all indices. */ static int loadStat4(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; /* Result codes from subroutines */ - assert( db->lookaside.bDisable ); + assert( sqlite3LookasideDisabled(&db->lookaside) ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ rc = loadStatTbl(db, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb @@ -1852,13 +1853,13 @@ } /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT4 if( rc==SQLITE_OK ){ - DisableLookaside; + sqlite3LookasideDisable(&db->lookaside); rc = loadStat4(db, sInfo.zDatabase); - EnableLookaside; + sqlite3LookasideEnable(&db->lookaside); } for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -21,10 +21,11 @@ ** BEGIN TRANSACTION ** COMMIT ** ROLLBACK */ #include "sqliteInt.h" +#include "lookaside.h" #ifndef SQLITE_OMIT_SHARED_CACHE /* ** The TableLock structure is only used by the sqlite3TableLock() and ** codeTableLocks() functions. @@ -624,11 +625,11 @@ ** If malloc has already failed, it may be that it failed while allocating ** a Table object that was going to be marked ephemeral. So do not check ** that no lookaside memory is used in this case either. */ int nLookaside = 0; if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ - nLookaside = sqlite3LookasideUsed(db, 0); + nLookaside = sqlite3LookasideUsed(&db->lookaside, 0); } #endif /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ @@ -660,11 +661,11 @@ sqlite3VtabClear(db, pTable); #endif sqlite3DbFree(db, pTable); /* Verify that no lookaside memory was used by schema tables */ - assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(db,0) ); + assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(&db->lookaside,0) ); } void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ if( !pTable ) return; if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -10,10 +10,11 @@ ************************************************************************* ** This file contains code used by the compiler to add foreign key ** support to compiled SQL statements. */ #include "sqliteInt.h" +#include "lookaside.h" #ifndef SQLITE_OMIT_FOREIGN_KEY #ifndef SQLITE_OMIT_TRIGGER /* @@ -1297,11 +1298,11 @@ ); pWhere = 0; } /* Disable lookaside memory allocation */ - DisableLookaside; + sqlite3LookasideDisable(&db->lookaside); pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ sizeof(TriggerStep) + /* Single step in trigger program */ nFrom + 1 /* Space for pStep->zTarget */ @@ -1319,11 +1320,11 @@ pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); } } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - EnableLookaside; + sqlite3LookasideEnable(&db->lookaside); sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); sqlite3ExprListDelete(db, pList); sqlite3SelectDelete(db, pSelect); ADDED src/lookaside.c Index: src/lookaside.c ================================================================== --- /dev/null +++ src/lookaside.c @@ -0,0 +1,305 @@ +/* +** 2019-10-02 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Lookaside memory allocation functions used throughout sqlite. +*/ + +#include "sqliteInt.h" +#include "lookaside.h" + +/* +** Return the number of LookasideSlot elements on the linked list +*/ +static u32 countLookasideSlots(LookasideSlot *p){ + u32 cnt = 0; + while( p ){ + p = p->pNext; + cnt++; + } + return cnt; +} + +/* +** Count the number of slots of lookaside memory that are outstanding +*/ +int sqlite3LookasideUsed(Lookaside *pLookaside, int *pHighwater){ + u32 nInit = countLookasideSlots(pLookaside->pInit); + u32 nFree = countLookasideSlots(pLookaside->pFree); + if( pHighwater ) *pHighwater = pLookaside->nSlot - nInit; + return pLookaside->nSlot - (nInit+nFree); +} + +void sqlite3LookasideResetUsed(Lookaside *pLookaside){ + LookasideSlot *p = pLookaside->pFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = pLookaside->pInit; + pLookaside->pInit = pLookaside->pFree; + pLookaside->pFree = 0; + } +} + +#ifndef SQLITE_OMIT_LOOKASIDE + +static void *lookasideSlotAlloc(Lookaside *pLookaside){ + LookasideSlot *pBuf; + if( (pBuf = pLookaside->pFree)!=0 ){ + pLookaside->pFree = pBuf->pNext; + pLookaside->anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = pLookaside->pInit)!=0 ){ + pLookaside->pInit = pBuf->pNext; + pLookaside->anStat[0]++; + return (void*)pBuf; + }else{ + pLookaside->anStat[2]++; + return 0; + } +} + +static void lookasideSlotFree(Lookaside *pLookaside, void *p){ + LookasideSlot *pBuf = (LookasideSlot*)p; +# ifdef SQLITE_DEBUG + /* Scribble over the content in the buffer being freed */ + memset(p, 0xaa, pLookaside->szTrue); +# endif + pBuf->pNext = pLookaside->pFree; + pLookaside->pFree = pBuf; +} + +# ifndef SQLITE_OMIT_MINI_LOOKASIDE +# ifndef SQLITE_MINI_LOOKASIDE_MIN_SLOT_SIZE +# define SQLITE_MINI_LOOKASIDE_MIN_SLOT_SIZE 128 +# endif + +static void *miniLookasideAlloc(Lookaside *pLookaside){ + LookasideSlot *pMiniSlot; + LookasideSlot *pSlot; + int iMiniSlot; + + if( !pLookaside->pMini ){ + pSlot = lookasideSlotAlloc(pLookaside); + if( !pSlot ){ + return 0; + } + bzero(pSlot, sizeof(LookasideSlot)); + pLookaside->pMini = pSlot; + }else{ + pSlot = pLookaside->pMini; + assert( pSlot->nAlloc ); + } + + assert( pSlot->nAlloc < pLookaside->nMini ); + + if( (pMiniSlot = pSlot->pFree) ){ + pSlot->pFree = pMiniSlot->pNext; + }else{ + iMiniSlot = pSlot->nAlloc; + assert(iMiniSlot < pLookaside->nMini); + pMiniSlot = (LookasideSlot *)((char *)pSlot + sizeof(LookasideSlot) + (pLookaside->szMini * iMiniSlot)); + } + + /* Remove slot from pMini if it is full of sub-allocations */ + if( ++(pSlot->nAlloc) == pLookaside->nMini ){ + /* Slot is full, dequeue from list */ + if( pSlot->pNext ){ + assert( pSlot->pNext->pPrev == pSlot ); + pSlot->pNext->pPrev = pSlot->pPrev; + } + if( pSlot->pPrev ){ + assert( pSlot->pPrev->pNext == pSlot ); + pSlot->pPrev->pNext = pSlot->pNext; + }else{ + assert( pLookaside->pMini == pSlot ); + pLookaside->pMini = pSlot->pNext; + } + pSlot->pNext = pSlot->pPrev = 0; + } + return pMiniSlot; +} + +static void miniLookasideFree(Lookaside *pLookaside, void *p){ + int iSlotNum = ((u8*)p - (u8*)pLookaside->pStart) / pLookaside->szTrue; + LookasideSlot *pSlot = (LookasideSlot *)(iSlotNum * pLookaside->szTrue + (u8*)pLookaside->pStart); + LookasideSlot *pMiniSlot = (LookasideSlot *)p; + + assert( pSlot->nAlloc ); + assert( pSlot->nAlloc <= pLookaside->nMini ); + assert( iMiniSlotnMini ); + + /* Return slot to pMini list if it was full */ + if( pSlot->nAlloc==pLookaside->nMini ){ + assert( pSlot->pNext == pSlot->pPrev && pSlot->pPrev == 0 ); + if( pLookaside->pMini ){ + assert( !pLookaside->pMini->pPrev ); + pSlot->pNext = pLookaside->pMini; + pSlot->pNext->pPrev = pSlot; + } + pLookaside->pMini = pSlot; + } + + +#ifdef SQLITE_DEBUG + memset(p, 0xaa, pLookaside->szMini); +#endif + pSlot->nAlloc--; + pMiniSlot->pNext = pSlot->pFree; + pSlot->pFree = pMiniSlot; + + /* Return slot to the lookaside pool if it is empty */ + if( pSlot->nAlloc == 0 ){ + if( pSlot->pNext ){ + assert( pSlot->pNext->pPrev == pSlot ); + pSlot->pNext->pPrev = pSlot->pPrev; + } + if( pSlot->pPrev ){ + assert( pSlot->pPrev->pNext == pSlot ); + pSlot->pPrev->pNext = pSlot->pNext; + }else{ + assert( pLookaside->pMini==pSlot ); + pLookaside->pMini = pSlot->pNext; + } + lookasideSlotFree(pLookaside, pSlot); + } +} + +# else +# define miniLookasideAlloc(A) lookasideSlotAlloc(A) +# define miniLookasideFree(A, B) lookasideSlotFree(A, B) +# endif /* !SQLITE_OMIT_MINI_LOOKASIDE */ + +int sqlite3LookasideOpen(void *pBuf, int sz, int cnt, Lookaside *pLookaside){ + void *pStart; + + if( sqlite3LookasideUsed(pLookaside,0)>0 ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( pLookaside->bMalloced ){ + sqlite3_free(pLookaside->pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than sizeof(LookasideSlot) to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*(sqlite3_int64)cnt ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + }else{ + pStart = pBuf; + } + pLookaside->pStart = pStart; + pLookaside->pInit = 0; + pLookaside->pFree = 0; + pLookaside->sz = (u16)sz; + pLookaside->szTrue = (u16)sz; +#ifndef SQLITE_OMIT_MINILOOKASIDE + pLookaside->pMini = 0; + pLookaside->nMini = (sz - sizeof(LookasideSlot)) / SQLITE_MINI_LOOKASIDE_MIN_SLOT_SIZE; + if( pLookaside->nMini ){ + pLookaside->szMini = ((sz - sizeof(LookasideSlot)) / pLookaside->nMini) & ~(sizeof(void *) - 1); + }else{ + pLookaside->szMini = 0; + } +#endif /* SQLITE_OMIT_MINILOOKASIDE */ + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + pLookaside->nSlot = cnt; + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = pLookaside->pInit; + pLookaside->pInit = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + pLookaside->pEnd = p; + pLookaside->bDisable = 0; + pLookaside->bMalloced = pBuf==0 ?1:0; + }else{ + pLookaside->pStart = 0; + pLookaside->pEnd = 0; + pLookaside->bDisable = 1; + pLookaside->bMalloced = 0; + pLookaside->nSlot = 0; + } + return SQLITE_OK; +} + +void sqlite3LookasideClose(Lookaside *pLookaside){ + assert( sqlite3LookasideUsed(pLookaside,0)==0 ); + if( pLookaside->bMalloced ){ + sqlite3_free(pLookaside->pStart); + } +} + +int sqlite3IsLookaside(Lookaside *pLookaside, void *p){ + return SQLITE_WITHIN(p, pLookaside->pStart, pLookaside->pEnd); +} + +/* +** Returns a pointer to a region at least n bytes in size, or NULL if the +** lookaside allocator has exhausted its available memory. +*/ +void *sqlite3LookasideAlloc(Lookaside *pLookaside, u64 n){ + if( n>pLookaside->sz ){ + if( !pLookaside->bDisable ){ + pLookaside->anStat[1]++; + } + return 0; + } + if( n<=pLookaside->szMini && pLookaside->nMini > 1 ){ + return miniLookasideAlloc(pLookaside); + } + return lookasideSlotAlloc(pLookaside); +} + +/* +** Free memory previously obtained from sqlite3LookasideAlloc(). +*/ +void sqlite3LookasideFree(Lookaside *pLookaside, void *p){ + assert( sqlite3IsLookaside(pLookaside, p) ); + if( ((u8*)p - (u8*)pLookaside->pStart) % pLookaside->szTrue == 0 ){ + lookasideSlotFree(pLookaside, p); + }else{ + miniLookasideFree(pLookaside, p); + } +} + +/* +** Return the size of a memory allocation previously obtained from +** sqlite3LookasideAlloc(). +*/ +int sqlite3LookasideSize(Lookaside *pLookaside, void *p){ + assert(sqlite3IsLookaside(pLookaside, p)); + +# ifndef SQLITE_OMIT_MINI_LOOKASIDE + if( ((u8*)p - (u8*)pLookaside->pStart) % pLookaside->szTrue != 0 ){ + return pLookaside->szMini; + }else +#endif /* SQLITE_OMIT_MINI_LOOKASIDE */ + { + return pLookaside->szTrue; + } +} + +#endif /* !SQLITE_OMIT_LOOKASIDE */ ADDED src/lookaside.h Index: src/lookaside.h ================================================================== --- /dev/null +++ src/lookaside.h @@ -0,0 +1,89 @@ +/* +** 2019-10-02 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the lookaside allocator +** +** This header defines the interface to the lookaside allocator. +** The lookaside allocator implements a two-size memory allocator using a +** buffer provided at initialization-time to exploit the fact that 75% of +** SQLite's allocations are <=128B. +*/ +#ifndef SQLITE_LOOKASIDE_H +#define SQLITE_LOOKASIDE_H + +/* +** Count the number of slots of lookaside memory that are outstanding +*/ +int sqlite3LookasideUsed(Lookaside *pLookaside, int *pHighwater); + +void sqlite3LookasideResetUsed(Lookaside *pLookaside); + +#define sqlite3LookasideDisable(pLookaside) do{(pLookaside)->bDisable++;\ + (pLookaside)->sz=0;}while(0) +#define sqlite3LookasideEnable(pLookaside) do{(pLookaside)->bDisable--;\ + (pLookaside)->sz=(pLookaside)->bDisable?0:(pLookaside)->szTrue;} while(0) +#define sqlite3LookasideEnableCnt(pLookaside, CNT) do{(pLookaside)->bDisable -= (CNT);\ + (pLookaside)->sz=(pLookaside)->bDisable?0:(pLookaside)->szTrue;} while(0) +#define sqlite3LookasideDisabled(pLookaside) ((pLookaside)->bDisable) + +# ifndef SQLITE_OMIT_LOOKASIDE + +/* +** Set up a lookaside allocator. +** Returns SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** If pStart is NULL the space for the lookaside memory is obtained from +** sqlite3_malloc(). If pStart is not NULL then it is sz*cnt bytes of memory +** to use for the lookaside memory. +*/ +int sqlite3LookasideOpen( + void *pBuf, /* NULL or sz*cnt bytes of memory */ + int sz, /* Number of bytes in each lookaside slot */ + int cnt, /* Number of slots */ + Lookaside *pLookaside /* Preallocated space for the Lookaside */ +); + +/* Reset and close the lookaside object */ +void sqlite3LookasideClose(Lookaside *pLookaside); + +/* +** Returns TRUE if p is a lookaside memory allocation from db +*/ +int sqlite3IsLookaside(Lookaside *pLookaside, void *p); + +/* +** Returns a pointer to a region at least n bytes in size, or NULL if the +** lookaside allocator has exhausted its available memory. +*/ +void *sqlite3LookasideAlloc(Lookaside *pLookaside, u64 n); + +/* +** Free memory previously obtained from sqlite3LookasideAlloc(). +*/ +void sqlite3LookasideFree(Lookaside *pLookaside, void *p); + +/* +** Return the size of a memory allocation previously obtained from +** sqlite3LookasideAlloc(). +*/ +int sqlite3LookasideSize(Lookaside *pLookaside, void *p); + +# else +# define sqlite3LookasideOpen(A, B, C, D) SQLITE_OK +# define sqlite3LookasideClose(A) +# define sqlite3IsLookaside(A, B) 0 +# define sqlite3LookasideAlloc(A, B) 0 +# define sqlite3LookasideFree(A, B) assert(0); +# define sqlite3LookasideSize(A, B) -1 +# endif + +#endif /* SQLITE_LOOKASIDE_H */ Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -13,10 +13,11 @@ ** implement the programmer interface to the library. Routines in ** other files are for internal use by SQLite and should not be ** accessed by users of the library. */ #include "sqliteInt.h" +#include "lookaside.h" #ifdef SQLITE_ENABLE_FTS3 # include "fts3.h" #endif #ifdef SQLITE_ENABLE_RTREE @@ -667,83 +668,10 @@ } va_end(ap); return rc; } -/* -** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. -** If lookaside is already active, return SQLITE_BUSY. -** -** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. -*/ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ -#ifndef SQLITE_OMIT_LOOKASIDE - void *pStart; - - if( sqlite3LookasideUsed(db,0)>0 ){ - return SQLITE_BUSY; - } - /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for - ** both at the same time. - */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } - /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. - */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ - sz = 0; - pStart = 0; - }else if( pBuf==0 ){ - sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*(sqlite3_int64)cnt ); /* IMP: R-61949-35727 */ - sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; - }else{ - pStart = pBuf; - } - db->lookaside.pStart = pStart; - db->lookaside.pInit = 0; - db->lookaside.pFree = 0; - db->lookaside.sz = (u16)sz; - db->lookaside.szTrue = (u16)sz; - if( pStart ){ - int i; - LookasideSlot *p; - assert( sz > (int)sizeof(LookasideSlot*) ); - db->lookaside.nSlot = cnt; - p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pInit; - db->lookaside.pInit = p; - p = (LookasideSlot*)&((u8*)p)[sz]; - } - db->lookaside.pEnd = p; - db->lookaside.bDisable = 0; - db->lookaside.bMalloced = pBuf==0 ?1:0; - }else{ - db->lookaside.pStart = db; - db->lookaside.pEnd = db; - db->lookaside.bDisable = 1; - db->lookaside.sz = 0; - db->lookaside.bMalloced = 0; - db->lookaside.nSlot = 0; - } -#endif /* SQLITE_OMIT_LOOKASIDE */ - return SQLITE_OK; -} - /* ** Return the mutex associated with a database connection. */ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR @@ -826,11 +754,11 @@ } case SQLITE_DBCONFIG_LOOKASIDE: { void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ - rc = setupLookaside(db, pBuf, sz, cnt); + rc = sqlite3LookasideOpen(pBuf, sz, cnt, &db->lookaside); break; } default: { static const struct { int op; /* The opcode */ @@ -1262,14 +1190,11 @@ */ sqlite3DbFree(db, db->aDb[1].pSchema); sqlite3_mutex_leave(db->mutex); db->magic = SQLITE_MAGIC_CLOSED; sqlite3_mutex_free(db->mutex); - assert( sqlite3LookasideUsed(db,0)==0 ); - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } + sqlite3LookasideClose(&db->lookaside); sqlite3_free(db); } /* ** Rollback all database files. If tripCode is not SQLITE_OK, then @@ -3063,12 +2988,11 @@ sqlite3_mutex_enter(db->mutex); db->errMask = 0xff; db->nDb = 2; db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; - db->lookaside.bDisable = 1; - db->lookaside.sz = 0; + sqlite3LookasideDisable(&db->lookaside); assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; @@ -3321,12 +3245,12 @@ #endif if( rc ) sqlite3Error(db, rc); /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); + sqlite3LookasideOpen(0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside, &db->lookaside); sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: if( db ){ Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -11,10 +11,11 @@ ************************************************************************* ** ** Memory allocation functions used throughout sqlite. */ #include "sqliteInt.h" +#include "lookaside.h" #include /* ** Attempt to release up to n bytes of non-essential memory currently ** held by SQLite. An example of non-essential memory is memory used to @@ -262,21 +263,10 @@ if( sqlite3_initialize() ) return 0; #endif return sqlite3Malloc(n); } -/* -** TRUE if p is a lookaside memory allocation from db -*/ -#ifndef SQLITE_OMIT_LOOKASIDE -static int isLookaside(sqlite3 *db, void *p){ - return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); -} -#else -#define isLookaside(A,B) 0 -#endif - /* ** Return the size of a memory allocation previously obtained from ** sqlite3Malloc() or sqlite3_malloc(). */ int sqlite3MallocSize(void *p){ @@ -283,11 +273,11 @@ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return sqlite3GlobalConfig.m.xSize(p); } int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( p!=0 ); - if( db==0 || !isLookaside(db,p) ){ + if( db==0 || !sqlite3IsLookaside(&db->lookaside,p) ){ #ifdef SQLITE_DEBUG if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); }else{ @@ -296,11 +286,11 @@ } #endif return sqlite3GlobalConfig.m.xSize(p); }else{ assert( sqlite3_mutex_held(db->mutex) ); - return db->lookaside.szTrue; + return sqlite3LookasideSize(&db->lookaside, p); } } sqlite3_uint64 sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -344,18 +334,12 @@ if( db ){ if( db->pnBytesFreed ){ measureAllocationSize(db, p); return; } - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; -#ifdef SQLITE_DEBUG - /* Trash all content in the buffer being freed */ - memset(p, 0xaa, db->lookaside.szTrue); -#endif - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; + if( sqlite3IsLookaside(&db->lookaside, p) ){ + sqlite3LookasideFree(&db->lookaside, p); return; } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -466,14 +450,15 @@ ** slower case when the allocation cannot be fulfilled using lookaside. */ static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ void *p; assert( db!=0 ); + if( db->mallocFailed ){ return 0; } p = sqlite3Malloc(n); if( !p ) sqlite3OomFault(db); sqlite3MemdebugSetType(p, - (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); + sqlite3LookasideDisabled(&db->lookaside) ? MEMTYPE_HEAP : MEMTYPE_LOOKASIDE); return p; } /* ** Allocate memory, either lookaside (if possible) or heap. @@ -503,40 +488,24 @@ p = sqlite3Malloc(n); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); return p; } void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ -#ifndef SQLITE_OMIT_LOOKASIDE - LookasideSlot *pBuf; assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( db->pnBytesFreed==0 ); - if( n>db->lookaside.sz ){ - if( db->lookaside.bDisable ){ - return db->mallocFailed ? 0 : dbMallocRawFinish(db, n); - } - db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)!=0 ){ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.anStat[0]++; - return (void*)pBuf; - }else if( (pBuf = db->lookaside.pInit)!=0 ){ - db->lookaside.pInit = pBuf->pNext; - db->lookaside.anStat[0]++; - return (void*)pBuf; - }else{ - db->lookaside.anStat[2]++; - } +#ifndef SQLITE_OMIT_LOOKASIDE + return sqlite3LookasideAlloc(&db->lookaside, n) ?: dbMallocRawFinish(db, n); #else assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( db->pnBytesFreed==0 ); if( db->mallocFailed ){ return 0; } -#endif return dbMallocRawFinish(db, n); +#endif } /* Forward declaration */ static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); @@ -546,22 +515,22 @@ */ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ assert( db!=0 ); if( p==0 ) return sqlite3DbMallocRawNN(db, n); assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db,p) && n<=db->lookaside.szTrue ) return p; + if( sqlite3IsLookaside(&db->lookaside,p) && n<=sqlite3LookasideSize(&db->lookaside, p) ) return p; return dbReallocFinish(db, p, n); } static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ void *pNew = 0; assert( db!=0 ); assert( p!=0 ); if( db->mallocFailed==0 ){ - if( isLookaside(db, p) ){ + if( sqlite3IsLookaside(&db->lookaside,p) ){ pNew = sqlite3DbMallocRawNN(db, n); if( pNew ){ - memcpy(pNew, p, db->lookaside.szTrue); + memcpy(pNew, p, sqlite3LookasideSize(&db->lookaside, p)); sqlite3DbFree(db, p); } }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -568,12 +537,12 @@ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc64(p, n); if( !pNew ){ sqlite3OomFault(db); } - sqlite3MemdebugSetType(pNew, - (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + sqlite3MemdebugSetType(p, + sqlite3LookasideDisabled(&db->lookaside) ? MEMTYPE_HEAP : MEMTYPE_LOOKASIDE); } } return pNew; } @@ -646,21 +615,20 @@ *pz = sqlite3DbStrDup(db, zNew); } /* ** Call this routine to record the fact that an OOM (out-of-memory) error -** has happened. This routine will set db->mallocFailed, and also -** temporarily disable the lookaside memory allocator and interrupt +** has happened. This routine will set db->mallocFailed and interrupt ** any running VDBEs. */ void sqlite3OomFault(sqlite3 *db){ if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ db->mallocFailed = 1; if( db->nVdbeExec>0 ){ db->u1.isInterrupted = 1; } - DisableLookaside; + sqlite3LookasideDisable(&db->lookaside);; if( db->pParse ){ db->pParse->rc = SQLITE_NOMEM_BKPT; } } } @@ -674,11 +642,11 @@ */ void sqlite3OomClear(sqlite3 *db){ if( db->mallocFailed && db->nVdbeExec==0 ){ db->mallocFailed = 0; db->u1.isInterrupted = 0; - assert( db->lookaside.bDisable>0 ); + assert( sqlite3LookasideDisabled(&db->lookaside) ); EnableLookaside; } } /* Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -49,10 +49,11 @@ // The following text is included near the beginning of the C source // code file that implements the parser. // %include { #include "sqliteInt.h" +#include "lookaside.h" /* ** Disable all error recovery processing in the parser push-down ** automaton. */ @@ -106,11 +107,11 @@ ** shared across database connections. */ static void disableLookaside(Parse *pParse){ sqlite3 *db = pParse->db; pParse->disableLookaside++; - DisableLookaside; + sqlite3LookasideDisable(&db->lookaside); } } // end %include // Input is a single SQL command Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -12,10 +12,11 @@ ** This file contains the implementation of the sqlite3_prepare() ** interface, and routines that contribute to loading the database schema ** from disk. */ #include "sqliteInt.h" +#include "lookaside.h" /* ** Fill the InitData structure with an error message that indicates ** that the database is corrupt. */ @@ -536,13 +537,12 @@ void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); if( db ){ - assert( db->lookaside.bDisable >= pParse->disableLookaside ); - db->lookaside.bDisable -= pParse->disableLookaside; - db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; + assert( sqlite3LookasideDisabled(&db->lookaside) >= pParse->disableLookaside ); + sqlite3LookasideEnableCnt(&db->lookaside, pParse->disableLookaside); } pParse->disableLookaside = 0; } /* @@ -572,11 +572,11 @@ /* For a long-term use prepared statement avoid the use of ** lookaside memory. */ if( prepFlags & SQLITE_PREPARE_PERSISTENT ){ sParse.disableLookaside++; - DisableLookaside; + sqlite3LookasideDisable(&db->lookaside); } sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -11,10 +11,11 @@ ************************************************************************* ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. */ #include "sqliteInt.h" +#include "lookaside.h" /* ** Trace output macros */ #if SELECTTRACE_ENABLED Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1287,13 +1287,24 @@ u32 anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ LookasideSlot *pInit; /* List of buffers not previously used */ LookasideSlot *pFree; /* List of available buffers */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ +#ifndef SQLITE_OMIT_MINILOOKASIDE + LookasideSlot *pMini; /* List of buffers used by mini allocator */ + u16 szMini; /* Size of each mini-allocator buffer in bytes */ + u16 nMini; /* Number of mini-allocator buffers per slot */ +#endif /* SQLITE_OMIT_MINILOOKASIDE */ }; struct LookasideSlot { - LookasideSlot *pNext; /* Next buffer in the list of free buffers */ + LookasideSlot *pNext; /* Next buffer in the list of buffers */ +#ifndef SQLITE_OMIT_MINILOOKASIDE + /* The members below are only valid for slots inside the mini-allocator */ + LookasideSlot *pPrev; /* Previous partially-used buffer in the list */ + u16 nAlloc; /* Number of sub-allocations ever returned from this slot */ + LookasideSlot *pFree; /* List of freed sub-allocations */ +#endif /* SQLITE_OMIT_MINILOOKASIDE */ }; #define DisableLookaside db->lookaside.bDisable++;db->lookaside.sz=0 #define EnableLookaside db->lookaside.bDisable--;\ db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue @@ -3823,11 +3834,10 @@ sqlite3_int64 sqlite3StatusValue(int); void sqlite3StatusUp(int, int); void sqlite3StatusDown(int, int); void sqlite3StatusHighwater(int, int); -int sqlite3LookasideUsed(sqlite3*,int*); /* Access to mutexes used by sqlite3_status() */ sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); Index: src/status.c ================================================================== --- src/status.c +++ src/status.c @@ -12,10 +12,11 @@ ** ** This module implements the sqlite3_status() interface and related ** functionality. */ #include "sqliteInt.h" +#include "lookaside.h" #include "vdbeInt.h" /* ** Variables in which to record status information. */ @@ -168,32 +169,10 @@ *pHighwater = (int)iHwtr; } return rc; } -/* -** Return the number of LookasideSlot elements on the linked list -*/ -static u32 countLookasideSlots(LookasideSlot *p){ - u32 cnt = 0; - while( p ){ - p = p->pNext; - cnt++; - } - return cnt; -} - -/* -** Count the number of slots of lookaside memory that are outstanding -*/ -int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ - u32 nInit = countLookasideSlots(db->lookaside.pInit); - u32 nFree = countLookasideSlots(db->lookaside.pFree); - if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; - return db->lookaside.nSlot - (nInit+nFree); -} - /* ** Query status information for a single database connection */ int sqlite3_db_status( sqlite3 *db, /* The database connection whose status is desired */ @@ -209,19 +188,13 @@ } #endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { - *pCurrent = sqlite3LookasideUsed(db, pHighwater); + *pCurrent = sqlite3LookasideUsed(&db->lookaside, pHighwater); if( resetFlag ){ - LookasideSlot *p = db->lookaside.pFree; - if( p ){ - while( p->pNext ) p = p->pNext; - p->pNext = db->lookaside.pInit; - db->lookaside.pInit = db->lookaside.pFree; - db->lookaside.pFree = 0; - } + sqlite3LookasideResetUsed(&db->lookaside); } break; } case SQLITE_DBSTATUS_LOOKASIDE_HIT: Index: tool/mksqlite3c-noext.tcl ================================================================== --- tool/mksqlite3c-noext.tcl +++ tool/mksqlite3c-noext.tcl @@ -96,10 +96,11 @@ btree.h btreeInt.h hash.h hwtime.h keywordhash.h + lookaside.h msvc.h mutex.h opcodes.h os_common.h os_setup.h @@ -281,10 +282,11 @@ status.c date.c os.c fault.c + lookaside.c mem0.c mem1.c mem2.c mem3.c mem5.c Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -101,10 +101,11 @@ fts3_tokenizer.h geopoly.c hash.h hwtime.h keywordhash.h + lookaside.h msvc.h mutex.h opcodes.h os_common.h os_setup.h @@ -298,10 +299,11 @@ status.c date.c os.c fault.c + lookaside.c mem0.c mem1.c mem2.c mem3.c mem5.c Index: tool/mksqlite3internalh.tcl ================================================================== --- tool/mksqlite3internalh.tcl +++ tool/mksqlite3internalh.tcl @@ -56,10 +56,11 @@ btree.h btreeInt.h hash.h hwtime.h keywordhash.h + lookaside.h msvc.h opcodes.h os_common.h os_setup.h os_win.h