Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Truncate the database file when the number of connections drops from one to zero. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
dc4fa9259613d17dbb6dc4c87dfc3c54 |
User & Date: | dan 2013-01-18 10:46:03.567 |
Context
2013-01-18
| ||
15:58 | Display the 'age' of each level in lsmview. check-in: 33cc91541c user: dan tags: trunk | |
10:46 | Truncate the database file when the number of connections drops from one to zero. check-in: dc4fa92596 user: dan tags: trunk | |
2013-01-17
| ||
19:13 | Fix an lsm bug causing it to choose the wrong block to reuse. check-in: 2ff461b422 user: dan tags: trunk | |
Changes
Changes to src/lsmInt.h.
︙ | ︙ | |||
636 637 638 639 640 641 642 | int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); /* Creating, populating, gobbling and deleting sorted runs. */ void lsmFsGobble(lsm_db *, Segment *, Pgno *, int); int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); int lsmFsSortedFinish(FileSystem *, Segment *); | | | 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 | int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); /* Creating, populating, gobbling and deleting sorted runs. */ void lsmFsGobble(lsm_db *, Segment *, Pgno *, int); int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); int lsmFsSortedFinish(FileSystem *, Segment *); int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); /* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ lsm_env *lsmFsEnv(FileSystem *); lsm_env *lsmPageEnv(Page *); FileSystem *lsmPageFS(Page *); |
︙ | ︙ |
Changes to src/lsm_ckpt.c.
︙ | ︙ | |||
882 883 884 885 886 887 888 | int lsmCheckpointLoadWorker(lsm_db *pDb){ int rc; ShmHeader *pShm = pDb->pShmhdr; int nInt1; int nInt2; | | > | > > > > | 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 | int lsmCheckpointLoadWorker(lsm_db *pDb){ int rc; ShmHeader *pShm = pDb->pShmhdr; int nInt1; int nInt2; /* Must be holding the WORKER lock to do this. Or DMS2. */ assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) ); /* Check that the two snapshots match. If not, repair them. */ nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ if( ckptChecksumOk(pShm->aSnap1) ){ memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); }else if( ckptChecksumOk(pShm->aSnap2) ){ memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); }else{ return LSM_PROTOCOL; } } rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); return rc; } int lsmCheckpointDeserialize( lsm_db *pDb, int bInclFreelist, /* If true, deserialize free-list */ |
︙ | ︙ |
Changes to src/lsm_file.c.
︙ | ︙ | |||
1547 1548 1549 1550 1551 1552 1553 | } rc = fsPageGet(pFS, iPg, 0, ppNext, 0); } return rc; } | | | > > > > > > > > > > > | | | | > | 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 | } rc = fsPageGet(pFS, iPg, 0, ppNext, 0); } return rc; } static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){ int i; Pgno *aiAppend = pFS->pDb->pWorker->aiAppend; u32 iRet = 0; for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ if( (iRet = aiAppend[i]) ){ if( pLvl ){ int iBlk = fsPageToBlock(pFS, iRet); int j; for(j=0; iRet && j<pLvl->nRight; j++){ if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ iRet = 0; } } } if( iRet ) aiAppend[i] = 0; } } return iRet; } /* ** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and ** return a pointer to it. The page is writable until either ** lsmFsPagePersist() is called on it or the ref-count drops to zero. */ int lsmFsSortedAppend( FileSystem *pFS, Snapshot *pSnapshot, Level *pLvl, int bDefer, Page **ppOut ){ int rc = LSM_OK; Page *pPg = 0; *ppOut = 0; int iApp = 0; int iNext = 0; Segment *p = &pLvl->lhs; int iPrev = p->iLastPg; if( pFS->pCompress || bDefer ){ /* In compressed database mode the page is not assigned a page number ** or location in the database file at this point. This will be done ** by the lsmFsPagePersist() call. */ rc = fsPageBuffer(pFS, 1, &pPg); |
︙ | ︙ | |||
1596 1597 1598 1599 1600 1601 1602 | if( pFS->pCompress==0 ) pPg->nData -= 4; pPg->nRef = 1; pFS->nOut++; } }else{ if( iPrev==0 ){ | | | 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 | if( pFS->pCompress==0 ) pPg->nData -= 4; pPg->nRef = 1; pFS->nOut++; } }else{ if( iPrev==0 ){ iApp = findAppendPoint(pFS, pLvl); }else if( fsIsLast(pFS, iPrev) ){ int iNext; rc = fsBlockNext(pFS, fsPageToBlock(pFS, iPrev), &iNext); if( rc!=LSM_OK ) return rc; iApp = fsFirstPageOnBlock(pFS, iNext); }else{ iApp = iPrev + 1; |
︙ | ︙ | |||
1838 1839 1840 1841 1842 1843 1844 | int nWrite; Pgno iLastOnBlock; Pgno iApp = pSeg->iLastPg+1; /* If this is the first data written into the segment, find an append-point ** or allocate a new block. */ if( iApp==1 ){ | | | 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 | int nWrite; Pgno iLastOnBlock; Pgno iApp = pSeg->iLastPg+1; /* If this is the first data written into the segment, find an append-point ** or allocate a new block. */ if( iApp==1 ){ pSeg->iFirst = iApp = findAppendPoint(pFS, 0); if( iApp==0 ){ int iBlk; rc = lsmBlockAllocate(pFS->pDb, &iBlk); pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); } } iRet = iApp; |
︙ | ︙ |
Changes to src/lsm_shared.c.
︙ | ︙ | |||
146 147 148 149 150 151 152 153 154 155 156 157 158 159 | /* Free the array of shm pointers */ lsmFree(pEnv, p->apShmChunk); /* Free the memory allocated for the Database struct itself */ lsmFree(pEnv, p); } } static void doDbDisconnect(lsm_db *pDb){ int rc; /* Block for an exclusive lock on DMS1. This lock serializes all calls ** to doDbConnect() and doDbDisconnect() across all processes. */ rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 | /* Free the array of shm pointers */ lsmFree(pEnv, p->apShmChunk); /* Free the memory allocated for the Database struct itself */ lsmFree(pEnv, p); } } typedef struct DbTruncateCtx DbTruncateCtx; struct DbTruncateCtx { int nBlock; i64 iInUse; }; static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ DbTruncateCtx *p = (DbTruncateCtx *)pCtx; if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; p->nBlock--; return 0; } static int dbTruncate(lsm_db *pDb, i64 iInUse){ int rc; int i; DbTruncateCtx ctx; assert( pDb->pWorker ); ctx.nBlock = pDb->pWorker->nBlock; ctx.iInUse = iInUse; rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ rc = freelistAppend(pDb, i, -1); } if( rc==LSM_OK ){ #ifdef LSM_LOG_FREELIST if( ctx.nBlock!=pDb->pWorker->nBlock ){ lsmLogMessage(pDb, 0, "dbTruncate(): truncated db to %d blocks",ctx.nBlock ); } #endif pDb->pWorker->nBlock = ctx.nBlock; } return rc; } /* ** This function is called during database shutdown (when the number of ** connections drops from one to zero). It truncates the database file ** to as small a size as possible without truncating away any blocks that ** contain data. */ static int dbTruncateFile(lsm_db *pDb){ int rc; assert( pDb->pWorker==0 ); assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) ); rc = lsmCheckpointLoadWorker(pDb); if( rc==LSM_OK ){ DbTruncateCtx ctx; /* Walk the database free-block-list in reverse order. Set ctx.nBlock ** to the block number of the last block in the database that actually ** contains data. */ ctx.nBlock = pDb->pWorker->nBlock; ctx.iInUse = -1; rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); /* If the last block that contains data is not already the last block in ** the database file, truncate the database file so that it is. */ if( rc==LSM_OK && ctx.nBlock!=pDb->pWorker->nBlock ){ rc = lsmFsTruncateDb( pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) ); } } lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); pDb->pWorker = 0; return rc; } static void doDbDisconnect(lsm_db *pDb){ int rc; /* Block for an exclusive lock on DMS1. This lock serializes all calls ** to doDbConnect() and doDbDisconnect() across all processes. */ rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); |
︙ | ︙ | |||
179 180 181 182 183 184 185 | } /* Write a checkpoint to disk. */ if( rc==LSM_OK ){ rc = lsmCheckpointWrite(pDb, 1, 0); } | | > > | 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 | } /* Write a checkpoint to disk. */ if( rc==LSM_OK ){ rc = lsmCheckpointWrite(pDb, 1, 0); } /* If the checkpoint was written successfully, delete the log file ** and, if possible, truncate the database file. */ if( rc==LSM_OK ){ Database *p = pDb->pDatabase; dbTruncateFile(pDb); lsmFsCloseAndDeleteLog(pDb->pFS); if( p->pFile ) lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); } } } lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); |
︙ | ︙ | |||
538 539 540 541 542 543 544 | } } } return rc; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 618 619 620 621 622 623 624 625 626 627 628 629 630 631 | } } } return rc; } typedef struct FindFreeblockCtx FindFreeblockCtx; struct FindFreeblockCtx { i64 iInUse; int iRet; }; |
︙ | ︙ | |||
819 820 821 822 823 824 825 | /* Attempt to take the WORKER lock */ rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); /* Deserialize the current worker snapshot */ if( rc==LSM_OK ){ rc = lsmCheckpointLoadWorker(pDb); | < | 859 860 861 862 863 864 865 866 867 868 869 870 871 872 | /* Attempt to take the WORKER lock */ rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); /* Deserialize the current worker snapshot */ if( rc==LSM_OK ){ rc = lsmCheckpointLoadWorker(pDb); } return rc; } void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ if( p ){ lsmSortedFreeLevel(pEnv, p->pLevel); |
︙ | ︙ |
Changes to src/lsm_sorted.c.
︙ | ︙ | |||
3104 3105 3106 3107 3108 3109 3110 | ** to the way the file.c module works some (the first and last in each block) ** are 4 bytes smaller than the others. */ static int mergeWorkerMoveHierarchy( MergeWorker *pMW, /* Merge worker */ int bSep /* True for separators run */ ){ | < < < | | 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 | ** to the way the file.c module works some (the first and last in each block) ** are 4 bytes smaller than the others. */ static int mergeWorkerMoveHierarchy( MergeWorker *pMW, /* Merge worker */ int bSep /* True for separators run */ ){ lsm_db *pDb = pMW->pDb; /* Database handle */ int rc = LSM_OK; /* Return code */ int i; Page **apHier = pMW->hier.apHier; int nHier = pMW->hier.nHier; for(i=0; rc==LSM_OK && i<nHier; i++){ Page *pNew = 0; rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &pNew); assert( rc==LSM_OK ); if( rc==LSM_OK ){ u8 *a1; int n1; u8 *a2; int n2; a1 = fsPageData(pNew, &n1); |
︙ | ︙ | |||
3293 3294 3295 3296 3297 3298 3299 | MergeWorker *pMW, u8 eType, Pgno iPtr, Pgno iKeyPg, void *pKey, int nKey ){ | < | 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 | MergeWorker *pMW, u8 eType, Pgno iPtr, Pgno iKeyPg, void *pKey, int nKey ){ Hierarchy *p = &pMW->hier; lsm_db *pDb = pMW->pDb; /* Database handle */ int rc = LSM_OK; /* Return Code */ int iLevel; /* Level of b-tree hierachy to write to */ int nData; /* Size of aData[] in bytes */ u8 *aData; /* Page data for level iLevel */ int iOff; /* Offset on b-tree page to write record to */ |
︙ | ︙ | |||
3361 3362 3363 3364 3365 3366 3367 | } } /* Allocate a new page for apHier[iLevel]. */ p->apHier[iLevel] = 0; if( rc==LSM_OK ){ rc = lsmFsSortedAppend( | | | 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 | } } /* Allocate a new page for apHier[iLevel]. */ p->apHier[iLevel] = 0; if( rc==LSM_OK ){ rc = lsmFsSortedAppend( pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] ); } if( rc!=LSM_OK ) return rc; aData = fsPageData(p->apHier[iLevel], &nData); memset(aData, 0, nData); lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], SEGMENT_BTREE_FLAG); |
︙ | ︙ | |||
3561 3562 3563 3564 3565 3566 3567 | Pgno iFPtr /* Pointer value for footer of new page */ ){ int rc = LSM_OK; /* Return code */ Page *pNext = 0; /* New page appended to run */ lsm_db *pDb = pMW->pDb; /* Database handle */ Segment *pSeg; /* Run to append to */ | < | | | 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 | Pgno iFPtr /* Pointer value for footer of new page */ ){ int rc = LSM_OK; /* Return code */ Page *pNext = 0; /* New page appended to run */ lsm_db *pDb = pMW->pDb; /* Database handle */ Segment *pSeg; /* Run to append to */ rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); if( rc==LSM_OK ){ u8 *aData; /* Data buffer belonging to page pNext */ int nData; /* Size of aData[] in bytes */ rc = mergeWorkerPersistAndRelease(pMW); |
︙ | ︙ | |||
4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 | while( rc==LSM_OK && sortedDbIsFull(pDb) ){ rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); } if( rc==LSM_OK ){ rc = sortedNewToplevel(pDb, TREE_BOTH, 0); } lsmFinishWork(pDb, 1, &rc); return rc; } /* ** Return a string representation of the segment passed as the only argument. ** Space for the returned string is allocated using lsmMalloc(), and should | > | 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 | while( rc==LSM_OK && sortedDbIsFull(pDb) ){ rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); } if( rc==LSM_OK ){ rc = sortedNewToplevel(pDb, TREE_BOTH, 0); } lsmFinishWork(pDb, 1, &rc); return rc; } /* ** Return a string representation of the segment passed as the only argument. ** Space for the returned string is allocated using lsmMalloc(), and should |
︙ | ︙ |