Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. This change is based on research by Martijn Blaauw. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
bacfe8cb3e4d3be736c6789b2ce55ef5 |
| User & Date: | dan 2017-06-30 11:44:11 |
| Original Comment: | Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. |
Context
|
2017-06-30
| ||
| 18:12 | Fix some minor typos in lsm1. check-in: 0ef777d7 user: mistachkin tags: trunk | |
| 11:44 | Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. This change is based on research by Martijn Blaauw. check-in: bacfe8cb user: dan tags: trunk | |
|
2017-06-29
| ||
| 21:33 | In the command-line shell, add the -quote option to start up in quote mode. Enhance the ".mode" command so that it reports the current output mode if given no arguments. check-in: 5e3f9ea5 user: drh tags: trunk | |
Changes
Changes to ext/lsm1/lsm_sorted.c.
94 95 96 97 98 99 100 101 102 103 104 105 106 107 .... 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 .... 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 .... 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 .... 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 .... 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 .... 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 .... 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 .... 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 .... 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 .... 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 .... 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 |
#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
#define SEGMENT_BTREE_FLAG 0x0001
#define PGFTR_SKIP_NEXT_FLAG 0x0002
#define PGFTR_SKIP_THIS_FLAG 0x0004
typedef struct SegmentPtr SegmentPtr;
typedef struct Blob Blob;
struct Blob {
lsm_env *pEnv;
void *pData;
int nData;
................................................................................
pLevel->nSplitKey = blob.nData;
lsmFsPageRelease(pPg);
}
*pRc = rc;
}
static void segmentPtrReset(SegmentPtr *pPtr){
lsmFsPageRelease(pPtr->pPg);
pPtr->pPg = 0;
pPtr->nCell = 0;
pPtr->pKey = 0;
pPtr->nKey = 0;
pPtr->pVal = 0;
pPtr->nVal = 0;
pPtr->eType = 0;
pPtr->iCell = 0;
sortedBlobFree(&pPtr->blob1);
sortedBlobFree(&pPtr->blob2);
}
static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){
return (pCsr->flags & CURSOR_READ_SEPARATORS)==0
|| (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]);
}
................................................................................
if( rc!=LSM_OK ) return rc;
if( svFlags && pPtr->pPg ){
int res = sortedKeyCompare(pCsr->pDb->xCmp,
rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey,
pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey
);
if( res<0 ) segmentPtrReset(pPtr);
}
if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){
Segment *pSeg = pPtr->pSeg;
rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg);
if( rc!=LSM_OK ) return rc;
pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE;
................................................................................
if( rc==LSM_OK && pPtr->pPg ){
rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0);
if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){
int res = sortedKeyCompare(pCsr->pDb->xCmp,
rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey,
pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey
);
if( res<0 ) segmentPtrReset(pPtr);
}
}
bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr);
if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){
rc = segmentPtrAdvance(pCsr, pPtr, bLast);
}
................................................................................
ptr.pLevel = pPtr->pLevel;
ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1];
rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr);
if( rc==LSM_OK ){
rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound);
ptr.pPg = 0;
}
segmentPtrReset(&ptr);
}
*piPtr = iOut;
return rc;
}
static int segmentPtrSeek(
................................................................................
#endif
assert( pPtr->nCell>0
|| pPtr->pSeg->nSize==1
|| lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg)
);
if( pPtr->nCell==0 ){
segmentPtrReset(pPtr);
}else{
iMin = 0;
iMax = pPtr->nCell-1;
while( 1 ){
int iTry = (iMin+iMax)/2;
void *pKeyT; int nKeyT; /* Key for cell iTry */
................................................................................
pCsr->eType = pPtr->eType;
rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey);
if( rc==LSM_OK ){
rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal);
}
pCsr->flags |= CURSOR_SEEK_EQ;
}
segmentPtrReset(pPtr);
break;
}
case LSM_SEEK_LE:
if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1);
break;
case LSM_SEEK_GE: {
/* Figure out if we need to 'skip' the pointer forward or not */
................................................................................
/* If the segment-pointer has settled on a key that is smaller than
** the splitkey, invalidate the segment-pointer. */
if( pPtr->pPg ){
res = sortedKeyCompare(pCsr->pDb->xCmp,
rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey,
pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey
);
if( res<0 ) segmentPtrReset(pPtr);
}
if( aPtr[i].pKey ) bHit = 1;
}
if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){
rc = segmentPtrEnd(pCsr, &aPtr[0], 1);
................................................................................
/* Close the tree cursor, if any. */
lsmTreeCursorDestroy(pCsr->apTreeCsr[0]);
lsmTreeCursorDestroy(pCsr->apTreeCsr[1]);
/* Reset the segment pointers */
for(i=0; i<pCsr->nPtr; i++){
segmentPtrReset(&pCsr->aPtr[i]);
}
/* And the b-tree cursor, if any */
btreeCursorFree(pCsr->pBtCsr);
/* Free allocations */
lsmFree(pEnv, pCsr->aPtr);
................................................................................
for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){
rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1);
if( pPtr[iRhs+1].pPg ) bHit = 1;
}
if( bHit==0 && rc==LSM_OK ){
rc = segmentPtrEnd(pCsr, pPtr, 1);
}else{
segmentPtrReset(pPtr);
}
}else{
int bLhs = (pPtr->pSeg==&pLvl->lhs);
assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] );
if( bLhs ){
rc = segmentPtrEnd(pCsr, pPtr, 0);
if( pPtr->pKey ) bHit = 1;
}
for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){
if( bHit ){
segmentPtrReset(&pPtr[iRhs+1]);
}else{
rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]);
}
}
}
i += pLvl->nRight;
}
................................................................................
}
void lsmMCursorReset(MultiCursor *pCsr){
int i;
lsmTreeCursorReset(pCsr->apTreeCsr[0]);
lsmTreeCursorReset(pCsr->apTreeCsr[1]);
for(i=0; i<pCsr->nPtr; i++){
segmentPtrReset(&pCsr->aPtr[i]);
}
pCsr->key.nData = 0;
}
static int treeCursorSeek(
MultiCursor *pCsr,
TreeCursor *pTreeCsr,
................................................................................
if( ptr1.pPg==0 ){
assert( 0 );
}
}
}
}
segmentPtrReset(&ptr1);
segmentPtrReset(&ptr2);
return LSM_OK;
}
/*
** This function is only included in the build if LSM_DEBUG_EXPENSIVE is
** defined. Its only purpose is to evaluate various assert() statements to
** verify that the database is well formed in certain respects.
|
> > > > > > > > > | | | | | | | | | | | | | | | |
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 .... 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 .... 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 .... 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 .... 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 .... 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 .... 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 .... 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 .... 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 .... 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 .... 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 .... 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 |
#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry) #define SEGMENT_BTREE_FLAG 0x0001 #define PGFTR_SKIP_NEXT_FLAG 0x0002 #define PGFTR_SKIP_THIS_FLAG 0x0004 #ifndef LSM_SEGMENTPTR_FREE_THRESHOLD # define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 #endif typedef struct SegmentPtr SegmentPtr; typedef struct Blob Blob; struct Blob { lsm_env *pEnv; void *pData; int nData; ................................................................................ pLevel->nSplitKey = blob.nData; lsmFsPageRelease(pPg); } *pRc = rc; } /* ** Reset a segment cursor. Also free its buffers if they are nThreshold ** bytes or larger in size. */ static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){ lsmFsPageRelease(pPtr->pPg); pPtr->pPg = 0; pPtr->nCell = 0; pPtr->pKey = 0; pPtr->nKey = 0; pPtr->pVal = 0; pPtr->nVal = 0; pPtr->eType = 0; pPtr->iCell = 0; if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1); if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2); } static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){ return (pCsr->flags & CURSOR_READ_SEPARATORS)==0 || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]); } ................................................................................ if( rc!=LSM_OK ) return rc; if( svFlags && pPtr->pPg ){ int res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ Segment *pSeg = pPtr->pSeg; rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); if( rc!=LSM_OK ) return rc; pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; ................................................................................ if( rc==LSM_OK && pPtr->pPg ){ rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ int res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } } bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ rc = segmentPtrAdvance(pCsr, pPtr, bLast); } ................................................................................ ptr.pLevel = pPtr->pLevel; ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); if( rc==LSM_OK ){ rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); ptr.pPg = 0; } segmentPtrReset(&ptr, 0); } *piPtr = iOut; return rc; } static int segmentPtrSeek( ................................................................................ #endif assert( pPtr->nCell>0 || pPtr->pSeg->nSize==1 || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) ); if( pPtr->nCell==0 ){ segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); }else{ iMin = 0; iMax = pPtr->nCell-1; while( 1 ){ int iTry = (iMin+iMax)/2; void *pKeyT; int nKeyT; /* Key for cell iTry */ ................................................................................ pCsr->eType = pPtr->eType; rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); if( rc==LSM_OK ){ rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); } pCsr->flags |= CURSOR_SEEK_EQ; } segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); break; } case LSM_SEEK_LE: if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1); break; case LSM_SEEK_GE: { /* Figure out if we need to 'skip' the pointer forward or not */ ................................................................................ /* If the segment-pointer has settled on a key that is smaller than ** the splitkey, invalidate the segment-pointer. */ if( pPtr->pPg ){ res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } if( aPtr[i].pKey ) bHit = 1; } if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ rc = segmentPtrEnd(pCsr, &aPtr[0], 1); ................................................................................ /* Close the tree cursor, if any. */ lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); /* Reset the segment pointers */ for(i=0; i<pCsr->nPtr; i++){ segmentPtrReset(&pCsr->aPtr[i], 0); } /* And the b-tree cursor, if any */ btreeCursorFree(pCsr->pBtCsr); /* Free allocations */ lsmFree(pEnv, pCsr->aPtr); ................................................................................ for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); if( pPtr[iRhs+1].pPg ) bHit = 1; } if( bHit==0 && rc==LSM_OK ){ rc = segmentPtrEnd(pCsr, pPtr, 1); }else{ segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } }else{ int bLhs = (pPtr->pSeg==&pLvl->lhs); assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); if( bLhs ){ rc = segmentPtrEnd(pCsr, pPtr, 0); if( pPtr->pKey ) bHit = 1; } for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ if( bHit ){ segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD); }else{ rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); } } } i += pLvl->nRight; } ................................................................................ } void lsmMCursorReset(MultiCursor *pCsr){ int i; lsmTreeCursorReset(pCsr->apTreeCsr[0]); lsmTreeCursorReset(pCsr->apTreeCsr[1]); for(i=0; i<pCsr->nPtr; i++){ segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); } pCsr->key.nData = 0; } static int treeCursorSeek( MultiCursor *pCsr, TreeCursor *pTreeCsr, ................................................................................ if( ptr1.pPg==0 ){ assert( 0 ); } } } } segmentPtrReset(&ptr1, 0); segmentPtrReset(&ptr2, 0); return LSM_OK; } /* ** This function is only included in the build if LSM_DEBUG_EXPENSIVE is ** defined. Its only purpose is to evaluate various assert() statements to ** verify that the database is well formed in certain respects. |