Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. This change is based on research by Martijn Blaauw. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
bacfe8cb3e4d3be736c6789b2ce55ef5 |
User & Date: | dan 2017-06-30 11:44:11.712 |
Original Comment: | Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. |
Context
2017-06-30
| ||
18:12 | Fix some minor typos in lsm1. (check-in: 0ef777d747 user: mistachkin tags: trunk) | |
11:44 | Avoid constantly freeing and reallocing small internal buffers associated with LSM cursors. Instead, allow them to persist for the lifetime of the cursor if they are LSM_SEGMENTPTR_FREE_THRESHOLD (default 1024) bytes or smaller. This change is based on research by Martijn Blaauw. (check-in: bacfe8cb3e user: dan tags: trunk) | |
2017-06-29
| ||
21:33 | In the command-line shell, add the -quote option to start up in quote mode. Enhance the ".mode" command so that it reports the current output mode if given no arguments. (check-in: 5e3f9ea5c4 user: drh tags: trunk) | |
Changes
Changes to ext/lsm1/lsm_sorted.c.
︙ | ︙ | |||
94 95 96 97 98 99 100 101 102 103 104 105 106 107 | #define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry) #define SEGMENT_BTREE_FLAG 0x0001 #define PGFTR_SKIP_NEXT_FLAG 0x0002 #define PGFTR_SKIP_THIS_FLAG 0x0004 typedef struct SegmentPtr SegmentPtr; typedef struct Blob Blob; struct Blob { lsm_env *pEnv; void *pData; int nData; | > > > > > | 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | #define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry) #define SEGMENT_BTREE_FLAG 0x0001 #define PGFTR_SKIP_NEXT_FLAG 0x0002 #define PGFTR_SKIP_THIS_FLAG 0x0004 #ifndef LSM_SEGMENTPTR_FREE_THRESHOLD # define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 #endif typedef struct SegmentPtr SegmentPtr; typedef struct Blob Blob; struct Blob { lsm_env *pEnv; void *pData; int nData; |
︙ | ︙ | |||
1144 1145 1146 1147 1148 1149 1150 | pLevel->nSplitKey = blob.nData; lsmFsPageRelease(pPg); } *pRc = rc; } | > > > > | | | | 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 | pLevel->nSplitKey = blob.nData; lsmFsPageRelease(pPg); } *pRc = rc; } /* ** Reset a segment cursor. Also free its buffers if they are nThreshold ** bytes or larger in size. */ static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){ lsmFsPageRelease(pPtr->pPg); pPtr->pPg = 0; pPtr->nCell = 0; pPtr->pKey = 0; pPtr->nKey = 0; pPtr->pVal = 0; pPtr->nVal = 0; pPtr->eType = 0; pPtr->iCell = 0; if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1); if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2); } static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){ return (pCsr->flags & CURSOR_READ_SEPARATORS)==0 || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]); } |
︙ | ︙ | |||
1202 1203 1204 1205 1206 1207 1208 | if( rc!=LSM_OK ) return rc; if( svFlags && pPtr->pPg ){ int res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); | | | 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 | if( rc!=LSM_OK ) return rc; if( svFlags && pPtr->pPg ){ int res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ Segment *pSeg = pPtr->pSeg; rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); if( rc!=LSM_OK ) return rc; pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; |
︙ | ︙ | |||
1271 1272 1273 1274 1275 1276 1277 | if( rc==LSM_OK && pPtr->pPg ){ rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ int res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); | | | 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 | if( rc==LSM_OK && pPtr->pPg ){ rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ int res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } } bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ rc = segmentPtrAdvance(pCsr, pPtr, bLast); } |
︙ | ︙ | |||
1607 1608 1609 1610 1611 1612 1613 | ptr.pLevel = pPtr->pLevel; ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); if( rc==LSM_OK ){ rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); ptr.pPg = 0; } | | | 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 | ptr.pLevel = pPtr->pLevel; ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); if( rc==LSM_OK ){ rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); ptr.pPg = 0; } segmentPtrReset(&ptr, 0); } *piPtr = iOut; return rc; } static int segmentPtrSeek( |
︙ | ︙ | |||
1654 1655 1656 1657 1658 1659 1660 | #endif assert( pPtr->nCell>0 || pPtr->pSeg->nSize==1 || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) ); if( pPtr->nCell==0 ){ | | | 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 | #endif assert( pPtr->nCell>0 || pPtr->pSeg->nSize==1 || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) ); if( pPtr->nCell==0 ){ segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); }else{ iMin = 0; iMax = pPtr->nCell-1; while( 1 ){ int iTry = (iMin+iMax)/2; void *pKeyT; int nKeyT; /* Key for cell iTry */ |
︙ | ︙ | |||
1712 1713 1714 1715 1716 1717 1718 | pCsr->eType = pPtr->eType; rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); if( rc==LSM_OK ){ rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); } pCsr->flags |= CURSOR_SEEK_EQ; } | | | 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 | pCsr->eType = pPtr->eType; rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); if( rc==LSM_OK ){ rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); } pCsr->flags |= CURSOR_SEEK_EQ; } segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); break; } case LSM_SEEK_LE: if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1); break; case LSM_SEEK_GE: { /* Figure out if we need to 'skip' the pointer forward or not */ |
︙ | ︙ | |||
1939 1940 1941 1942 1943 1944 1945 | /* If the segment-pointer has settled on a key that is smaller than ** the splitkey, invalidate the segment-pointer. */ if( pPtr->pPg ){ res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); | | | 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 | /* If the segment-pointer has settled on a key that is smaller than ** the splitkey, invalidate the segment-pointer. */ if( pPtr->pPg ){ res = sortedKeyCompare(pCsr->pDb->xCmp, rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey ); if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } if( aPtr[i].pKey ) bHit = 1; } if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ rc = segmentPtrEnd(pCsr, &aPtr[0], 1); |
︙ | ︙ | |||
2215 2216 2217 2218 2219 2220 2221 | /* Close the tree cursor, if any. */ lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); /* Reset the segment pointers */ for(i=0; i<pCsr->nPtr; i++){ | | | 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 | /* Close the tree cursor, if any. */ lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); /* Reset the segment pointers */ for(i=0; i<pCsr->nPtr; i++){ segmentPtrReset(&pCsr->aPtr[i], 0); } /* And the b-tree cursor, if any */ btreeCursorFree(pCsr->pBtCsr); /* Free allocations */ lsmFree(pEnv, pCsr->aPtr); |
︙ | ︙ | |||
2870 2871 2872 2873 2874 2875 2876 | for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); if( pPtr[iRhs+1].pPg ) bHit = 1; } if( bHit==0 && rc==LSM_OK ){ rc = segmentPtrEnd(pCsr, pPtr, 1); }else{ | | | | 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 | for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); if( pPtr[iRhs+1].pPg ) bHit = 1; } if( bHit==0 && rc==LSM_OK ){ rc = segmentPtrEnd(pCsr, pPtr, 1); }else{ segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); } }else{ int bLhs = (pPtr->pSeg==&pLvl->lhs); assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); if( bLhs ){ rc = segmentPtrEnd(pCsr, pPtr, 0); if( pPtr->pKey ) bHit = 1; } for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ if( bHit ){ segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD); }else{ rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); } } } i += pLvl->nRight; } |
︙ | ︙ | |||
2965 2966 2967 2968 2969 2970 2971 | } void lsmMCursorReset(MultiCursor *pCsr){ int i; lsmTreeCursorReset(pCsr->apTreeCsr[0]); lsmTreeCursorReset(pCsr->apTreeCsr[1]); for(i=0; i<pCsr->nPtr; i++){ | | | 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 | } void lsmMCursorReset(MultiCursor *pCsr){ int i; lsmTreeCursorReset(pCsr->apTreeCsr[0]); lsmTreeCursorReset(pCsr->apTreeCsr[1]); for(i=0; i<pCsr->nPtr; i++){ segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); } pCsr->key.nData = 0; } static int treeCursorSeek( MultiCursor *pCsr, TreeCursor *pTreeCsr, |
︙ | ︙ | |||
6075 6076 6077 6078 6079 6080 6081 | if( ptr1.pPg==0 ){ assert( 0 ); } } } } | | | | 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 | if( ptr1.pPg==0 ){ assert( 0 ); } } } } segmentPtrReset(&ptr1, 0); segmentPtrReset(&ptr2, 0); return LSM_OK; } /* ** This function is only included in the build if LSM_DEBUG_EXPENSIVE is ** defined. Its only purpose is to evaluate various assert() statements to ** verify that the database is well formed in certain respects. |
︙ | ︙ |