Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| SHA1 Hash: | a9f8129cce4a6adbdf70f80efb6d6f5dab752209 |
|---|---|
| Date: | 2012-11-29 09:48:22 |
| User: | dan |
| Comment: | In mmap-mode, ensure a checkpointer has the entire database file mapped before calling msync(). |
Tags And Properties
- branch=trunk inherited from [84d5dea8fd]
- sym-trunk inherited from [84d5dea8fd]
Changes
Changes to lsm-test/lsmtest_tdb3.c
11 11 12 #include <sys/time.h> 12 #include <sys/time.h> 13 13 14 typedef struct LsmDb LsmDb; 14 typedef struct LsmDb LsmDb; 15 typedef struct LsmWorker LsmWorker; 15 typedef struct LsmWorker LsmWorker; 16 typedef struct LsmFile LsmFile; 16 typedef struct LsmFile LsmFile; 17 17 > 18 #define LSMTEST_DFLT_MT_MAX_CKPT (8*1024*1024) > 19 #define LSMTEST_DFLT_MT_MIN_CKPT (2*1024*1024) > 20 18 #ifdef LSM_MUTEX_PTHREADS 21 #ifdef LSM_MUTEX_PTHREADS 19 #include <pthread.h> 22 #include <pthread.h> 20 23 21 #define LSMTEST_THREAD_CKPT 1 24 #define LSMTEST_THREAD_CKPT 1 22 #define LSMTEST_THREAD_WORKER 2 25 #define LSMTEST_THREAD_WORKER 2 23 #define LSMTEST_THREAD_WORKER_AC 3 26 #define LSMTEST_THREAD_WORKER_AC 3 24 27 ................................................................................................................................................................................ 505 rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nByte); 508 rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nByte); 506 if( rc!=LSM_OK || nByte<pDb->nMtMaxCkpt ) break; 509 if( rc!=LSM_OK || nByte<pDb->nMtMaxCkpt ) break; 507 usleep(5000); 510 usleep(5000); 508 nSleep += 5; 511 nSleep += 5; 509 }while( 1 ); 512 }while( 1 ); 510 513 511 #if 0 514 #if 0 512 if( nSleep ) printf("waitOnCheckpointer(): nSleep=%d\n", nSleep); | 515 if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); 513 #endif 516 #endif 514 517 515 return rc; 518 return rc; 516 } 519 } 517 520 518 static int waitOnWorker(LsmDb *pDb){ 521 static int waitOnWorker(LsmDb *pDb){ 519 int rc; 522 int rc; ................................................................................................................................................................................ 527 if( rc!=LSM_OK ) return rc; 530 if( rc!=LSM_OK ) return rc; 528 if( bOld==0 || nNew<nLimit ) break; 531 if( bOld==0 || nNew<nLimit ) break; 529 usleep(5000); 532 usleep(5000); 530 nSleep += 5; 533 nSleep += 5; 531 }while( 1 ); 534 }while( 1 ); 532 535 533 #if 0 536 #if 0 534 if( nSleep ) printf("waitOnWorker(): nSleep=%d\n", nSleep); | 537 if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); 535 #endif 538 #endif 536 539 537 return rc; 540 return rc; 538 } 541 } 539 542 540 static int test_lsm_write( 543 static int test_lsm_write( 541 TestDb *pTestDb, 544 TestDb *pTestDb, ................................................................................................................................................................................ 915 ** Todo: There should be an OS method to obtain this value - just as 918 ** Todo: There should be an OS method to obtain this value - just as 916 ** there is in SQLite. For now, LSM assumes that it is smaller than 919 ** there is in SQLite. For now, LSM assumes that it is smaller than 917 ** the page size (default 4KB). 920 ** the page size (default 4KB). 918 */ 921 */ 919 pDb->szSector = 256; 922 pDb->szSector = 256; 920 923 921 /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ 924 /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ 922 pDb->nMtMinCkpt = 2 * (1<<20); | 925 pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; 923 pDb->nMtMaxCkpt = 16 * (1<<20); | 926 pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; 924 927 925 memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); 928 memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); 926 pDb->env.pVfsCtx = (void *)pDb; 929 pDb->env.pVfsCtx = (void *)pDb; 927 pDb->env.xFullpath = testEnvFullpath; 930 pDb->env.xFullpath = testEnvFullpath; 928 pDb->env.xOpen = testEnvOpen; 931 pDb->env.xOpen = testEnvOpen; 929 pDb->env.xRead = testEnvRead; 932 pDb->env.xRead = testEnvRead; 930 pDb->env.xWrite = testEnvWrite; 933 pDb->env.xWrite = testEnvWrite;
Changes to src/lsmInt.h
530 int lsmCheckpointLoadWorker(lsm_db *pDb); 530 int lsmCheckpointLoadWorker(lsm_db *pDb); 531 int lsmCheckpointStore(lsm_db *pDb, int); 531 int lsmCheckpointStore(lsm_db *pDb, int); 532 532 533 int lsmCheckpointLoad(lsm_db *pDb, int *); 533 int lsmCheckpointLoad(lsm_db *pDb, int *); 534 int lsmCheckpointLoadOk(lsm_db *pDb, int); 534 int lsmCheckpointLoadOk(lsm_db *pDb, int); 535 int lsmCheckpointClientCacheOk(lsm_db *); 535 int lsmCheckpointClientCacheOk(lsm_db *); 536 536 > 537 u32 lsmCheckpointNBlock(u32 *); 537 i64 lsmCheckpointId(u32 *, int); 538 i64 lsmCheckpointId(u32 *, int); 538 u32 lsmCheckpointNWrite(u32 *, int); 539 u32 lsmCheckpointNWrite(u32 *, int); 539 i64 lsmCheckpointLogOffset(u32 *); 540 i64 lsmCheckpointLogOffset(u32 *); 540 int lsmCheckpointPgsz(u32 *); 541 int lsmCheckpointPgsz(u32 *); 541 int lsmCheckpointBlksz(u32 *); 542 int lsmCheckpointBlksz(u32 *); 542 void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); 543 void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); 543 void lsmCheckpointZeroLogoffset(lsm_db *); 544 void lsmCheckpointZeroLogoffset(lsm_db *); ................................................................................................................................................................................ 685 int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); 686 int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); 686 int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); 687 int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); 687 int lsmFsCloseAndDeleteLog(FileSystem *pFS); 688 int lsmFsCloseAndDeleteLog(FileSystem *pFS); 688 689 689 void lsmFsDeferClose(FileSystem *pFS, LsmFile **pp); 690 void lsmFsDeferClose(FileSystem *pFS, LsmFile **pp); 690 691 691 /* And to sync the db file */ 692 /* And to sync the db file */ 692 int lsmFsSyncDb(FileSystem *); | 693 int lsmFsSyncDb(FileSystem *, int); 693 694 694 void lsmFsFlushWaiting(FileSystem *, int *); 695 void lsmFsFlushWaiting(FileSystem *, int *); 695 696 696 /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ 697 /* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ 697 int lsmInfoArrayStructure(lsm_db *pDb, Pgno iFirst, char **pzOut); 698 int lsmInfoArrayStructure(lsm_db *pDb, Pgno iFirst, char **pzOut); 698 int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut); 699 int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut); 699 int lsmConfigMmap(lsm_db *pDb, int *piParam); 700 int lsmConfigMmap(lsm_db *pDb, int *piParam);
Changes to src/lsm_ckpt.c
1087 iId = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32); 1087 iId = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32); 1088 iId += ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); 1088 iId += ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); 1089 }else{ 1089 }else{ 1090 iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; 1090 iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; 1091 } 1091 } 1092 return iId; 1092 return iId; 1093 } 1093 } > 1094 > 1095 u32 lsmCheckpointNBlock(u32 *aCkpt){ > 1096 return aCkpt[CKPT_HDR_NBLOCK]; > 1097 } 1094 1098 1095 u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ 1099 u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ 1096 if( bDisk ){ 1100 if( bDisk ){ 1097 return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); 1101 return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); 1098 }else{ 1102 }else{ 1099 return aCkpt[CKPT_HDR_NWRITE]; 1103 return aCkpt[CKPT_HDR_NWRITE]; 1100 } 1104 }
Changes to src/lsm_file.c
852 } 852 } 853 lsmSortedRemap(pFS->pDb); 853 lsmSortedRemap(pFS->pDb); 854 } 854 } 855 *pRc = rc; 855 *pRc = rc; 856 } 856 } 857 } 857 } 858 858 > 859 > 860 /* > 861 ** fsync() the database file. > 862 */ > 863 int lsmFsSyncDb(FileSystem *pFS, int nBlock){ > 864 if( nBlock && pFS->bUseMmap ){ > 865 int rc = LSM_OK; > 866 i64 nMin = (i64)nBlock * (i64)pFS->nBlocksize; > 867 fsGrowMapping(pFS, nMin, &rc); > 868 if( rc!=LSM_OK ) return rc; > 869 } > 870 return lsmEnvSync(pFS->pEnv, pFS->fdDb); > 871 } > 872 859 static int fsPageGet(FileSystem *, Pgno, int, Page **, int *); 873 static int fsPageGet(FileSystem *, Pgno, int, Page **, int *); 860 874 861 /* 875 /* 862 ** Parameter iBlock is a database file block. This function reads the value 876 ** Parameter iBlock is a database file block. This function reads the value 863 ** stored in the blocks "next block" pointer and stores it in *piNext. 877 ** stored in the blocks "next block" pointer and stores it in *piNext. 864 ** LSM_OK is returned if everything is successful, or an LSM error code 878 ** LSM_OK is returned if everything is successful, or an LSM error code 865 ** otherwise. 879 ** otherwise. ................................................................................................................................................................................ 976 nByte += (aBuf[2] & 0x7F); 990 nByte += (aBuf[2] & 0x7F); 977 *pbFree = !(aBuf[1] & 0x80); 991 *pbFree = !(aBuf[1] & 0x80); 978 return nByte; 992 return nByte; 979 } 993 } 980 994 981 static int fsSubtractOffset(FileSystem *pFS, i64 iOff, int iSub, i64 *piRes){ 995 static int fsSubtractOffset(FileSystem *pFS, i64 iOff, int iSub, i64 *piRes){ 982 i64 iStart; 996 i64 iStart; 983 int iBlk; | 997 int iBlk = 0; 984 int rc; 998 int rc; 985 999 986 assert( pFS->pCompress ); 1000 assert( pFS->pCompress ); 987 1001 988 iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); 1002 iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); 989 if( (iOff-iSub)>=iStart ){ 1003 if( (iOff-iSub)>=iStart ){ 990 *piRes = (iOff-iSub); 1004 *piRes = (iOff-iSub); ................................................................................................................................................................................ 2161 int lsmFsNRead(FileSystem *pFS){ return pFS->nRead; } 2175 int lsmFsNRead(FileSystem *pFS){ return pFS->nRead; } 2162 2176 2163 /* 2177 /* 2164 ** Return the total number of pages written to the database file. 2178 ** Return the total number of pages written to the database file. 2165 */ 2179 */ 2166 int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } 2180 int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } 2167 2181 2168 /* < 2169 ** fsync() the database file. < 2170 */ < 2171 int lsmFsSyncDb(FileSystem *pFS){ < 2172 return lsmEnvSync(pFS->pEnv, pFS->fdDb); < 2173 } < 2174 < 2175 /* 2182 /* 2176 ** Return a copy of the environment pointer used by the file-system object. 2183 ** Return a copy of the environment pointer used by the file-system object. 2177 */ 2184 */ 2178 lsm_env *lsmFsEnv(FileSystem *pFS) { 2185 lsm_env *lsmFsEnv(FileSystem *pFS) { 2179 return pFS->pEnv; 2186 return pFS->pEnv; 2180 } 2187 } 2181 2188
Changes to src/lsm_shared.c
686 } 686 } 687 bDone = (iDisk>=iCkpt); 687 bDone = (iDisk>=iCkpt); 688 } 688 } 689 689 690 if( rc==LSM_OK && bDone==0 ){ 690 if( rc==LSM_OK && bDone==0 ){ 691 int iMeta = (pShm->iMetaPage % 2) + 1; 691 int iMeta = (pShm->iMetaPage % 2) + 1; 692 if( pDb->eSafety!=LSM_SAFETY_OFF ){ 692 if( pDb->eSafety!=LSM_SAFETY_OFF ){ > 693 int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); 693 rc = lsmFsSyncDb(pDb->pFS); | 694 rc = lsmFsSyncDb(pDb->pFS, nBlock); 694 } 695 } 695 if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); 696 if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); 696 if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ 697 if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ 697 rc = lsmFsSyncDb(pDb->pFS); | 698 rc = lsmFsSyncDb(pDb->pFS, 0); 698 } 699 } 699 if( rc==LSM_OK ){ 700 if( rc==LSM_OK ){ 700 pShm->iMetaPage = iMeta; 701 pShm->iMetaPage = iMeta; 701 nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; 702 nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; 702 } 703 } 703 #ifdef LSM_LOG_WORK 704 #ifdef LSM_LOG_WORK 704 lsmLogMessage(pDb, 0, "finish checkpoint %d", 705 lsmLogMessage(pDb, 0, "finish checkpoint %d",
Changes to tool/lsmperf.tcl
186 append script $data3 186 append script $data3 187 append script $data4 187 append script $data4 188 188 189 append script "pause -1\n" 189 append script "pause -1\n" 190 exec_gnuplot_script $script $zPng 190 exec_gnuplot_script $script $zPng 191 } 191 } 192 192 193 do_write_test x.png 60 50000 50000 200 { | 193 do_write_test x.png 400 100000 100000 100 { 194 lsm-mt "mt_mode=4 multi_proc=0 autoflush=4M" | 194 lsm-4M "mt_mode=4 multi_proc=0 autoflush=4M page_size=1024" 195 leveldb leveldb < 196 } 195 } 197 196 198 197 199 #lsm "mmap=1 multi_proc=0 page_size=4096 block_size=2097152 autocheckpoint=419 198 #lsm "mmap=1 multi_proc=0 page_size=4096 block_size=2097152 autocheckpoint=419 200 #lsm-mt "mmap=1 multi_proc=0 threads=2 autowork=0 autocheckpoint=4196000" 199 #lsm-mt "mmap=1 multi_proc=0 threads=2 autowork=0 autocheckpoint=4196000" 201 200 202 # lsm "safety=1 multi_proc=0" 201 # lsm "safety=1 multi_proc=0"