Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| SHA1 Hash: | 21db5f73f5d8ae85b917ebe63323b40af5fbd9ec |
|---|---|
| Date: | 2012-11-28 14:54:23 |
| User: | dan |
| Comment: | Fix an mmap mode bug. Improve upon multi-threaded setups in lsmtest_tdb3.c. |
Tags And Properties
- branch=trunk inherited from [84d5dea8fd]
- sym-trunk inherited from [84d5dea8fd]
Changes
Changes to lsm-test/lsmtest_tdb3.c
1 1 2 #include "lsmtest_tdb.h" 2 #include "lsmtest_tdb.h" 3 #include "lsm.h" 3 #include "lsm.h" 4 < 5 #include "lsmtest.h" 4 #include "lsmtest.h" 6 5 7 #include <stdlib.h> 6 #include <stdlib.h> 8 #include <string.h> 7 #include <string.h> 9 #include <assert.h> 8 #include <assert.h> 10 #include <unistd.h> 9 #include <unistd.h> 11 #include <stdio.h> 10 #include <stdio.h> ................................................................................................................................................................................ 14 13 15 typedef struct LsmDb LsmDb; 14 typedef struct LsmDb LsmDb; 16 typedef struct LsmWorker LsmWorker; 15 typedef struct LsmWorker LsmWorker; 17 typedef struct LsmFile LsmFile; 16 typedef struct LsmFile LsmFile; 18 17 19 #ifdef LSM_MUTEX_PTHREADS 18 #ifdef LSM_MUTEX_PTHREADS 20 #include <pthread.h> 19 #include <pthread.h> > 20 > 21 #define LSMTEST_THREAD_CKPT 1 > 22 #define LSMTEST_THREAD_WORKER 2 > 23 #define LSMTEST_THREAD_WORKER_AC 3 > 24 > 25 /* > 26 ** There are several different types of worker threads that run in different > 27 ** test configurations, depending on the value of LsmWorker.eType. > 28 ** > 29 ** 1. Checkpointer. > 30 ** 2. Worker with auto-checkpoint. > 31 ** 3. Worker without auto-checkpoint. > 32 */ 21 struct LsmWorker { 33 struct LsmWorker { 22 LsmDb *pDb; /* Main database structure */ 34 LsmDb *pDb; /* Main database structure */ 23 lsm_db *pWorker; /* Worker database handle */ 35 lsm_db *pWorker; /* Worker database handle */ 24 pthread_t worker_thread; /* Worker thread */ 36 pthread_t worker_thread; /* Worker thread */ 25 pthread_cond_t worker_cond; /* Condition var the worker waits on */ 37 pthread_cond_t worker_cond; /* Condition var the worker waits on */ 26 pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ 38 pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ 27 int bDoWork; /* Set to true by client when there is work */ 39 int bDoWork; /* Set to true by client when there is work */ 28 int worker_rc; /* Store error code here */ 40 int worker_rc; /* Store error code here */ 29 < 30 int lsm_work_flags; /* Flags to pass to lsm_work() */ < 31 int lsm_work_npage; /* nPage parameter to pass to lsm_work() */ < 32 int bCkpt; /* True to call lsm_checkpoint() */ | 41 int eType; /* LSMTEST_THREAD_XXX constant */ > 42 int bBlock; 33 }; 43 }; 34 #else 44 #else 35 struct LsmWorker { int worker_rc; }; | 45 struct LsmWorker { int worker_rc; int bBlock; }; 36 #endif 46 #endif 37 47 38 static void mt_shutdown(LsmDb *); 48 static void mt_shutdown(LsmDb *); 39 49 40 lsm_env *tdb_lsm_env(void){ 50 lsm_env *tdb_lsm_env(void){ 41 static int bInit = 0; 51 static int bInit = 0; 42 static lsm_env env; 52 static lsm_env env; ................................................................................................................................................................................ 477 return rc; 487 return rc; 478 } 488 } 479 489 480 static int test_lsm_write( 490 static int test_lsm_write( 481 TestDb *pTestDb, 491 TestDb *pTestDb, 482 void *pKey, 492 void *pKey, 483 int nKey, 493 int nKey, 484 void *pVal, | 494 void *pVal, 485 int nVal 495 int nVal 486 ){ 496 ){ 487 LsmDb *pDb = (LsmDb *)pTestDb; 497 LsmDb *pDb = (LsmDb *)pTestDb; 488 498 > 499 int nSleep = 0; > 500 while( pDb->aWorker && pDb->aWorker[0].bBlock ){ > 501 usleep(1000); > 502 nSleep++; > 503 } > 504 #if 0 > 505 if( nSleep ) printf("nSleep=%d\n", nSleep); 489 if( pDb->aWorker ){ 506 if( pDb->aWorker ){ 490 int nLimit = -1; 507 int nLimit = -1; 491 int nSleep = 0; 508 int nSleep = 0; 492 lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); 509 lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); 493 do { 510 do { 494 int bOld, nNew, rc; 511 int bOld, nNew, rc; 495 rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &bOld, &nNew); 512 rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &bOld, &nNew); ................................................................................................................................................................................ 498 usleep(1000); 515 usleep(1000); 499 nSleep += 1; 516 nSleep += 1; 500 }while( 1 ); 517 }while( 1 ); 501 #if 0 518 #if 0 502 if( nSleep ) printf("nSleep=%d\n", nSleep); 519 if( nSleep ) printf("nSleep=%d\n", nSleep); 503 #endif 520 #endif 504 } 521 } > 522 #endif 505 523 506 return lsm_insert(pDb->db, pKey, nKey, pVal, nVal); 524 return lsm_insert(pDb->db, pKey, nKey, pVal, nVal); 507 } 525 } 508 526 509 static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ 527 static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ 510 LsmDb *pDb = (LsmDb *)pTestDb; 528 LsmDb *pDb = (LsmDb *)pTestDb; 511 return lsm_delete(pDb->db, pKey, nKey); 529 return lsm_delete(pDb->db, pKey, nKey); ................................................................................................................................................................................ 869 lsm_config_log(pDb->db, xLog, 0); 887 lsm_config_log(pDb->db, xLog, 0); 870 lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); 888 lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); 871 889 872 rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); 890 rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); 873 if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); 891 if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); 874 892 875 #ifdef LSM_MUTEX_PTHREADS 893 #ifdef LSM_MUTEX_PTHREADS 876 if( rc==LSM_OK && (nThread==2 || nThread==3) ){ | 894 if( rc==LSM_OK && nThread>1 ){ 877 testLsmStartWorkers(pDb, nThread-1, zFilename, zCfg); | 895 testLsmStartWorkers(pDb, nThread, zFilename, zCfg); 878 } 896 } 879 #endif 897 #endif 880 898 881 if( rc!=LSM_OK ){ 899 if( rc!=LSM_OK ){ 882 test_lsm_close((TestDb *)pDb); 900 test_lsm_close((TestDb *)pDb); 883 pDb = 0; 901 pDb = 0; 884 } 902 } ................................................................................................................................................................................ 933 } 951 } 934 return 0; 952 return 0; 935 } 953 } 936 954 937 void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ 955 void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ 938 lsm_db *db = tdb_lsm(pDb); 956 lsm_db *db = tdb_lsm(pDb); 939 if( db ){ 957 if( db ){ 940 LsmDb *p = (LsmDb *)pDb; < 941 int i; < 942 lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); 958 lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); 943 } 959 } 944 } 960 } 945 961 946 void tdb_lsm_application_crash(TestDb *pDb){ 962 void tdb_lsm_application_crash(TestDb *pDb){ 947 if( tdb_lsm(pDb) ){ 963 if( tdb_lsm(pDb) ){ 948 LsmDb *p = (LsmDb *)pDb; 964 LsmDb *p = (LsmDb *)pDb; ................................................................................................................................................................................ 1023 LsmWorker *p = &pDb->aWorker[iWorker]; 1039 LsmWorker *p = &pDb->aWorker[iWorker]; 1024 pthread_mutex_lock(&p->worker_mutex); 1040 pthread_mutex_lock(&p->worker_mutex); 1025 p->bDoWork = 1; 1041 p->bDoWork = 1; 1026 pthread_cond_signal(&p->worker_cond); 1042 pthread_cond_signal(&p->worker_cond); 1027 pthread_mutex_unlock(&p->worker_mutex); 1043 pthread_mutex_unlock(&p->worker_mutex); 1028 } 1044 } 1029 1045 > 1046 /* > 1047 ** This routine is used as the main() for all worker threads. > 1048 */ 1030 static void *worker_main(void *pArg){ 1049 static void *worker_main(void *pArg){ 1031 LsmWorker *p = (LsmWorker *)pArg; 1050 LsmWorker *p = (LsmWorker *)pArg; 1032 lsm_db *pWorker; /* Connection to access db through */ 1051 lsm_db *pWorker; /* Connection to access db through */ 1033 1052 1034 pthread_mutex_lock(&p->worker_mutex); 1053 pthread_mutex_lock(&p->worker_mutex); 1035 while( (pWorker = p->pWorker) ){ 1054 while( (pWorker = p->pWorker) ){ 1036 int rc = LSM_OK; 1055 int rc = LSM_OK; 1037 int nCkpt = -1; 1056 int nCkpt = -1; 1038 1057 1039 /* Do some work. If an error occurs, exit. */ 1058 /* Do some work. If an error occurs, exit. */ 1040 pthread_mutex_unlock(&p->worker_mutex); 1059 pthread_mutex_unlock(&p->worker_mutex); 1041 1060 1042 if( p->bCkpt ){ | 1061 if( p->eType==LSMTEST_THREAD_CKPT ){ > 1062 int nByte = 0; > 1063 rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nByte); > 1064 if( rc==LSM_OK && nByte>=(2*1024*1024) ){ > 1065 if( nByte>(8*1024*1024) ) p->bBlock = 1; 1043 rc = lsm_checkpoint(pWorker, 0); | 1066 rc = lsm_checkpoint(pWorker, 0); > 1067 p->bBlock = 0; > 1068 } 1044 }else{ 1069 }else{ 1045 int nWrite = 0; /* Pages written by lsm_work() call */ 1070 int nWrite = 0; /* Pages written by lsm_work() call */ 1046 int nAuto = -1; /* Configured AUTOCHECKPOINT value */ 1071 int nAuto = -1; /* Configured AUTOCHECKPOINT value */ 1047 int nLimit = -1; /* Configured AUTOFLUSH value */ 1072 int nLimit = -1; /* Configured AUTOFLUSH value */ 1048 1073 1049 lsm_config(pWorker, LSM_CONFIG_AUTOFLUSH, &nLimit); 1074 lsm_config(pWorker, LSM_CONFIG_AUTOFLUSH, &nLimit); 1050 lsm_config(pWorker, LSM_CONFIG_AUTOCHECKPOINT, &nAuto); 1075 lsm_config(pWorker, LSM_CONFIG_AUTOCHECKPOINT, &nAuto); 1051 do { 1076 do { 1052 int nSleep = 0; < 1053 lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nCkpt); 1077 lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nCkpt); > 1078 #if 0 > 1079 int nSleep = 0; 1054 while( nAuto==0 && nCkpt>(nLimit*4) ){ 1080 while( nAuto==0 && nCkpt>(nLimit*4) ){ 1055 usleep(1000); 1081 usleep(1000); 1056 mt_signal_worker(p->pDb, 1); 1082 mt_signal_worker(p->pDb, 1); 1057 nSleep++; 1083 nSleep++; 1058 lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nCkpt); 1084 lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nCkpt); 1059 } 1085 } 1060 #if 0 | 1086 if( nSleep ) printf("nLimit=%d nSleep=%d (worker)\n", nLimit, nSleep); 1061 if( nSleep ) printf("nLimit=%d nSleep=%d (worker)\n", nLimit, nSleep); | 1087 #endif 1062 #endif | 1088 rc = lsm_work(pWorker, 0, 256, &nWrite); 1063 | 1089 if( p->eType==LSMTEST_THREAD_WORKER_AC && nWrite && rc==LSM_OK ){ 1064 rc = lsm_work(pWorker, p->lsm_work_flags, p->lsm_work_npage, &nWrite); | 1090 mt_signal_worker(p->pDb, 1); 1065 if( nAuto==0 && nWrite && rc==LSM_OK ) mt_signal_worker(p->pDb, 1); | 1091 } 1066 }while( nWrite && p->pWorker ); 1092 }while( nWrite && p->pWorker ); 1067 } 1093 } 1068 pthread_mutex_lock(&p->worker_mutex); 1094 pthread_mutex_lock(&p->worker_mutex); 1069 1095 1070 if( rc!=LSM_OK && rc!=LSM_BUSY ){ 1096 if( rc!=LSM_OK && rc!=LSM_BUSY ){ 1071 p->worker_rc = rc; 1097 p->worker_rc = rc; 1072 break; 1098 break; ................................................................................................................................................................................ 1122 ** This callback is invoked by LSM when the client database writes to 1148 ** This callback is invoked by LSM when the client database writes to 1123 ** the database file (i.e. to flush the contents of the in-memory tree). 1149 ** the database file (i.e. to flush the contents of the in-memory tree). 1124 ** This implies there may be work to do on the database, so signal 1150 ** This implies there may be work to do on the database, so signal 1125 ** the worker threads. 1151 ** the worker threads. 1126 */ 1152 */ 1127 static void mt_client_work_hook(lsm_db *db, void *pArg){ 1153 static void mt_client_work_hook(lsm_db *db, void *pArg){ 1128 LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ 1154 LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ 1129 int i; /* Iterator variable */ < 1130 1155 1131 /* Invoke the user level work-hook, if any. */ 1156 /* Invoke the user level work-hook, if any. */ 1132 if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); 1157 if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); 1133 1158 1134 /* Signal the lsm_work() thread */ 1159 /* Signal the lsm_work() thread */ 1135 mt_signal_worker(pDb, 0); 1160 mt_signal_worker(pDb, 0); 1136 } 1161 } ................................................................................................................................................................................ 1145 /* 1170 /* 1146 ** Launch worker thread iWorker for database connection pDb. 1171 ** Launch worker thread iWorker for database connection pDb. 1147 */ 1172 */ 1148 static int mt_start_worker( 1173 static int mt_start_worker( 1149 LsmDb *pDb, /* Main database structure */ 1174 LsmDb *pDb, /* Main database structure */ 1150 int iWorker, /* Worker number to start */ 1175 int iWorker, /* Worker number to start */ 1151 const char *zFilename, /* File name of database to open */ 1176 const char *zFilename, /* File name of database to open */ 1152 const char *zCfg, | 1177 const char *zCfg, /* Connection configuration string */ 1153 int flags, /* flags parameter to lsm_work() */ < 1154 int nPage, /* nPage parameter to lsm_work() */ < 1155 int bCkpt /* True to call lsm_checkpoint() */ | 1178 int eType /* Type of worker thread */ 1156 ){ 1179 ){ 1157 int rc = 0; /* Return code */ 1180 int rc = 0; /* Return code */ 1158 LsmWorker *p; /* Object to initialize */ 1181 LsmWorker *p; /* Object to initialize */ 1159 1182 1160 assert( iWorker<pDb->nWorker ); 1183 assert( iWorker<pDb->nWorker ); > 1184 assert( eType==LSMTEST_THREAD_CKPT > 1185 || eType==LSMTEST_THREAD_WORKER > 1186 || eType==LSMTEST_THREAD_WORKER_AC > 1187 ); 1161 1188 1162 p = &pDb->aWorker[iWorker]; 1189 p = &pDb->aWorker[iWorker]; 1163 p->lsm_work_flags = flags; < 1164 p->lsm_work_npage = nPage; < 1165 p->bCkpt = bCkpt; < > 1190 p->eType = eType; 1166 p->pDb = pDb; 1191 p->pDb = pDb; 1167 1192 1168 /* Open the worker connection */ 1193 /* Open the worker connection */ 1169 if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); 1194 if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); 1170 if( zCfg ){ 1195 if( zCfg ){ 1171 test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0); 1196 test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0); 1172 } 1197 } ................................................................................................................................................................................ 1184 if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); 1209 if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); 1185 1210 1186 return rc; 1211 return rc; 1187 } 1212 } 1188 1213 1189 1214 1190 static int testLsmStartWorkers( 1215 static int testLsmStartWorkers( 1191 LsmDb *pDb, int nWorker, const char *zFilename, const char *zCfg | 1216 LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg 1192 ){ 1217 ){ 1193 int rc; 1218 int rc; 1194 int bAutowork = 0; < > 1219 1195 assert( nWorker==1 || nWorker==2 ); | 1220 if( eModel<1 || eModel>4 ) return 1; > 1221 if( eModel==1 ) return 0; 1196 1222 1197 /* Configure a work-hook for the client connection. */ | 1223 /* Configure a work-hook for the client connection. Worker 0 is signalled > 1224 ** every time the users connection writes to the database. */ 1198 lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); 1225 lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); 1199 1226 > 1227 /* Allocate space for two worker connections. They may not both be > 1228 ** used, but both are allocated. */ 1200 pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * nWorker); | 1229 pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); 1201 memset(pDb->aWorker, 0, sizeof(LsmWorker) * nWorker); | 1230 memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); 1202 pDb->nWorker = nWorker; < 1203 1231 > 1232 switch( eModel ){ > 1233 case 2: 1204 if( nWorker==1 ){ | 1234 pDb->nWorker = 1; > 1235 test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); 1205 rc = mt_start_worker(pDb, 0, zFilename, zCfg, 0, 256, 0); | 1236 rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); 1206 }else{ < 1207 rc = mt_start_worker(pDb, 0, zFilename, zCfg, 0, 256, 0); < 1208 if( rc==LSM_OK ){ < 1209 rc = mt_start_worker(pDb, 1, zFilename, zCfg, 0, 0, 1); < > 1237 break; 1210 } | 1238 > 1239 case 3: > 1240 pDb->nWorker = 2; > 1241 assert( 0 ); > 1242 break; > 1243 > 1244 case 4: > 1245 pDb->nWorker = 2; > 1246 assert( 0 ); > 1247 break; 1211 } 1248 } 1212 1249 1213 return rc; 1250 return rc; 1214 } 1251 } 1215 1252 1216 1253 1217 int test_lsm_mt2(const char *zFilename, int bClear, TestDb **ppDb){ 1254 int test_lsm_mt2(const char *zFilename, int bClear, TestDb **ppDb){
Changes to src/lsm.h
392 #define LSM_INFO_ARRAY_STRUCTURE 5 392 #define LSM_INFO_ARRAY_STRUCTURE 5 393 #define LSM_INFO_PAGE_ASCII_DUMP 6 393 #define LSM_INFO_PAGE_ASCII_DUMP 6 394 #define LSM_INFO_PAGE_HEX_DUMP 7 394 #define LSM_INFO_PAGE_HEX_DUMP 7 395 #define LSM_INFO_FREELIST 8 395 #define LSM_INFO_FREELIST 8 396 #define LSM_INFO_ARRAY_PAGES 9 396 #define LSM_INFO_ARRAY_PAGES 9 397 #define LSM_INFO_CHECKPOINT_SIZE 10 397 #define LSM_INFO_CHECKPOINT_SIZE 10 398 #define LSM_INFO_TREE_SIZE 11 398 #define LSM_INFO_TREE_SIZE 11 > 399 > 400 #define LSM_INFO_FREELIST_SIZE 12 399 401 400 402 401 /* 403 /* 402 ** CAPI: Opening and Closing Write Transactions 404 ** CAPI: Opening and Closing Write Transactions 403 ** 405 ** 404 ** These functions are used to open and close transactions and nested 406 ** These functions are used to open and close transactions and nested 405 ** sub-transactions. 407 ** sub-transactions.
Changes to src/lsmInt.h
786 int lsmLogCommit(lsm_db *); 786 int lsmLogCommit(lsm_db *); 787 void lsmLogEnd(lsm_db *pDb, int bCommit); 787 void lsmLogEnd(lsm_db *pDb, int bCommit); 788 void lsmLogTell(lsm_db *, LogMark *); 788 void lsmLogTell(lsm_db *, LogMark *); 789 void lsmLogSeek(lsm_db *, LogMark *); 789 void lsmLogSeek(lsm_db *, LogMark *); 790 790 791 int lsmLogRecover(lsm_db *); 791 int lsmLogRecover(lsm_db *); 792 int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); 792 int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); 793 int lsmInfoFreelist(lsm_db *, char **pzVal); < 794 793 795 794 796 /************************************************************************** 795 /************************************************************************** 797 ** Functions from file "lsm_shared.c". 796 ** Functions from file "lsm_shared.c". 798 */ 797 */ 799 798 800 int lsmDbDatabaseConnect(lsm_db*, const char *); 799 int lsmDbDatabaseConnect(lsm_db*, const char *);
Changes to src/lsm_ckpt.c
32 ** 2. The checkpoint id LSW. 32 ** 2. The checkpoint id LSW. 33 ** 3. The number of integer values in the entire checkpoint, including 33 ** 3. The number of integer values in the entire checkpoint, including 34 ** the two checksum values. 34 ** the two checksum values. 35 ** 4. The total number of blocks in the database. 35 ** 4. The total number of blocks in the database. 36 ** 5. The block size. 36 ** 5. The block size. 37 ** 6. The number of levels. 37 ** 6. The number of levels. 38 ** 7. The nominal database page size. 38 ** 7. The nominal database page size. > 39 ** 8. The number of pages (in total) written to the database file. 39 ** 40 ** 40 ** Log pointer: 41 ** Log pointer: 41 ** 42 ** 42 ** 1. The log offset MSW. 43 ** 1. The log offset MSW. 43 ** 2. The log offset LSW. 44 ** 2. The log offset LSW. 44 ** 3. Log checksum 0. 45 ** 3. Log checksum 0. 45 ** 4. Log checksum 1. 46 ** 4. Log checksum 1. ................................................................................................................................................................................ 68 ** 9. Current pointer value (64-bits - 2 integers). 69 ** 9. Current pointer value (64-bits - 2 integers). 69 ** 70 ** 70 ** The in-memory freelist entries. Each entry is either an insert or a 71 ** The in-memory freelist entries. Each entry is either an insert or a 71 ** delete. The in-memory freelist is to the free-block-list as the 72 ** delete. The in-memory freelist is to the free-block-list as the 72 ** in-memory tree is to the users database content. 73 ** in-memory tree is to the users database content. 73 ** 74 ** 74 ** 1. Number of free-list entries stored in checkpoint header. 75 ** 1. Number of free-list entries stored in checkpoint header. > 76 ** 2. Number of free blocks (in total). > 77 ** 3. Total number of blocks freed during database lifetime. 75 ** 2. For each entry: | 78 ** 4. For each entry: 76 ** 2a. Block number of free block. 79 ** 2a. Block number of free block. 77 ** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, 80 ** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, 78 ** or the associated checkpoint id for an insert. 81 ** or the associated checkpoint id for an insert. 79 ** 82 ** 80 ** The checksum: 83 ** The checksum: 81 ** 84 ** 82 ** 1. Checksum value 1. 85 ** 1. Checksum value 1.
Changes to src/lsm_file.c
1114 int rc = LSM_OK; 1114 int rc = LSM_OK; 1115 1115 1116 assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); 1116 assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); 1117 *ppPg = 0; 1117 *ppPg = 0; 1118 1118 1119 assert( pFS->bUseMmap==0 || pFS->pCompress==0 ); 1119 assert( pFS->bUseMmap==0 || pFS->pCompress==0 ); 1120 if( pFS->bUseMmap ){ 1120 if( pFS->bUseMmap ){ > 1121 Page *pTest; 1121 i64 iEnd = (i64)iPg * pFS->nPagesize; 1122 i64 iEnd = (i64)iPg * pFS->nPagesize; 1122 fsGrowMapping(pFS, iEnd, &rc); 1123 fsGrowMapping(pFS, iEnd, &rc); 1123 if( rc!=LSM_OK ) return rc; 1124 if( rc!=LSM_OK ) return rc; 1124 1125 > 1126 p = 0; > 1127 for(pTest=pFS->pWaiting; pTest; pTest=pTest->pNextWaiting){ > 1128 if( pTest->iPg==iPg ){ > 1129 p = pTest; > 1130 p->nRef++; > 1131 *ppPg = p; > 1132 return LSM_OK; > 1133 } > 1134 } 1125 if( pFS->pFree ){ 1135 if( pFS->pFree ){ 1126 p = pFS->pFree; 1136 p = pFS->pFree; 1127 pFS->pFree = p->pHashNext; 1137 pFS->pFree = p->pHashNext; 1128 assert( p->nRef==0 ); 1138 assert( p->nRef==0 ); 1129 }else{ 1139 }else{ 1130 p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); 1140 p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); 1131 if( rc ) return rc; 1141 if( rc ) return rc; ................................................................................................................................................................................ 1974 ** lsmFsPagePersist() to write an out-of-order page. Instead a page 1984 ** lsmFsPagePersist() to write an out-of-order page. Instead a page 1975 ** number is assigned here so that the page data will be appended 1985 ** number is assigned here so that the page data will be appended 1976 ** to the current segment. 1986 ** to the current segment. 1977 */ 1987 */ 1978 Page **pp; 1988 Page **pp; 1979 int iPrev = 0; 1989 int iPrev = 0; 1980 int iNext = 0; 1990 int iNext = 0; 1981 int iHash; < 1982 1991 1983 assert( pPg->pSeg->iFirst ); 1992 assert( pPg->pSeg->iFirst ); 1984 assert( pPg->flags & PAGE_FREE ); 1993 assert( pPg->flags & PAGE_FREE ); 1985 assert( (pPg->flags & PAGE_HASPREV)==0 ); 1994 assert( (pPg->flags & PAGE_HASPREV)==0 ); 1986 assert( pPg->nData==pFS->nPagesize-4 ); 1995 assert( pPg->nData==pFS->nPagesize-4 ); 1987 1996 1988 rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); 1997 rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); 1989 if( rc!=LSM_OK ) return rc; 1998 if( rc!=LSM_OK ) return rc; 1990 1999 > 2000 if( pFS->bUseMmap==0 ){ 1991 iHash = fsHashKey(pFS->nHash, pPg->iPg); | 2001 int iHash = fsHashKey(pFS->nHash, pPg->iPg); 1992 pPg->pHashNext = pFS->apHash[iHash]; | 2002 pPg->pHashNext = pFS->apHash[iHash]; 1993 pFS->apHash[iHash] = pPg; | 2003 pFS->apHash[iHash] = pPg; > 2004 assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); > 2005 } 1994 2006 1995 if( iPrev ){ 2007 if( iPrev ){ 1996 assert( iNext==0 ); 2008 assert( iNext==0 ); 1997 memmove(&pPg->aData[4], pPg->aData, pPg->nData); 2009 memmove(&pPg->aData[4], pPg->aData, pPg->nData); 1998 lsmPutU32(pPg->aData, iPrev); 2010 lsmPutU32(pPg->aData, iPrev); 1999 pPg->flags |= PAGE_HASPREV; 2011 pPg->flags |= PAGE_HASPREV; 2000 pPg->aData += 4; 2012 pPg->aData += 4; ................................................................................................................................................................................ 2459 int rc; 2471 int rc; 2460 Freelist freelist = {0, 0, 0}; 2472 Freelist freelist = {0, 0, 0}; 2461 u8 *aUsed; 2473 u8 *aUsed; 2462 Level *pLevel; 2474 Level *pLevel; 2463 Snapshot *pWorker = pDb->pWorker; 2475 Snapshot *pWorker = pDb->pWorker; 2464 int nBlock = pWorker->nBlock; 2476 int nBlock = pWorker->nBlock; 2465 2477 > 2478 #if 0 > 2479 static int nCall = 0; > 2480 nCall++; > 2481 printf("%d calls\n", nCall); > 2482 #endif > 2483 2466 aUsed = lsmMallocZero(pDb->pEnv, nBlock); 2484 aUsed = lsmMallocZero(pDb->pEnv, nBlock); 2467 if( aUsed==0 ){ 2485 if( aUsed==0 ){ 2468 /* Malloc has failed. Since this function is only called within debug 2486 /* Malloc has failed. Since this function is only called within debug 2469 ** builds, this probably means the user is running an OOM injection test. 2487 ** builds, this probably means the user is running an OOM injection test. 2470 ** Regardless, it will not be possible to run the integrity-check at this 2488 ** Regardless, it will not be possible to run the integrity-check at this 2471 ** time, so assume the database is Ok and return non-zero. */ 2489 ** time, so assume the database is Ok and return non-zero. */ 2472 return 1; 2490 return 1;
Changes to src/lsm_main.c
421 421 422 static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ 422 static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ 423 LsmString *pStr = (LsmString *)pCtx; 423 LsmString *pStr = (LsmString *)pCtx; 424 lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); 424 lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); 425 return 0; 425 return 0; 426 } 426 } 427 427 428 int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ | 428 static int infoFreelist(lsm_db *pDb, char **pzOut){ 429 Snapshot *pWorker; /* Worker snapshot */ 429 Snapshot *pWorker; /* Worker snapshot */ 430 int bUnlock = 0; 430 int bUnlock = 0; 431 LsmString s; 431 LsmString s; 432 int i; 432 int i; 433 int rc; 433 int rc; 434 434 435 /* Obtain the worker snapshot */ 435 /* Obtain the worker snapshot */ ................................................................................................................................................................................ 444 *pzOut = s.z; 444 *pzOut = s.z; 445 } 445 } 446 446 447 /* Release the snapshot and return */ 447 /* Release the snapshot and return */ 448 infoFreeWorker(pDb, bUnlock); 448 infoFreeWorker(pDb, bUnlock); 449 return rc; 449 return rc; 450 } 450 } > 451 > 452 static int infoFreelistSize(lsm_db *pDb, int *pnFree, int *pnWaiting){ > 453 } 451 454 452 static int infoTreeSize(lsm_db *db, int *pnOld, int *pnNew){ 455 static int infoTreeSize(lsm_db *db, int *pnOld, int *pnNew){ 453 ShmHeader *pShm = db->pShmhdr; 456 ShmHeader *pShm = db->pShmhdr; 454 TreeHeader *p = &pShm->hdr1; 457 TreeHeader *p = &pShm->hdr1; 455 458 456 /* The following code suffers from two race conditions, as it accesses and 459 /* The following code suffers from two race conditions, as it accesses and 457 ** trusts the contents of shared memory without verifying checksums: 460 ** trusts the contents of shared memory without verifying checksums: ................................................................................................................................................................................ 536 char **pzVal = va_arg(ap, char **); 539 char **pzVal = va_arg(ap, char **); 537 rc = lsmInfoLogStructure(pDb, pzVal); 540 rc = lsmInfoLogStructure(pDb, pzVal); 538 break; 541 break; 539 } 542 } 540 543 541 case LSM_INFO_FREELIST: { 544 case LSM_INFO_FREELIST: { 542 char **pzVal = va_arg(ap, char **); 545 char **pzVal = va_arg(ap, char **); 543 rc = lsmInfoFreelist(pDb, pzVal); | 546 rc = infoFreelist(pDb, pzVal); 544 break; 547 break; 545 } 548 } 546 549 547 case LSM_INFO_CHECKPOINT_SIZE: { 550 case LSM_INFO_CHECKPOINT_SIZE: { 548 int *pnByte = va_arg(ap, int *); 551 int *pnByte = va_arg(ap, int *); 549 rc = lsmCheckpointSize(pDb, pnByte); 552 rc = lsmCheckpointSize(pDb, pnByte); 550 break; 553 break;
Changes to src/lsm_sorted.c
4138 assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); 4138 assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); 4139 lsmDbSnapshotSetLevel(pDb->pWorker, pNext); 4139 lsmDbSnapshotSetLevel(pDb->pWorker, pNext); 4140 sortedFreeLevel(pDb->pEnv, pNew); 4140 sortedFreeLevel(pDb->pEnv, pNew); 4141 }else{ 4141 }else{ 4142 if( pDel ) pDel->iRoot = 0; 4142 if( pDel ) pDel->iRoot = 0; 4143 4143 4144 #if 0 4144 #if 0 4145 lsmSortedDumpStructure(pDb, pDb->pWorker, 1, 1, "new-toplevel"); | 4145 lsmSortedDumpStructure(pDb, pDb->pWorker, 0, 0, "new-toplevel"); 4146 #endif 4146 #endif 4147 4147 4148 if( freelist.nEntry ){ 4148 if( freelist.nEntry ){ 4149 Freelist *p = &pDb->pWorker->freelist; 4149 Freelist *p = &pDb->pWorker->freelist; 4150 lsmFree(pDb->pEnv, p->aEntry); 4150 lsmFree(pDb->pEnv, p->aEntry); 4151 memcpy(p, &freelist, sizeof(freelist)); 4151 memcpy(p, &freelist, sizeof(freelist)); 4152 freelist.aEntry = 0; 4152 freelist.aEntry = 0; ................................................................................................................................................................................ 4577 /* Clean up the MergeWorker object initialized above. If no error 4577 /* Clean up the MergeWorker object initialized above. If no error 4578 ** has occurred, invoke the work-hook to inform the application that 4578 ** has occurred, invoke the work-hook to inform the application that 4579 ** the database structure has changed. */ 4579 ** the database structure has changed. */ 4580 mergeWorkerShutdown(&mergeworker, &rc); 4580 mergeWorkerShutdown(&mergeworker, &rc); 4581 if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); 4581 if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); 4582 4582 4583 #if 0 4583 #if 0 4584 lsmSortedDumpStructure(pDb, pDb->pWorker, 1, 1, "work"); | 4584 lsmSortedDumpStructure(pDb, pDb->pWorker, 0, 0, "work"); 4585 #endif 4585 #endif 4586 assertBtreeOk(pDb, &pLevel->lhs); 4586 assertBtreeOk(pDb, &pLevel->lhs); 4587 assertRunInOrder(pDb, &pLevel->lhs); 4587 assertRunInOrder(pDb, &pLevel->lhs); 4588 4588 4589 /* If bFlush is true and the database is no longer considered "full", 4589 /* If bFlush is true and the database is no longer considered "full", 4590 ** break out of the loop even if nRemaining is still greater than 4590 ** break out of the loop even if nRemaining is still greater than 4591 ** zero. The caller has an in-memory tree to flush to disk. */ 4591 ** zero. The caller has an in-memory tree to flush to disk. */
Changes to src/lsm_unix.c
186 off_t iSz; 186 off_t iSz; 187 int prc; 187 int prc; 188 PosixFile *p = (PosixFile *)pFile; 188 PosixFile *p = (PosixFile *)pFile; 189 struct stat buf; 189 struct stat buf; 190 190 191 if( p->pMap ){ 191 if( p->pMap ){ 192 munmap(p->pMap, p->nMap); 192 munmap(p->pMap, p->nMap); 193 p->pMap = 0; | 193 *ppOut = p->pMap = 0; 194 p->nMap = 0; | 194 *pnOut = p->nMap = 0; 195 } 195 } 196 196 197 memset(&buf, 0, sizeof(buf)); 197 memset(&buf, 0, sizeof(buf)); 198 prc = fstat(p->fd, &buf); 198 prc = fstat(p->fd, &buf); 199 if( prc!=0 ) return LSM_IOERR_BKPT; 199 if( prc!=0 ) return LSM_IOERR_BKPT; 200 iSz = buf.st_size; 200 iSz = buf.st_size; 201 if( iSz<iMin ){ 201 if( iSz<iMin ){
Changes to tool/lsmperf.tcl
186 append script $data3 186 append script $data3 187 append script $data4 187 append script $data4 188 188 189 append script "pause -1\n" 189 append script "pause -1\n" 190 exec_gnuplot_script $script $zPng 190 exec_gnuplot_script $script $zPng 191 } 191 } 192 192 193 do_write_test x.png 100 50000 50000 20 { | 193 do_write_test x.png 100 50000 0 20 { 194 lsm safety=0 | 194 lsm-mt "threads=2 multi_proc=0" > 195 leveldb leveldb 195 } 196 } 196 197 197 198 198 #lsm "mmap=1 multi_proc=0 page_size=4096 block_size=2097152 autocheckpoint=419 199 #lsm "mmap=1 multi_proc=0 page_size=4096 block_size=2097152 autocheckpoint=419 199 #lsm-mt "mmap=1 multi_proc=0 threads=2 autowork=0 autocheckpoint=4196000" 200 #lsm-mt "mmap=1 multi_proc=0 threads=2 autowork=0 autocheckpoint=4196000" 200 201 201 # lsm "safety=1 multi_proc=0" 202 # lsm "safety=1 multi_proc=0"