Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add a field to the database header to identify the compression scheme in use. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | compression-id |
| Files: | files | file ages | folders |
| SHA1: |
3bf1db970967961125b768d65e65dda8 |
| User & Date: | dan 2013-02-06 19:03:43 |
Context
|
2013-02-06
| ||
| 19:43 | Add API to register a compression-factory method with an lsm handle. check-in: 60908fd4d1 user: dan tags: compression-id | |
| 19:03 | Add a field to the database header to identify the compression scheme in use. check-in: 3bf1db9709 user: dan tags: compression-id | |
| 11:58 | Fix bug to do with block redirection. check-in: 7cc153f523 user: dan tags: trunk | |
Changes
Changes to src/lsm.h.
20 20 extern "C" { 21 21 #endif 22 22 23 23 /* 24 24 ** Opaque handle types. 25 25 */ 26 26 typedef struct lsm_compress lsm_compress; /* Compression library functions */ 27 +typedef struct lsm_compress_factory lsm_compress_factory; 27 28 typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ 28 29 typedef struct lsm_db lsm_db; /* Database connection handle */ 29 30 typedef struct lsm_env lsm_env; /* Runtime environment */ 30 31 typedef struct lsm_file lsm_file; /* OS file handle */ 31 32 typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ 32 33 33 34 /* 64-bit integer type used for file offsets. */ ................................................................................ 102 103 #define LSM_IOERR 10 103 104 #define LSM_CORRUPT 11 104 105 #define LSM_FULL 13 105 106 #define LSM_CANTOPEN 14 106 107 #define LSM_PROTOCOL 15 107 108 #define LSM_MISUSE 21 108 109 110 +#define LSM_MISMATCH 50 111 + 109 112 /* 110 113 ** CAPI: Creating and Destroying Database Connection Handles 111 114 ** 112 115 ** Open and close a database connection handle. 113 116 */ 114 117 int lsm_new(lsm_env*, lsm_db **ppDb); 115 118 int lsm_close(lsm_db *pDb); ................................................................................ 116 119 117 120 /* 118 121 ** CAPI: Connecting to a Database 119 122 */ 120 123 int lsm_open(lsm_db *pDb, const char *zFilename); 121 124 122 125 /* 123 -** CAPI: Obtaining pointers to databases environments 126 +** CAPI: Obtaining pointers to database environments 124 127 ** 125 128 ** Return a pointer to the environment used by the database connection 126 129 ** passed as the first argument. Assuming the argument is valid, this 127 130 ** function always returns a valid environment pointer - it cannot fail. 128 131 */ 129 132 lsm_env *lsm_get_env(lsm_db *pDb); 130 133 ................................................................................ 251 254 ** 252 255 ** This option may only be used before lsm_open() is called. Invoking it 253 256 ** after lsm_open() has been called results in an LSM_MISUSE error. 254 257 ** 255 258 ** LSM_CONFIG_GET_COMPRESSION: 256 259 ** Query the compression methods used to compress and decompress database 257 260 ** content. 261 +** 262 +** LSM_CONFIG_SET_COMPRESSION_FACTORY: 263 +** Configure a factory method to be invoked in case of an LSM_MISMATCH 264 +** error. 258 265 */ 259 -#define LSM_CONFIG_AUTOFLUSH 1 260 -#define LSM_CONFIG_PAGE_SIZE 2 261 -#define LSM_CONFIG_SAFETY 3 262 -#define LSM_CONFIG_BLOCK_SIZE 4 263 -#define LSM_CONFIG_AUTOWORK 5 264 -#define LSM_CONFIG_MMAP 7 265 -#define LSM_CONFIG_USE_LOG 8 266 -#define LSM_CONFIG_AUTOMERGE 9 267 -#define LSM_CONFIG_MAX_FREELIST 10 268 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 269 -#define LSM_CONFIG_AUTOCHECKPOINT 12 270 -#define LSM_CONFIG_SET_COMPRESSION 13 271 -#define LSM_CONFIG_GET_COMPRESSION 14 266 +#define LSM_CONFIG_AUTOFLUSH 1 267 +#define LSM_CONFIG_PAGE_SIZE 2 268 +#define LSM_CONFIG_SAFETY 3 269 +#define LSM_CONFIG_BLOCK_SIZE 4 270 +#define LSM_CONFIG_AUTOWORK 5 271 +#define LSM_CONFIG_MMAP 7 272 +#define LSM_CONFIG_USE_LOG 8 273 +#define LSM_CONFIG_AUTOMERGE 9 274 +#define LSM_CONFIG_MAX_FREELIST 10 275 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 276 +#define LSM_CONFIG_AUTOCHECKPOINT 12 277 +#define LSM_CONFIG_SET_COMPRESSION 13 278 +#define LSM_CONFIG_GET_COMPRESSION 14 279 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 272 280 273 281 #define LSM_SAFETY_OFF 0 274 282 #define LSM_SAFETY_NORMAL 1 275 283 #define LSM_SAFETY_FULL 2 276 284 277 285 /* 278 286 ** CAPI: Compression and/or Encryption Hooks ................................................................................ 279 287 */ 280 288 struct lsm_compress { 281 289 void *pCtx; 282 290 unsigned int iId; 283 291 int (*xBound)(void *, int nSrc); 284 292 int (*xCompress)(void *, char *, int *, const char *, int); 285 293 int (*xUncompress)(void *, char *, int *, const char *, int); 294 + void (*xFree)(void *pCtx); 295 +}; 296 + 297 +struct lsm_compress_factory { 298 + void *pCtx; 299 + int (*xFactory)(void *, lsm_db *, unsigned int); 300 + void (*xFree)(void *pCtx); 286 301 }; 302 + 303 +#define LSM_COMPRESSION_EMPTY 0 304 +#define LSM_COMPRESSION_NONE 1 287 305 288 306 289 307 /* 290 308 ** CAPI: Allocating and Freeing Memory 291 309 ** 292 310 ** Invoke the memory allocation functions that belong to environment 293 311 ** pEnv. Or the system defaults if no memory allocation functions have
Changes to src/lsmInt.h.
312 312 int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ 313 313 int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ 314 314 int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ 315 315 int bMmap; /* Configured by LSM_CONFIG_MMAP */ 316 316 i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ 317 317 int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ 318 318 lsm_compress compress; /* Compression callbacks */ 319 + lsm_compress_factory factory; /* Compression callback factory */ 319 320 320 321 /* Sub-system handles */ 321 322 FileSystem *pFS; /* On-disk portion of database */ 322 323 Database *pDatabase; /* Database shared data */ 323 324 324 325 /* Client transaction context */ 325 326 Snapshot *pClient; /* Client snapshot */ ................................................................................ 522 523 /* 523 524 ** A snapshot of a database. A snapshot contains all the information required 524 525 ** to read or write a database file on disk. See the description of struct 525 526 ** Database below for futher details. 526 527 */ 527 528 struct Snapshot { 528 529 Database *pDatabase; /* Database this snapshot belongs to */ 530 + u32 iCmpId; /* Id of compression scheme */ 529 531 Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ 530 532 i64 iId; /* Snapshot id */ 531 533 i64 iLogOff; /* Log file offset */ 532 534 Redirect redirect; /* Block redirection array */ 533 535 534 536 /* Used by worker snapshots only */ 535 537 int nBlock; /* Number of blocks in database file */ ................................................................................ 639 641 #endif 640 642 641 643 /************************************************************************** 642 644 ** Start of functions from "lsm_file.c". 643 645 */ 644 646 int lsmFsOpen(lsm_db *, const char *); 645 647 void lsmFsClose(FileSystem *); 648 + 649 +int lsmFsConfigure(lsm_db *db); 646 650 647 651 int lsmFsBlockSize(FileSystem *); 648 652 void lsmFsSetBlockSize(FileSystem *, int); 649 653 650 654 int lsmFsPageSize(FileSystem *); 651 655 void lsmFsSetPageSize(FileSystem *, int); 652 656
Changes to src/lsm_ckpt.c.
28 28 ** 29 29 ** Checkpoint header (see the CKPT_HDR_XXX #defines): 30 30 ** 31 31 ** 1. The checkpoint id MSW. 32 32 ** 2. The checkpoint id LSW. 33 33 ** 3. The number of integer values in the entire checkpoint, including 34 34 ** the two checksum values. 35 -** 4. The total number of blocks in the database. 36 -** 5. The block size. 37 -** 6. The number of levels. 38 -** 7. The nominal database page size. 39 -** 8. The number of pages (in total) written to the database file. 35 +** 4. The compression scheme id. 36 +** 5. The total number of blocks in the database. 37 +** 6. The block size. 38 +** 7. The number of levels. 39 +** 8. The nominal database page size. 40 +** 9. The number of pages (in total) written to the database file. 40 41 ** 41 42 ** Log pointer: 42 43 ** 43 44 ** 1. The log offset MSW. 44 45 ** 2. The log offset LSW. 45 46 ** 3. Log checksum 0. 46 47 ** 4. Log checksum 1. ................................................................................ 170 171 + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ 171 172 ) 172 173 173 174 static const int one = 1; 174 175 #define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) 175 176 176 177 /* Sizes, in integers, of various parts of the checkpoint. */ 177 -#define CKPT_HDR_SIZE 8 178 +#define CKPT_HDR_SIZE 9 178 179 #define CKPT_LOGPTR_SIZE 4 179 180 #define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) 180 181 181 182 /* A #define to describe each integer in the checkpoint header. */ 182 183 #define CKPT_HDR_ID_MSW 0 183 184 #define CKPT_HDR_ID_LSW 1 184 185 #define CKPT_HDR_NCKPT 2 185 -#define CKPT_HDR_NBLOCK 3 186 -#define CKPT_HDR_BLKSZ 4 187 -#define CKPT_HDR_NLEVEL 5 188 -#define CKPT_HDR_PGSZ 6 189 -#define CKPT_HDR_NWRITE 7 186 +#define CKPT_HDR_CMPID 3 187 +#define CKPT_HDR_NBLOCK 4 188 +#define CKPT_HDR_BLKSZ 5 189 +#define CKPT_HDR_NLEVEL 6 190 +#define CKPT_HDR_PGSZ 7 191 +#define CKPT_HDR_NWRITE 8 190 192 191 -#define CKPT_HDR_LO_MSW 8 192 -#define CKPT_HDR_LO_LSW 9 193 -#define CKPT_HDR_LO_CKSUM1 10 194 -#define CKPT_HDR_LO_CKSUM2 11 193 +#define CKPT_HDR_LO_MSW 9 194 +#define CKPT_HDR_LO_LSW 10 195 +#define CKPT_HDR_LO_CKSUM1 11 196 +#define CKPT_HDR_LO_CKSUM2 12 195 197 196 198 typedef struct CkptBuffer CkptBuffer; 197 199 198 200 /* 199 201 ** Dynamic buffer used to accumulate data for a checkpoint. 200 202 */ 201 203 struct CkptBuffer { ................................................................................ 448 450 } 449 451 450 452 /* Write the checkpoint header */ 451 453 assert( iId>=0 ); 452 454 ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); 453 455 ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); 454 456 ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); 457 + ckptSetValue(&ckpt, CKPT_HDR_CMPID, pSnap->iCmpId, &rc); 455 458 ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); 456 459 ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); 457 460 ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); 458 461 ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); 459 462 ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); 460 463 461 464 if( bCksum ){ ................................................................................ 757 760 758 761 /* 759 762 ** Initialize the shared-memory header with an empty snapshot. This function 760 763 ** is called when no valid snapshot can be found in the database header. 761 764 */ 762 765 static void ckptLoadEmpty(lsm_db *pDb){ 763 766 u32 aCkpt[] = { 764 - 0, /* CKPT_HDR_ID_MSW */ 765 - 10, /* CKPT_HDR_ID_LSW */ 766 - 0, /* CKPT_HDR_NCKPT */ 767 - 0, /* CKPT_HDR_NBLOCK */ 768 - 0, /* CKPT_HDR_BLKSZ */ 769 - 0, /* CKPT_HDR_NLEVEL */ 770 - 0, /* CKPT_HDR_PGSZ */ 771 - 0, /* CKPT_HDR_OVFL */ 772 - 0, /* CKPT_HDR_NWRITE */ 773 - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ 774 - 0,0,0,0, 0,0,0,0, /* The append list */ 775 - 0, /* The free block list */ 776 - 0, 0 /* Space for checksum values */ 767 + 0, /* CKPT_HDR_ID_MSW */ 768 + 10, /* CKPT_HDR_ID_LSW */ 769 + 0, /* CKPT_HDR_NCKPT */ 770 + LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ 771 + 0, /* CKPT_HDR_NBLOCK */ 772 + 0, /* CKPT_HDR_BLKSZ */ 773 + 0, /* CKPT_HDR_NLEVEL */ 774 + 0, /* CKPT_HDR_PGSZ */ 775 + 0, /* CKPT_HDR_OVFL */ 776 + 0, /* CKPT_HDR_NWRITE */ 777 + 0, 0, 1234, 5678, /* The log pointer and initial checksum */ 778 + 0,0,0,0, 0,0,0,0, /* The append list */ 779 + 0, /* The free block list */ 780 + 0, 0 /* Space for checksum values */ 777 781 }; 778 782 u32 nCkpt = array_size(aCkpt); 779 783 ShmHeader *pShm = pDb->pShmhdr; 780 784 781 785 aCkpt[CKPT_HDR_NCKPT] = nCkpt; 782 786 aCkpt[CKPT_HDR_BLKSZ] = pDb->nDfltBlksz; 783 787 aCkpt[CKPT_HDR_PGSZ] = pDb->nDfltPgsz; ................................................................................ 917 921 }else{ 918 922 return LSM_PROTOCOL; 919 923 } 920 924 } 921 925 922 926 rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); 923 927 if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; 928 + 929 + if( rc==LSM_OK && pDb->pWorker->iCmpId!=pDb->compress.iId ){ 930 + lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); 931 + rc = LSM_MISMATCH; 932 + pDb->pWorker = 0; 933 + } 924 934 925 935 #if 0 926 936 assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); 927 937 #endif 928 938 return rc; 929 939 } 930 940 ................................................................................ 946 956 int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; 947 957 948 958 pNew->iId = lsmCheckpointId(aCkpt, 0); 949 959 pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; 950 960 pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; 951 961 rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); 952 962 pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); 963 + 964 + pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; 965 + if( pNew->iCmpId==LSM_COMPRESSION_EMPTY ){ 966 + pNew->iCmpId = pDb->compress.iId; 967 + } 953 968 954 969 /* Make a copy of the append-list */ 955 970 for(i=0; i<LSM_APPLIST_SZ; i++){ 956 971 u32 *a = &aCkpt[CKPT_HDR_SIZE + CKPT_LOGPTR_SIZE + i*2]; 957 972 pNew->aiAppend[i] = ckptRead64(a); 958 973 } 959 974
Changes to src/lsm_file.c.
506 506 pFS->zDb = (char *)&pFS[1]; 507 507 pFS->zLog = &pFS->zDb[nDb+1]; 508 508 pFS->nPagesize = LSM_DFLT_PAGE_SIZE; 509 509 pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; 510 510 pFS->nMetasize = 4 * 1024; 511 511 pFS->pDb = pDb; 512 512 pFS->pEnv = pDb->pEnv; 513 - if( pDb->compress.xCompress ){ 514 - pFS->pCompress = &pDb->compress; 515 - }else{ 516 - pFS->bUseMmap = pDb->bMmap; 517 - } 518 513 519 514 /* Make a copy of the database and log file names. */ 520 515 memcpy(pFS->zDb, zDb, nDb+1); 521 516 memcpy(pFS->zLog, zDb, nDb); 522 517 memcpy(&pFS->zLog[nDb], "-log", 5); 523 518 524 519 /* Allocate the hash-table here. At some point, it should be changed ................................................................................ 547 542 pFS->szSector = lsmEnvSectorSize(pFS->pEnv, pFS->fdDb); 548 543 } 549 544 } 550 545 551 546 pDb->pFS = pFS; 552 547 return rc; 553 548 } 549 + 550 +/* 551 +** Configure the file-system object according to the current values of 552 +** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. 553 +*/ 554 +int lsmFsConfigure(lsm_db *db){ 555 + FileSystem *pFS = db->pFS; 556 + lsm_env *pEnv = pFS->pEnv; 557 + Page *pPg; 558 + 559 + assert( pFS->nOut==0 ); 560 + assert( pFS->pWaiting==0 ); 561 + 562 + /* Reset any compression/decompression buffers already allocated */ 563 + lsmFree(pEnv, pFS->aIBuffer); 564 + lsmFree(pEnv, pFS->aOBuffer); 565 + pFS->nBuffer = 0; 566 + 567 + /* Unmap the file, if it is currently mapped */ 568 + if( pFS->pMap ){ 569 + lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); 570 + pFS->bUseMmap = 0; 571 + } 572 + 573 + /* Free all allocate page structures */ 574 + pPg = pFS->pLruFirst; 575 + while( pPg ){ 576 + Page *pNext = pPg->pLruNext; 577 + if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); 578 + lsmFree(pEnv, pPg); 579 + pPg = pNext; 580 + } 581 + 582 + /* Zero pointers that point to deleted page objects */ 583 + pFS->nCacheAlloc = 0; 584 + pFS->pLruFirst = 0; 585 + pFS->pLruLast = 0; 586 + pFS->pFree = 0; 587 + 588 + /* Configure the FileSystem object */ 589 + if( db->compress.xCompress ){ 590 + pFS->pCompress = &db->compress; 591 + pFS->bUseMmap = 0; 592 + }else{ 593 + pFS->pCompress = 0; 594 + pFS->bUseMmap = db->bMmap; 595 + } 596 + 597 + return LSM_OK; 598 +} 554 599 555 600 /* 556 601 ** Close and destroy a FileSystem object. 557 602 */ 558 603 void lsmFsClose(FileSystem *pFS){ 559 604 if( pFS ){ 560 605 Page *pPg;
Changes to src/lsm_main.c.
92 92 pDb->nMerge = LSM_DFLT_AUTOMERGE; 93 93 pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; 94 94 pDb->bUseLog = LSM_DFLT_USE_LOG; 95 95 pDb->iReader = -1; 96 96 pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; 97 97 pDb->bMmap = LSM_DFLT_MMAP; 98 98 pDb->xLog = xLog; 99 + pDb->compress.iId = LSM_COMPRESSION_NONE; 99 100 return LSM_OK; 100 101 } 101 102 102 103 lsm_env *lsm_get_env(lsm_db *pDb){ 103 104 assert( pDb->pEnv ); 104 105 return pDb->pEnv; 105 106 } ................................................................................ 190 191 rc = LSM_MISUSE_BKPT; 191 192 }else{ 192 193 lsmFreeSnapshot(pDb->pEnv, pDb->pClient); 193 194 pDb->pClient = 0; 194 195 lsmDbDatabaseRelease(pDb); 195 196 lsmLogClose(pDb); 196 197 lsmFsClose(pDb->pFS); 198 + 199 + /* Invoke any destructors registered for the compression or 200 + ** compression factory callbacks. */ 201 + if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); 202 + if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); 203 + 197 204 lsmFree(pDb->pEnv, pDb->rollback.aArray); 198 205 lsmFree(pDb->pEnv, pDb->aTrans); 199 206 lsmFree(pDb->pEnv, pDb->apShm); 200 207 lsmFree(pDb->pEnv, pDb); 201 208 } 202 209 } 203 210 return rc; ................................................................................ 333 340 pDb->bMultiProc = *piVal = (*piVal!=0); 334 341 } 335 342 break; 336 343 } 337 344 338 345 case LSM_CONFIG_SET_COMPRESSION: { 339 346 lsm_compress *p = va_arg(ap, lsm_compress *); 340 - if( pDb->pDatabase ){ 341 - /* If lsm_open() has been called, this call is against the rules. */ 347 + if( pDb->iReader>=0 ){ 348 + /* May not change compression schemes with an open transaction */ 342 349 rc = LSM_MISUSE_BKPT; 343 350 }else{ 344 - memcpy(&pDb->compress, p, sizeof(lsm_compress)); 351 + if( p->xBound==0 ){ 352 + memset(&pDb->compress, 0, sizeof(lsm_compress)); 353 + pDb->compress.iId = LSM_COMPRESSION_NONE; 354 + }else{ 355 + memcpy(&pDb->compress, p, sizeof(lsm_compress)); 356 + } 357 + rc = lsmFsConfigure(pDb); 358 + } 359 + break; 360 + } 361 + 362 + case LSM_CONFIG_SET_COMPRESSION_FACTORY: { 363 + lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); 364 + if( pDb->factory.xFree ){ 365 + /* Invoke any destructor belonging to the current factory. */ 366 + pDb->factory.xFree(pDb->factory.pCtx); 345 367 } 368 + memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); 346 369 break; 347 370 } 348 371 349 372 case LSM_CONFIG_GET_COMPRESSION: { 350 373 lsm_compress *p = va_arg(ap, lsm_compress *); 351 374 memcpy(p, &pDb->compress, sizeof(lsm_compress)); 352 375 break;
Changes to src/lsm_shared.c.
430 430 if( rc==LSM_OK ){ 431 431 assert( p ); 432 432 rc = lsmFsOpen(pDb, zName); 433 433 } 434 434 if( rc==LSM_OK ){ 435 435 rc = doDbConnect(pDb); 436 436 } 437 + if( rc==LSM_OK ){ 438 + rc = lsmFsConfigure(pDb); 439 + } 437 440 438 441 return rc; 439 442 } 440 443 441 444 static void dbDeferClose(lsm_db *pDb){ 442 445 if( pDb->pFS ){ 443 446 LsmFile *pLsmFile = 0; ................................................................................ 971 974 ** lsm_sorted.c is changed to work directly from the serialized 972 975 ** version of the snapshot. */ 973 976 if( pDb->pClient==0 ){ 974 977 rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); 975 978 } 976 979 assert( (rc==LSM_OK)==(pDb->pClient!=0) ); 977 980 assert( pDb->iReader>=0 ); 981 + 982 + /* Check that the client has the right compression hooks loaded. 983 + ** If not, set rc to LSM_MISMATCH. */ 984 + assert( rc!=LSM_OK || pDb->pClient->iCmpId!=LSM_COMPRESSION_EMPTY ); 985 + if( rc==LSM_OK && pDb->pClient->iCmpId!=pDb->compress.iId ){ 986 + rc = LSM_MISMATCH; 987 + } 978 988 }else{ 979 989 rc = lsmReleaseReadlock(pDb); 980 990 } 981 991 } 992 + 982 993 if( rc==LSM_BUSY ){ 983 994 rc = LSM_OK; 984 995 } 985 996 } 986 997 #if 0 987 998 if( rc==LSM_OK && pDb->pClient ){ 988 999 fprintf(stderr,
Changes to src/lsm_unix.c.
190 190 191 191 if( p->pMap ){ 192 192 munmap(p->pMap, p->nMap); 193 193 *ppOut = p->pMap = 0; 194 194 *pnOut = p->nMap = 0; 195 195 } 196 196 197 - memset(&buf, 0, sizeof(buf)); 198 - prc = fstat(p->fd, &buf); 199 - if( prc!=0 ) return LSM_IOERR_BKPT; 200 - iSz = buf.st_size; 201 - if( iSz<iMin ){ 202 - iSz = ((iMin + (2<<20) - 1) / (2<<20)) * (2<<20); 203 - prc = ftruncate(p->fd, iSz); 197 + if( iMin>=0 ){ 198 + memset(&buf, 0, sizeof(buf)); 199 + prc = fstat(p->fd, &buf); 204 200 if( prc!=0 ) return LSM_IOERR_BKPT; 201 + iSz = buf.st_size; 202 + if( iSz<iMin ){ 203 + iSz = ((iMin + (2<<20) - 1) / (2<<20)) * (2<<20); 204 + prc = ftruncate(p->fd, iSz); 205 + if( prc!=0 ) return LSM_IOERR_BKPT; 206 + } 207 + 208 + p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); 209 + p->nMap = iSz; 205 210 } 206 211 207 - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); 208 - p->nMap = iSz; 209 - 210 212 *ppOut = p->pMap; 211 213 *pnOut = p->nMap; 212 214 return LSM_OK; 213 215 } 214 216 215 217 static int lsmPosixOsFullpath( 216 218 lsm_env *pEnv,
Added test/lsm4.test.
1 +# 2013 February 06 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 + 13 +set testdir [file dirname $argv0] 14 +source $testdir/tester.tcl 15 +set testprefix lsm4 16 +db close 17 + 18 +proc db_fetch {db key} { 19 + db csr_open csr 20 + csr seek $key eq 21 + set ret [csr value] 22 + csr close 23 + set ret 24 +} 25 + 26 +do_test 1.1 { 27 + lsm_open db test.db 28 + db config {set_compression noop} 29 + db write 1 abc 30 + db write 2 def 31 + db close 32 +} {} 33 + 34 +do_test 1.2 { 35 + lsm_open db test.db 36 + db config {set_compression noop} 37 + list [db_fetch db 1] [db_fetch db 2] 38 +} {abc def} 39 + 40 +do_test 1.3 { 41 + db close 42 + lsm_open db test.db 43 + db config {set_compression rle} 44 + list [catch {db_fetch db 1} msg] $msg 45 +} {1 {error in lsm_csr_open() - 50}} 46 + 47 +do_test 1.4 { 48 + db close 49 + lsm_open db test.db 50 +breakpoint 51 + list [catch {db_fetch db 1} msg] $msg 52 +} {1 {error in lsm_csr_open() - 50}} 53 + 54 + 55 + 56 +finish_test 57 +
Changes to test/test_lsm.c.
5 5 ** a legal notice, here is a blessing: 6 6 ** 7 7 ** May you do good and not evil. 8 8 ** May you find forgiveness for yourself and forgive others. 9 9 ** May you share freely, never taking more than you give. 10 10 ** 11 11 ************************************************************************* 12 -** 13 12 */ 14 13 15 14 #include <tcl.h> 16 15 #include "lsm.h" 17 16 #include "sqlite4.h" 18 17 #include <assert.h> 19 18 #include <string.h> 20 19 21 20 extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite4 **ppDb); 22 21 extern const char *sqlite4TestErrorName(int); 22 + 23 +/************************************************************************* 24 +*/ 25 +#define ENCRYPTION_XOR_MASK 0xa3b2bbb6 26 +static int testCompressEncBound(void *pCtx, int nSrc){ 27 + return nSrc; 28 +} 29 +static int testCompressEncCompress( 30 + void *pCtx, 31 + char *pOut, int *pnOut, 32 + const char *pIn, int nIn 33 +){ 34 + int i; 35 + unsigned int *aIn = (unsigned int *)pOut; 36 + unsigned int *aOut = (unsigned int *)pIn; 37 + 38 + assert( (nIn%4)==0 ); 39 + for(i=0; i<(nIn/4); i++){ 40 + aOut[i] = (aIn[i] ^ ENCRYPTION_XOR_MASK); 41 + } 42 + *pnOut = nIn; 43 + 44 + return LSM_OK; 45 +} 46 +static int testCompressEncUncompress( 47 + void *pCtx, 48 + char *pOut, int *pnOut, 49 + const char *pIn, int nIn 50 +){ 51 + return testCompressEncUncompress(pCtx, pOut, pnOut, pIn, nIn); 52 +} 53 +static void testCompressEncFree(void *pCtx){ 54 + /* no-op */ 55 +} 56 +/* 57 +** End of compression routines "encrypt". 58 +*************************************************************************/ 59 + 60 +/************************************************************************* 61 +*/ 62 +static int testCompressRleBound(void *pCtx, int nSrc){ 63 + return nSrc*2; 64 +} 65 +static int testCompressRleCompress( 66 + void *pCtx, 67 + char *pOut, int *pnOut, 68 + const char *pIn, int nIn 69 +){ 70 + int iOut = 0; 71 + int i; 72 + char c; 73 + int n; 74 + 75 + c = pIn[0]; 76 + n = 1; 77 + for(i=1; i<nIn; i++){ 78 + if( pIn[i]==c && n<127 ){ 79 + n++; 80 + }else{ 81 + pOut[iOut++] = c; 82 + pOut[iOut++] = (char)n; 83 + c = pIn[i]; 84 + n = 1; 85 + } 86 + } 87 + 88 + pOut[iOut++] = c; 89 + pOut[iOut++] = (char)n; 90 + *pnOut = iOut; 91 + 92 + return LSM_OK; 93 +} 94 +static int testCompressRleUncompress( 95 + void *pCtx, 96 + char *pOut, int *pnOut, 97 + const char *pIn, int nIn 98 +){ 99 + int i; 100 + int iOut = 0; 101 + 102 + for(i=0; i<nIn; i+=2){ 103 + int iRep; 104 + char c = pIn[i]; 105 + int n = (int)(pIn[i+1]); 106 + 107 + for(iRep=0; iRep<n; iRep++){ 108 + pOut[iOut++] = c; 109 + } 110 + } 111 + 112 + *pnOut = iOut; 113 + return LSM_OK; 114 +} 115 +static void testCompressRleFree(void *pCtx){ 116 +} 117 +/* 118 +** End of compression routines "rle". 119 +*************************************************************************/ 120 + 121 +/************************************************************************* 122 +*/ 123 +static int testCompressNoopBound(void *pCtx, int nSrc){ 124 + return nSrc; 125 +} 126 +static int testCompressNoopCompress( 127 + void *pCtx, 128 + char *pOut, int *pnOut, 129 + const char *pIn, int nIn 130 +){ 131 + *pnOut = nIn; 132 + memcpy(pOut, pIn, nIn); 133 + return LSM_OK; 134 +} 135 +static int testCompressNoopUncompress( 136 + void *pCtx, 137 + char *pOut, int *pnOut, 138 + const char *pIn, int nIn 139 +){ 140 + *pnOut = nIn; 141 + memcpy(pOut, pIn, nIn); 142 + return LSM_OK; 143 +} 144 +static void testCompressNoopFree(void *pCtx){ 145 +} 146 +/* 147 +** End of compression routines "noop". 148 +*************************************************************************/ 23 149 24 150 /* 25 151 ** TCLCMD: sqlite4_lsm_config DB DBNAME PARAM ... 26 152 */ 27 153 static int test_sqlite4_lsm_config( 28 154 void * clientData, 29 155 Tcl_Interp *interp, ................................................................................ 30 156 int objc, 31 157 Tcl_Obj *CONST objv[] 32 158 ){ 33 159 struct Switch { 34 160 const char *zSwitch; 35 161 int iVal; 36 162 } aParam[] = { 37 - { "log-size", LSM_CONFIG_LOG_SIZE }, 38 163 { "safety", LSM_CONFIG_SAFETY }, 39 164 { "autoflush", LSM_CONFIG_AUTOFLUSH }, 40 165 { "mmap", LSM_CONFIG_MMAP }, 41 166 { "page-size", LSM_CONFIG_PAGE_SIZE }, 42 167 { "autowork", LSM_CONFIG_AUTOWORK }, 43 168 { 0, 0 } 44 169 }; ................................................................................ 282 407 Tcl_SetResult(interp, (char *)sqlite4TestErrorName(rc), TCL_STATIC); 283 408 return TCL_ERROR; 284 409 } 285 410 286 411 Tcl_ResetResult(interp); 287 412 return TCL_OK; 288 413 } 414 + 415 +static int testConfigureSetCompression( 416 + Tcl_Interp *interp, 417 + lsm_db *db, 418 + Tcl_Obj *pCmp 419 +){ 420 + struct CompressionScheme { 421 + const char *zName; 422 + lsm_compress cmp; 423 + } aCmp[] = { 424 + { "encrypt", { 0, 43, 425 + testCompressEncBound, testCompressEncCompress, 426 + testCompressEncUncompress, testCompressEncFree 427 + } }, 428 + { "rle", { 0, 44, 429 + testCompressRleBound, testCompressRleCompress, 430 + testCompressRleUncompress, testCompressRleFree 431 + } }, 432 + { "noop", { 0, 45, 433 + testCompressNoopBound, testCompressNoopCompress, 434 + testCompressNoopUncompress, testCompressNoopFree 435 + } }, 436 + { 0, {0, 0, 0, 0, 0, 0} } 437 + }; 438 + int iOpt; 439 + int rc; 440 + 441 + rc = Tcl_GetIndexFromObjStruct( 442 + interp, pCmp, aCmp, sizeof(aCmp[0]), "scheme", 0, &iOpt 443 + ); 444 + if( rc!=TCL_OK ) return rc; 445 + 446 + rc = lsm_config(db, LSM_CONFIG_SET_COMPRESSION, &aCmp[iOpt].cmp); 447 + 448 + return rc; 449 +} 289 450 290 451 static int testConfigureLsm(Tcl_Interp *interp, lsm_db *db, Tcl_Obj *pObj){ 291 452 struct Lsmconfig { 292 453 const char *zOpt; 293 454 int eOpt; 294 455 } aConfig[] = { 295 456 { "autoflush", LSM_CONFIG_AUTOFLUSH }, 296 457 { "page_size", LSM_CONFIG_PAGE_SIZE }, 297 458 { "block_size", LSM_CONFIG_BLOCK_SIZE }, 298 459 { "safety", LSM_CONFIG_SAFETY }, 299 460 { "autowork", LSM_CONFIG_AUTOWORK }, 300 461 { "autocheckpoint", LSM_CONFIG_AUTOCHECKPOINT }, 301 - { "log_size", LSM_CONFIG_LOG_SIZE }, 302 462 { "mmap", LSM_CONFIG_MMAP }, 303 463 { "use_log", LSM_CONFIG_USE_LOG }, 304 464 { "automerge", LSM_CONFIG_AUTOMERGE }, 305 465 { "max_freelist", LSM_CONFIG_MAX_FREELIST }, 306 466 { "multi_proc", LSM_CONFIG_MULTIPLE_PROCESSES }, 467 + { "set_compression", LSM_CONFIG_SET_COMPRESSION }, 307 468 { 0, 0 } 308 469 }; 309 470 int nElem; 310 471 int i; 311 472 Tcl_Obj **apElem; 312 473 int rc; 313 474 ................................................................................ 321 482 if( i==(nElem-1) ){ 322 483 Tcl_ResetResult(interp); 323 484 Tcl_AppendResult(interp, "option \"", Tcl_GetString(apElem[i]), 324 485 "\" requires an argument", 0 325 486 ); 326 487 rc = TCL_ERROR; 327 488 }else{ 328 - int iVal; 329 - rc = Tcl_GetIntFromObj(interp, apElem[i+1], &iVal); 330 - if( rc==TCL_OK ){ 331 - lsm_config(db, aConfig[iOpt].eOpt, &iVal); 489 + if( aConfig[iOpt].eOpt==LSM_CONFIG_SET_COMPRESSION ){ 490 + rc = testConfigureSetCompression(interp, db, apElem[i+1]); 491 + } 492 + else { 493 + int iVal; 494 + rc = Tcl_GetIntFromObj(interp, apElem[i+1], &iVal); 495 + if( rc==TCL_OK ){ 496 + lsm_config(db, aConfig[iOpt].eOpt, &iVal); 497 + } 498 + Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); 332 499 } 333 500 } 334 501 } 335 502 } 336 503 337 504 return rc; 338 505 } ................................................................................ 619 786 return TCL_OK; 620 787 } 621 788 622 789 case 8: assert( 0==strcmp(aCmd[8].zCmd, "work") ); { 623 790 int nWork = 0; 624 791 int nMerge = 1; 625 792 int nWrite = 0; 626 - int i; 627 793 628 794 if( objc==3 ){ 629 795 rc = Tcl_GetIntFromObj(interp, objv[2], &nWork); 630 796 }else if( objc==4 ){ 631 797 rc = Tcl_GetIntFromObj(interp, objv[2], &nMerge); 632 798 if( rc!=TCL_OK ) return rc; 633 799 rc = Tcl_GetIntFromObj(interp, objv[3], &nWork);
Changes to www/lsmapi.wiki.
17 17 18 18 <h1>LSM API Topics</h1> 19 19 <ol> 20 20 <li><a href="#database" style=text-decoration:none>Database Runtime Environment</a> 21 21 <li><a href="#lsm" style=text-decoration:none>LSM Error Codes</a> 22 22 <li><a href="#creating" style=text-decoration:none>Creating and Destroying Database Connection Handles</a> 23 23 <li><a href="#connecting" style=text-decoration:none>Connecting to a Database</a> 24 -<li><a href="#obtaining" style=text-decoration:none>Obtaining pointers to databases environments</a> 24 +<li><a href="#obtaining" style=text-decoration:none>Obtaining pointers to database environments</a> 25 25 <li><a href="#configuring" style=text-decoration:none>Configuring a database connection.</a> 26 26 <li><a href="#compression" style=text-decoration:none>Compression and/or Encryption Hooks</a> 27 27 <li><a href="#allocating" style=text-decoration:none>Allocating and Freeing Memory</a> 28 28 <li><a href="#querying" style=text-decoration:none>Querying a Connection For Operational Data</a> 29 29 <li><a href="#opening" style=text-decoration:none>Opening and Closing Write Transactions</a> 30 30 <li><a href="#writing" style=text-decoration:none>Writing to a Database</a> 31 31 <li><a href="#explicit" style=text-decoration:none>Explicit Database Work and Checkpointing</a> ................................................................................ 63 63 <span style=display:block;float:left;width:35ex><a href=#lsm_new>lsm_new</a></span> 64 64 <span style=display:block;float:left;width:35ex><a href=#lsm_open>lsm_open</a></span> 65 65 <span style=display:block;float:left;width:35ex><a href=#lsm_rollback>lsm_rollback</a></span> 66 66 <span style=display:block;float:left;width:35ex><a href=#lsm_work>lsm_work</a></span> 67 67 <br style=clear:both> 68 68 <h1 style=clear:both>All LSM API Types</h1> 69 69 <span style=display:block;float:left;width:35ex><a href=#lsm_compress>lsm_compress</a></span> 70 +<span style=display:block;float:left;width:35ex><a href=#lsm_compress>lsm_compress</a></span> 70 71 <span style=display:block;float:left;width:35ex><a href=#lsm_env>lsm_env</a></span> 71 72 <br style=clear:both> 72 73 <h1>All LSM API Constants</h1> 73 74 <span style=display:block;float:left;width:35ex><a href=#LSM_BUSY>LSM_BUSY</a></span> 74 75 <span style=display:block;float:left;width:35ex><a href=#LSM_CANTOPEN>LSM_CANTOPEN</a></span> 75 76 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOCHECKPOINT>LSM_CONFIG_AUTOCHECKPOINT</a></span> 76 77 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOFLUSH>LSM_CONFIG_AUTOFLUSH</a></span> 77 78 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOMERGE>LSM_CONFIG_AUTOMERGE</a></span> 78 79 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOWORK>LSM_CONFIG_AUTOWORK</a></span> 79 80 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_BLOCK_SIZE>LSM_CONFIG_BLOCK_SIZE</a></span> 80 81 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_GET_COMPRESSION>LSM_CONFIG_GET_COMPRESSION</a></span> 81 -<span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_LOG_SIZE>LSM_CONFIG_LOG_SIZE</a></span> 82 82 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_MAX_FREELIST>LSM_CONFIG_MAX_FREELIST</a></span> 83 83 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_MMAP>LSM_CONFIG_MMAP</a></span> 84 84 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_MULTIPLE_PROCESSES>LSM_CONFIG_MULTIPLE_PROCESSES</a></span> 85 85 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_PAGE_SIZE>LSM_CONFIG_PAGE_SIZE</a></span> 86 86 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_SAFETY>LSM_CONFIG_SAFETY</a></span> 87 87 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_SET_COMPRESSION>LSM_CONFIG_SET_COMPRESSION</a></span> 88 +<span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_SET_COMPRESSION_FACTORY>LSM_CONFIG_SET_COMPRESSION_FACTORY</a></span> 88 89 <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_USE_LOG>LSM_CONFIG_USE_LOG</a></span> 89 90 <span style=display:block;float:left;width:35ex><a href=#LSM_CORRUPT>LSM_CORRUPT</a></span> 90 91 <span style=display:block;float:left;width:35ex><a href=#LSM_ERROR>LSM_ERROR</a></span> 91 92 <span style=display:block;float:left;width:35ex><a href=#LSM_FULL>LSM_FULL</a></span> 92 93 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_ARRAY_PAGES>LSM_INFO_ARRAY_PAGES</a></span> 93 94 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_ARRAY_STRUCTURE>LSM_INFO_ARRAY_STRUCTURE</a></span> 94 95 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_CHECKPOINT_SIZE>LSM_INFO_CHECKPOINT_SIZE</a></span> 95 96 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_DB_STRUCTURE>LSM_INFO_DB_STRUCTURE</a></span> 96 97 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_FREELIST>LSM_INFO_FREELIST</a></span> 98 +<span style=display:block;float:left;width:35ex><a href=#LSM_INFO_FREELIST_SIZE>LSM_INFO_FREELIST_SIZE</a></span> 97 99 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_LOG_STRUCTURE>LSM_INFO_LOG_STRUCTURE</a></span> 98 100 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_NREAD>LSM_INFO_NREAD</a></span> 99 101 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_NWRITE>LSM_INFO_NWRITE</a></span> 100 102 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_PAGE_ASCII_DUMP>LSM_INFO_PAGE_ASCII_DUMP</a></span> 101 103 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_PAGE_HEX_DUMP>LSM_INFO_PAGE_HEX_DUMP</a></span> 102 104 <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_TREE_SIZE>LSM_INFO_TREE_SIZE</a></span> 103 105 <span style=display:block;float:left;width:35ex><a href=#LSM_IOERR>LSM_IOERR</a></span> ................................................................................ 178 180 <verbatim>int lsm_new(lsm_env*, lsm_db **ppDb); 179 181 int lsm_close(lsm_db *pDb); 180 182 </verbatim> 181 183 <p>Open and close a database connection handle. 182 184 <h2 id=connecting>Connecting to a Database<a id=lsm_open></a></h2> 183 185 <verbatim>int lsm_open(lsm_db *pDb, const char *zFilename); 184 186 </verbatim> 185 -<h2 id=obtaining>Obtaining pointers to databases environments</h2> 187 +<h2 id=obtaining>Obtaining pointers to database environments</h2> 186 188 <verbatim>lsm_env *lsm_get_env(lsm_db *pDb); 187 189 lsm_env *lsm_default_env(void); 188 190 </verbatim> 189 191 <p>Return a pointer to the environment used by the database connection 190 192 passed as the first argument. Assuming the argument is valid, this 191 193 function always returns a valid environment pointer - it cannot fail. 192 194 The lsm_default_env() function returns a pointer to the default LSM 193 195 environment for the current platform. 194 -<h2 id=configuring>Configuring a database connection.<a id=lsm_config></a><a id=LSM_CONFIG_AUTOFLUSH></a><a id=LSM_CONFIG_PAGE_SIZE></a><a id=LSM_CONFIG_SAFETY></a><a id=LSM_CONFIG_BLOCK_SIZE></a><a id=LSM_CONFIG_AUTOWORK></a><a id=LSM_CONFIG_LOG_SIZE></a><a id=LSM_CONFIG_MMAP></a><a id=LSM_CONFIG_USE_LOG></a><a id=LSM_CONFIG_AUTOMERGE></a><a id=LSM_CONFIG_MAX_FREELIST></a><a id=LSM_CONFIG_MULTIPLE_PROCESSES></a><a id=LSM_CONFIG_AUTOCHECKPOINT></a><a id=LSM_CONFIG_SET_COMPRESSION></a><a id=LSM_CONFIG_GET_COMPRESSION></a><a id=LSM_SAFETY_OFF></a><a id=LSM_SAFETY_NORMAL></a><a id=LSM_SAFETY_FULL></a></h2> 196 +<h2 id=configuring>Configuring a database connection.<a id=lsm_config></a><a id=LSM_CONFIG_AUTOFLUSH></a><a id=LSM_CONFIG_PAGE_SIZE></a><a id=LSM_CONFIG_SAFETY></a><a id=LSM_CONFIG_BLOCK_SIZE></a><a id=LSM_CONFIG_AUTOWORK></a><a id=LSM_CONFIG_MMAP></a><a id=LSM_CONFIG_USE_LOG></a><a id=LSM_CONFIG_AUTOMERGE></a><a id=LSM_CONFIG_MAX_FREELIST></a><a id=LSM_CONFIG_MULTIPLE_PROCESSES></a><a id=LSM_CONFIG_AUTOCHECKPOINT></a><a id=LSM_CONFIG_SET_COMPRESSION></a><a id=LSM_CONFIG_GET_COMPRESSION></a><a id=LSM_CONFIG_SET_COMPRESSION_FACTORY></a><a id=LSM_SAFETY_OFF></a><a id=LSM_SAFETY_NORMAL></a><a id=LSM_SAFETY_FULL></a></h2> 195 197 <verbatim>int lsm_config(lsm_db *, int, ...); 196 -#define LSM_CONFIG_AUTOFLUSH 1 197 -#define LSM_CONFIG_PAGE_SIZE 2 198 -#define LSM_CONFIG_SAFETY 3 199 -#define LSM_CONFIG_BLOCK_SIZE 4 200 -#define LSM_CONFIG_AUTOWORK 5 201 -#define LSM_CONFIG_LOG_SIZE 6 202 -#define LSM_CONFIG_MMAP 7 203 -#define LSM_CONFIG_USE_LOG 8 204 -#define LSM_CONFIG_AUTOMERGE 9 205 -#define LSM_CONFIG_MAX_FREELIST 10 206 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 207 -#define LSM_CONFIG_AUTOCHECKPOINT 12 208 -#define LSM_CONFIG_SET_COMPRESSION 13 209 -#define LSM_CONFIG_GET_COMPRESSION 14 198 +#define LSM_CONFIG_AUTOFLUSH 1 199 +#define LSM_CONFIG_PAGE_SIZE 2 200 +#define LSM_CONFIG_SAFETY 3 201 +#define LSM_CONFIG_BLOCK_SIZE 4 202 +#define LSM_CONFIG_AUTOWORK 5 203 +#define LSM_CONFIG_MMAP 7 204 +#define LSM_CONFIG_USE_LOG 8 205 +#define LSM_CONFIG_AUTOMERGE 9 206 +#define LSM_CONFIG_MAX_FREELIST 10 207 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 208 +#define LSM_CONFIG_AUTOCHECKPOINT 12 209 +#define LSM_CONFIG_SET_COMPRESSION 13 210 +#define LSM_CONFIG_GET_COMPRESSION 14 211 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 210 212 #define LSM_SAFETY_OFF 0 211 213 #define LSM_SAFETY_NORMAL 1 212 214 #define LSM_SAFETY_FULL 2 213 215 </verbatim> 214 216 <p>The lsm_config() function is used to configure a database connection. 215 217 The following values may be passed as the second argument to lsm_config(). 216 -<p><dl><dt>LSM_CONFIG_AUTOFLUSH<dd>A read/write integer parameter. This value determines the maximum amount 217 -of space (in bytes) used to accumulate writes in main-memory before 218 -they are flushed to a level 0 segment. 218 +<p><dl><dt>LSM_CONFIG_AUTOFLUSH<dd>A read/write integer parameter. 219 +<p>This value determines the amount of data allowed to accumulate in a 220 +live in-memory tree before it is marked as old. After committing a 221 +transaction, a connection checks if the size of the live in-memory tree, 222 +including data structure overhead, is greater than the value of this 223 +option in KB. If it is, and there is not already an old in-memory tree, 224 +the live in-memory tree is marked as old. 225 +<p>The maximum allowable value is 1048576 (1GB). There is no minimum 226 +value. If this parameter is set to zero, then an attempt is made to 227 +mark the live in-memory tree as old after each transaction is committed. 228 +<p>The default value is 1024 (1MB). 219 229 <p><dt>LSM_CONFIG_PAGE_SIZE<dd>A read/write integer parameter. This parameter may only be set before 220 230 lsm_open() has been called. 221 -<p><dt>LSM_CONFIG_BLOCK_SIZE<dd>A read/write integer parameter. This parameter may only be set before 222 -lsm_open() has been called. 223 -<p><dt>LSM_CONFIG_LOG_SIZE<dd>A read/write integer parameter. 231 +<p><dt>LSM_CONFIG_BLOCK_SIZE<dd>A read/write integer parameter. 232 +<p>This parameter may only be set before lsm_open() has been called. It 233 +must be set to a power of two between 64 and 65536, inclusive (block 234 +sizes between 64KB and 64MB). 235 +<p>If the connection creates a new database, the block size of the new 236 +database is set to the value of this option in KB. After lsm_open() 237 +has been called, querying this parameter returns the actual block 238 +size of the opened database. 239 +<p>The default value is 1024 (1MB blocks). 224 240 <p><dt>LSM_CONFIG_SAFETY<dd>A read/write integer parameter. Valid values are 0, 1 (the default) 225 241 and 2. This parameter determines how robust the database is in the 226 242 face of a system crash (e.g. a power failure or operating system 227 243 crash). As follows: 228 244 <p>0 (off): No robustness. A system crash may corrupt the database. 229 245 <p>1 (normal): Some robustness. A system crash may not corrupt the 230 246 database file, but recently committed transactions may 231 247 be lost following recovery. 232 248 <p>2 (full): Full robustness. A system crash may not corrupt the 233 249 database file. Following recovery the database file 234 250 contains all successfully committed transactions. 235 251 <p><dt>LSM_CONFIG_AUTOWORK<dd>A read/write integer parameter. 236 252 <p><dt>LSM_CONFIG_AUTOCHECKPOINT<dd>A read/write integer parameter. 253 +<p>If this option is set to non-zero value N, then a checkpoint is 254 +automatically attempted after each N KB of data have been written to 255 +the database file. 256 +<p>The amount of uncheckpointed data already written to the database file 257 +is a global parameter. After performing database work (writing to the 258 +database file), the process checks if the total amount of uncheckpointed 259 +data exceeds the value of this paramter. If so, a checkpoint is performed. 260 +This means that this option may cause the connection to perform a 261 +checkpoint even if the current connection has itself written very little 262 +data into the database file. 263 +<p>The default value is 2048 (checkpoint every 2MB). 237 264 <p><dt>LSM_CONFIG_MMAP<dd>A read/write integer parameter. True to use mmap() to access the 238 265 database file. False otherwise. 239 266 <p><dt>LSM_CONFIG_USE_LOG<dd>A read/write boolean parameter. True (the default) to use the log 240 267 file normally. False otherwise. 241 268 <p><dt>LSM_CONFIG_AUTOMERGE<dd>A read/write integer parameter. The minimum number of segments to 242 269 merge together at a time. Default value 4. 243 270 <p><dt>LSM_CONFIG_MAX_FREELIST<dd>A read/write integer parameter. The maximum number of free-list ................................................................................ 255 282 content. The argument to this option should be a pointer to a structure 256 283 of type lsm_compress. The lsm_config() method takes a copy of the 257 284 structures contents. 258 285 <p>This option may only be used before lsm_open() is called. Invoking it 259 286 after lsm_open() has been called results in an LSM_MISUSE error. 260 287 <p><dt>LSM_CONFIG_GET_COMPRESSION<dd>Query the compression methods used to compress and decompress database 261 288 content. 262 -</dl><h2 id=compression>Compression and/or Encryption Hooks<a id=lsm_compress></a></h2> 289 +<p><dt>LSM_CONFIG_SET_COMPRESSION_FACTORY<dd>Configure a factory method to be invoked in case of an LSM_MISMATCH 290 +error. 291 +</dl><h2 id=compression>Compression and/or Encryption Hooks<a id=lsm_compress></a><a id=lsm_compress></a></h2> 263 292 <verbatim>struct lsm_compress { 264 293 void *pCtx; 265 294 unsigned int iId; 266 295 int (*xBound)(void *, int nSrc); 267 296 int (*xCompress)(void *, char *, int *, const char *, int); 268 297 int (*xUncompress)(void *, char *, int *, const char *, int); 298 + void (*xFree)(void *pCtx); 299 +}; 300 +struct lsm_compress_factory { 301 + void *pCtx; 302 + int (*xFactory)(void *, lsm_db *, u32); 303 + void (*xFree)(void *pCtx); 269 304 }; 270 305 </verbatim> 271 306 <h2 id=allocating>Allocating and Freeing Memory<a id=lsm_free></a></h2> 272 307 <verbatim>void *lsm_malloc(lsm_env*, size_t); 273 308 void *lsm_realloc(lsm_env*, void *, size_t); 274 309 void lsm_free(lsm_env*, void *); 275 310 </verbatim> 276 311 <p>Invoke the memory allocation functions that belong to environment 277 312 pEnv. Or the system defaults if no memory allocation functions have 278 313 been registered. 279 -<h2 id=querying>Querying a Connection For Operational Data<a id=lsm_info></a><a id=LSM_INFO_NWRITE></a><a id=LSM_INFO_NREAD></a><a id=LSM_INFO_DB_STRUCTURE></a><a id=LSM_INFO_LOG_STRUCTURE></a><a id=LSM_INFO_ARRAY_STRUCTURE></a><a id=LSM_INFO_PAGE_ASCII_DUMP></a><a id=LSM_INFO_PAGE_HEX_DUMP></a><a id=LSM_INFO_FREELIST></a><a id=LSM_INFO_ARRAY_PAGES></a><a id=LSM_INFO_CHECKPOINT_SIZE></a><a id=LSM_INFO_TREE_SIZE></a></h2> 314 +<h2 id=querying>Querying a Connection For Operational Data<a id=lsm_info></a><a id=LSM_INFO_NWRITE></a><a id=LSM_INFO_NREAD></a><a id=LSM_INFO_DB_STRUCTURE></a><a id=LSM_INFO_LOG_STRUCTURE></a><a id=LSM_INFO_ARRAY_STRUCTURE></a><a id=LSM_INFO_PAGE_ASCII_DUMP></a><a id=LSM_INFO_PAGE_HEX_DUMP></a><a id=LSM_INFO_FREELIST></a><a id=LSM_INFO_ARRAY_PAGES></a><a id=LSM_INFO_CHECKPOINT_SIZE></a><a id=LSM_INFO_TREE_SIZE></a><a id=LSM_INFO_FREELIST_SIZE></a></h2> 280 315 <verbatim>int lsm_info(lsm_db *, int, ...); 281 316 #define LSM_INFO_NWRITE 1 282 317 #define LSM_INFO_NREAD 2 283 318 #define LSM_INFO_DB_STRUCTURE 3 284 319 #define LSM_INFO_LOG_STRUCTURE 4 285 320 #define LSM_INFO_ARRAY_STRUCTURE 5 286 321 #define LSM_INFO_PAGE_ASCII_DUMP 6 287 322 #define LSM_INFO_PAGE_HEX_DUMP 7 288 323 #define LSM_INFO_FREELIST 8 289 324 #define LSM_INFO_ARRAY_PAGES 9 290 325 #define LSM_INFO_CHECKPOINT_SIZE 10 291 326 #define LSM_INFO_TREE_SIZE 11 327 +#define LSM_INFO_FREELIST_SIZE 12 292 328 </verbatim> 293 329 <p>Query a database connection for operational statistics or data. 294 330 The following values may be passed as the second argument to lsm_info(). 295 331 <p><dl><dt>LSM_INFO_NWRITE<dd>The third parameter should be of type (int *). The location pointed 296 332 to by the third parameter is set to the number of 4KB pages written to 297 333 the database file during the lifetime of this connection. 298 334 <p><dt>LSM_INFO_NREAD<dd>The third parameter should be of type (int *). The location pointed ................................................................................ 350 386 to is populated with a pointer to a nul-terminated string containing 351 387 the string representation of a Tcl data-structure. The returned 352 388 string should be eventually freed by the caller using lsm_free(). 353 389 <p>The Tcl structure returned is a list containing one element for each 354 390 free block in the database. The element itself consists of two 355 391 integers - the block number and the id of the snapshot that freed it. 356 392 <p><dt>LSM_INFO_CHECKPOINT_SIZE<dd>The third argument should be of type (int *). The location pointed to 357 -by this argument is populated with the number of bytes written to the 393 +by this argument is populated with the number of KB written to the 358 394 database file since the most recent checkpoint. 359 395 <p><dt>LSM_INFO_TREE_SIZE<dd>If this value is passed as the second argument to an lsm_info() call, it 360 396 should be followed by two arguments of type (int *) (for a total of four 361 397 arguments). 362 398 <p>At any time, there are either one or two tree structures held in shared 363 399 memory that new database clients will access (there may also be additional 364 400 tree structures being used by older clients - this API does not provide 365 401 information on them). One tree structure - the current tree - is used to 366 402 accumulate new data written to the database. The other tree structure - 367 403 the old tree - is a read-only tree holding older data and may be flushed 368 404 to disk at any time. 369 405 <p>Assuming no error occurs, the location pointed to by the first of the two 370 -(int *) arguments is set to the size of the old in-memory tree in bytes. 406 +(int *) arguments is set to the size of the old in-memory tree in KB. 371 407 The second is set to the size of the current, or live in-memory tree. 372 408 </dl><h2 id=opening>Opening and Closing Write Transactions<a id=lsm_begin></a><a id=lsm_commit></a><a id=lsm_rollback></a></h2> 373 409 <verbatim>int lsm_begin(lsm_db *pDb, int iLevel); 374 410 int lsm_commit(lsm_db *pDb, int iLevel); 375 411 int lsm_rollback(lsm_db *pDb, int iLevel); 376 412 </verbatim> 377 413 <p>These functions are used to open and close transactions and nested ................................................................................ 405 441 Delete a value from the database. No error is returned if the specified 406 442 key value does not exist in the database. 407 443 Delete all database entries with keys that are greater than (pKey1/nKey1) 408 444 and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and 409 445 (pKey2/nKey2) themselves, if they exist in the database, are not deleted. 410 446 <p>Return LSM_OK if successful, or an LSM error code otherwise. 411 447 <h2 id=explicit>Explicit Database Work and Checkpointing<a id=lsm_work></a><a id=lsm_flush></a><a id=lsm_checkpoint></a></h2> 412 -<verbatim>int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite); 448 +<verbatim>int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); 413 449 int lsm_flush(lsm_db *pDb); 414 -int lsm_checkpoint(lsm_db *pDb, int *pnByte); 450 +int lsm_checkpoint(lsm_db *pDb, int *pnKB); 415 451 </verbatim> 416 452 <p>This function is called by a thread to work on the database structure. 417 453 Attempt to checkpoint the current database snapshot. Return an LSM 418 454 error code if an error occurs or LSM_OK otherwise. 419 455 <p>If the current snapshot has already been checkpointed, calling this 420 -function is a no-op. In this case if pnByte is not NULL, *pnByte is 456 +function is a no-op. In this case if pnKB is not NULL, *pnKB is 421 457 set to 0. Or, if the current snapshot is successfully checkpointed 422 -by this function and pbCkpt is not NULL, *pnByte is set to the number 458 +by this function and pbKB is not NULL, *pnKB is set to the number 423 459 of bytes written to the database file since the previous checkpoint 424 460 (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). 425 461 <h2 id=opening>Opening and Closing Database Cursors<a id=lsm_csr_open></a><a id=lsm_csr_close></a></h2> 426 462 <verbatim>int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); 427 463 int lsm_csr_close(lsm_cursor *pCsr); 428 464 </verbatim> 429 465 <p>Open and close a database cursor.
Changes to www/lsmusr.wiki.
1146 1146 <p>The example code below might be executed in a background thread or process 1147 1147 in order to perform database work and checkpointing. In this case all other 1148 1148 clients should set the LSM_CONFIG_AUTOWORK parameter to zero. 1149 1149 1150 1150 <verbatim> 1151 1151 int rc; 1152 1152 lsm_db *db; 1153 - int nCkpt = 4*1024*1024; 1153 + int nCkpt = 4*1024; /* 4096KB == 4MB */ 1154 1154 1155 1155 /* Open a database connection to database "test.db". 1156 1156 ** 1157 1157 ** Configure the connection to automatically checkpoint the database after 1158 1158 ** writing each 4MB of data to it (instead of the default 2MB). As well 1159 1159 ** as to auto-work, the LSM_CONFIG_AUTOCHECKPOINT parameter applies to data 1160 1160 ** written by explicit calls to lsm_work(). ................................................................................ 1164 1164 lsm_open(db, "test.db"); 1165 1165 1166 1166 while( 1 ){ 1167 1167 int nWrite; 1168 1168 1169 1169 /* Attempt up to 512KB of work. Set nWrite to the number of bytes 1170 1170 ** actually written to disk. */ 1171 - rc = lsm_work(db, 2, 512*1024, &nWrite); 1171 + rc = lsm_work(db, 2, 512, &nWrite); 1172 1172 if( rc!=LSM_OK && rc!=LSM_BUSY ){ 1173 1173 /* Anything other than LSM_OK or LSM_BUSY is a problem. LSM_BUSY 1174 1174 ** indicates that some other client has taken the WORKER lock. Any 1175 1175 ** other error indicates something has gone quite wrong. */ 1176 1176 lsm_close(db); 1177 1177 return rc; 1178 1178 }