Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes to src/lsm.h.

    20     20   extern "C" {
    21     21   #endif
    22     22   
    23     23   /*
    24     24   ** Opaque handle types.
    25     25   */
    26     26   typedef struct lsm_compress lsm_compress;   /* Compression library functions */
           27  +typedef struct lsm_compress_factory lsm_compress_factory;
    27     28   typedef struct lsm_cursor lsm_cursor;       /* Database cursor handle */
    28     29   typedef struct lsm_db lsm_db;               /* Database connection handle */
    29     30   typedef struct lsm_env lsm_env;             /* Runtime environment */
    30     31   typedef struct lsm_file lsm_file;           /* OS file handle */
    31     32   typedef struct lsm_mutex lsm_mutex;         /* Mutex handle */
    32     33   
    33     34   /* 64-bit integer type used for file offsets. */
................................................................................
   102    103   #define LSM_IOERR     10
   103    104   #define LSM_CORRUPT   11
   104    105   #define LSM_FULL      13
   105    106   #define LSM_CANTOPEN  14
   106    107   #define LSM_PROTOCOL  15
   107    108   #define LSM_MISUSE    21
   108    109   
          110  +#define LSM_MISMATCH  50
          111  +
   109    112   /* 
   110    113   ** CAPI: Creating and Destroying Database Connection Handles
   111    114   **
   112    115   ** Open and close a database connection handle.
   113    116   */
   114    117   int lsm_new(lsm_env*, lsm_db **ppDb);
   115    118   int lsm_close(lsm_db *pDb);
................................................................................
   116    119   
   117    120   /* 
   118    121   ** CAPI: Connecting to a Database
   119    122   */
   120    123   int lsm_open(lsm_db *pDb, const char *zFilename);
   121    124   
   122    125   /*
   123         -** CAPI: Obtaining pointers to databases environments
          126  +** CAPI: Obtaining pointers to database environments
   124    127   **
   125    128   ** Return a pointer to the environment used by the database connection 
   126    129   ** passed as the first argument. Assuming the argument is valid, this 
   127    130   ** function always returns a valid environment pointer - it cannot fail.
   128    131   */
   129    132   lsm_env *lsm_get_env(lsm_db *pDb);
   130    133   
................................................................................
   251    254   **
   252    255   **   This option may only be used before lsm_open() is called. Invoking it
   253    256   **   after lsm_open() has been called results in an LSM_MISUSE error.
   254    257   **
   255    258   ** LSM_CONFIG_GET_COMPRESSION:
   256    259   **   Query the compression methods used to compress and decompress database
   257    260   **   content.
          261  +**
          262  +** LSM_CONFIG_SET_COMPRESSION_FACTORY:
          263  +**   Configure a factory method to be invoked in case of an LSM_MISMATCH
          264  +**   error.
   258    265   */
   259         -#define LSM_CONFIG_AUTOFLUSH           1
   260         -#define LSM_CONFIG_PAGE_SIZE           2
   261         -#define LSM_CONFIG_SAFETY              3
   262         -#define LSM_CONFIG_BLOCK_SIZE          4
   263         -#define LSM_CONFIG_AUTOWORK            5
   264         -#define LSM_CONFIG_MMAP                7
   265         -#define LSM_CONFIG_USE_LOG             8
   266         -#define LSM_CONFIG_AUTOMERGE           9
   267         -#define LSM_CONFIG_MAX_FREELIST       10
   268         -#define LSM_CONFIG_MULTIPLE_PROCESSES 11
   269         -#define LSM_CONFIG_AUTOCHECKPOINT     12
   270         -#define LSM_CONFIG_SET_COMPRESSION    13
   271         -#define LSM_CONFIG_GET_COMPRESSION    14
          266  +#define LSM_CONFIG_AUTOFLUSH                1
          267  +#define LSM_CONFIG_PAGE_SIZE                2
          268  +#define LSM_CONFIG_SAFETY                   3
          269  +#define LSM_CONFIG_BLOCK_SIZE               4
          270  +#define LSM_CONFIG_AUTOWORK                 5
          271  +#define LSM_CONFIG_MMAP                     7
          272  +#define LSM_CONFIG_USE_LOG                  8
          273  +#define LSM_CONFIG_AUTOMERGE                9
          274  +#define LSM_CONFIG_MAX_FREELIST            10
          275  +#define LSM_CONFIG_MULTIPLE_PROCESSES      11
          276  +#define LSM_CONFIG_AUTOCHECKPOINT          12
          277  +#define LSM_CONFIG_SET_COMPRESSION         13
          278  +#define LSM_CONFIG_GET_COMPRESSION         14
          279  +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15
   272    280   
   273    281   #define LSM_SAFETY_OFF    0
   274    282   #define LSM_SAFETY_NORMAL 1
   275    283   #define LSM_SAFETY_FULL   2
   276    284   
   277    285   /*
   278    286   ** CAPI: Compression and/or Encryption Hooks
................................................................................
   279    287   */
   280    288   struct lsm_compress {
   281    289     void *pCtx;
   282    290     unsigned int iId;
   283    291     int (*xBound)(void *, int nSrc);
   284    292     int (*xCompress)(void *, char *, int *, const char *, int);
   285    293     int (*xUncompress)(void *, char *, int *, const char *, int);
          294  +  void (*xFree)(void *pCtx);
          295  +};
          296  +
          297  +struct lsm_compress_factory {
          298  +  void *pCtx;
          299  +  int (*xFactory)(void *, lsm_db *, unsigned int);
          300  +  void (*xFree)(void *pCtx);
   286    301   };
          302  +
          303  +#define LSM_COMPRESSION_EMPTY 0
          304  +#define LSM_COMPRESSION_NONE  1
   287    305   
   288    306   
   289    307   /*
   290    308   ** CAPI: Allocating and Freeing Memory
   291    309   **
   292    310   ** Invoke the memory allocation functions that belong to environment
   293    311   ** pEnv. Or the system defaults if no memory allocation functions have 

Changes to src/lsmInt.h.

   312    312     int nDfltPgsz;                  /* Configured by LSM_CONFIG_PAGE_SIZE */
   313    313     int nDfltBlksz;                 /* Configured by LSM_CONFIG_BLOCK_SIZE */
   314    314     int nMaxFreelist;               /* Configured by LSM_CONFIG_MAX_FREELIST */
   315    315     int bMmap;                      /* Configured by LSM_CONFIG_MMAP */
   316    316     i64 nAutockpt;                  /* Configured by LSM_CONFIG_AUTOCHECKPOINT */
   317    317     int bMultiProc;                 /* Configured by L_C_MULTIPLE_PROCESSES */
   318    318     lsm_compress compress;          /* Compression callbacks */
          319  +  lsm_compress_factory factory;   /* Compression callback factory */
   319    320   
   320    321     /* Sub-system handles */
   321    322     FileSystem *pFS;                /* On-disk portion of database */
   322    323     Database *pDatabase;            /* Database shared data */
   323    324   
   324    325     /* Client transaction context */
   325    326     Snapshot *pClient;              /* Client snapshot */
................................................................................
   522    523   /*
   523    524   ** A snapshot of a database. A snapshot contains all the information required
   524    525   ** to read or write a database file on disk. See the description of struct
   525    526   ** Database below for futher details.
   526    527   */
   527    528   struct Snapshot {
   528    529     Database *pDatabase;            /* Database this snapshot belongs to */
          530  +  u32 iCmpId;                     /* Id of compression scheme */
   529    531     Level *pLevel;                  /* Pointer to level 0 of snapshot (or NULL) */
   530    532     i64 iId;                        /* Snapshot id */
   531    533     i64 iLogOff;                    /* Log file offset */
   532    534     Redirect redirect;              /* Block redirection array */
   533    535   
   534    536     /* Used by worker snapshots only */
   535    537     int nBlock;                     /* Number of blocks in database file */
................................................................................
   639    641   #endif
   640    642   
   641    643   /**************************************************************************
   642    644   ** Start of functions from "lsm_file.c".
   643    645   */
   644    646   int lsmFsOpen(lsm_db *, const char *);
   645    647   void lsmFsClose(FileSystem *);
          648  +
          649  +int lsmFsConfigure(lsm_db *db);
   646    650   
   647    651   int lsmFsBlockSize(FileSystem *);
   648    652   void lsmFsSetBlockSize(FileSystem *, int);
   649    653   
   650    654   int lsmFsPageSize(FileSystem *);
   651    655   void lsmFsSetPageSize(FileSystem *, int);
   652    656   

Changes to src/lsm_ckpt.c.

    28     28   **
    29     29   **   Checkpoint header (see the CKPT_HDR_XXX #defines):
    30     30   **
    31     31   **     1. The checkpoint id MSW.
    32     32   **     2. The checkpoint id LSW.
    33     33   **     3. The number of integer values in the entire checkpoint, including 
    34     34   **        the two checksum values.
    35         -**     4. The total number of blocks in the database.
    36         -**     5. The block size.
    37         -**     6. The number of levels.
    38         -**     7. The nominal database page size.
    39         -**     8. The number of pages (in total) written to the database file.
           35  +**     4. The compression scheme id.
           36  +**     5. The total number of blocks in the database.
           37  +**     6. The block size.
           38  +**     7. The number of levels.
           39  +**     8. The nominal database page size.
           40  +**     9. The number of pages (in total) written to the database file.
    40     41   **
    41     42   **   Log pointer:
    42     43   **
    43     44   **     1. The log offset MSW.
    44     45   **     2. The log offset LSW.
    45     46   **     3. Log checksum 0.
    46     47   **     4. Log checksum 1.
................................................................................
   170    171    + (((x)&0x00FF0000)>>8)  + (((x)&0xFF000000)>>24) \
   171    172   )
   172    173   
   173    174   static const int one = 1;
   174    175   #define LSM_LITTLE_ENDIAN (*(u8 *)(&one))
   175    176   
   176    177   /* Sizes, in integers, of various parts of the checkpoint. */
   177         -#define CKPT_HDR_SIZE         8
          178  +#define CKPT_HDR_SIZE         9
   178    179   #define CKPT_LOGPTR_SIZE      4
   179    180   #define CKPT_APPENDLIST_SIZE  (LSM_APPLIST_SZ * 2)
   180    181   
   181    182   /* A #define to describe each integer in the checkpoint header. */
   182    183   #define CKPT_HDR_ID_MSW   0
   183    184   #define CKPT_HDR_ID_LSW   1
   184    185   #define CKPT_HDR_NCKPT    2
   185         -#define CKPT_HDR_NBLOCK   3
   186         -#define CKPT_HDR_BLKSZ    4
   187         -#define CKPT_HDR_NLEVEL   5
   188         -#define CKPT_HDR_PGSZ     6
   189         -#define CKPT_HDR_NWRITE   7
          186  +#define CKPT_HDR_CMPID    3
          187  +#define CKPT_HDR_NBLOCK   4
          188  +#define CKPT_HDR_BLKSZ    5
          189  +#define CKPT_HDR_NLEVEL   6
          190  +#define CKPT_HDR_PGSZ     7
          191  +#define CKPT_HDR_NWRITE   8
   190    192   
   191         -#define CKPT_HDR_LO_MSW     8
   192         -#define CKPT_HDR_LO_LSW     9
   193         -#define CKPT_HDR_LO_CKSUM1 10
   194         -#define CKPT_HDR_LO_CKSUM2 11
          193  +#define CKPT_HDR_LO_MSW     9
          194  +#define CKPT_HDR_LO_LSW    10
          195  +#define CKPT_HDR_LO_CKSUM1 11
          196  +#define CKPT_HDR_LO_CKSUM2 12
   195    197   
   196    198   typedef struct CkptBuffer CkptBuffer;
   197    199   
   198    200   /*
   199    201   ** Dynamic buffer used to accumulate data for a checkpoint.
   200    202   */
   201    203   struct CkptBuffer {
................................................................................
   448    450     }
   449    451   
   450    452     /* Write the checkpoint header */
   451    453     assert( iId>=0 );
   452    454     ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc);
   453    455     ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc);
   454    456     ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc);
          457  +  ckptSetValue(&ckpt, CKPT_HDR_CMPID, pSnap->iCmpId, &rc);
   455    458     ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc);
   456    459     ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc);
   457    460     ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc);
   458    461     ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc);
   459    462     ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc);
   460    463   
   461    464     if( bCksum ){
................................................................................
   757    760   
   758    761   /*
   759    762   ** Initialize the shared-memory header with an empty snapshot. This function
   760    763   ** is called when no valid snapshot can be found in the database header.
   761    764   */
   762    765   static void ckptLoadEmpty(lsm_db *pDb){
   763    766     u32 aCkpt[] = {
   764         -    0,                  /* CKPT_HDR_ID_MSW */
   765         -    10,                 /* CKPT_HDR_ID_LSW */
   766         -    0,                  /* CKPT_HDR_NCKPT */
   767         -    0,                  /* CKPT_HDR_NBLOCK */
   768         -    0,                  /* CKPT_HDR_BLKSZ */
   769         -    0,                  /* CKPT_HDR_NLEVEL */
   770         -    0,                  /* CKPT_HDR_PGSZ */
   771         -    0,                  /* CKPT_HDR_OVFL */
   772         -    0,                  /* CKPT_HDR_NWRITE */
   773         -    0, 0, 1234, 5678,   /* The log pointer and initial checksum */
   774         -    0,0,0,0, 0,0,0,0,   /* The append list */
   775         -    0,                  /* The free block list */
   776         -    0, 0                /* Space for checksum values */
          767  +    0,                       /* CKPT_HDR_ID_MSW */
          768  +    10,                      /* CKPT_HDR_ID_LSW */
          769  +    0,                       /* CKPT_HDR_NCKPT */
          770  +    LSM_COMPRESSION_EMPTY,   /* CKPT_HDR_CMPID */
          771  +    0,                       /* CKPT_HDR_NBLOCK */
          772  +    0,                       /* CKPT_HDR_BLKSZ */
          773  +    0,                       /* CKPT_HDR_NLEVEL */
          774  +    0,                       /* CKPT_HDR_PGSZ */
          775  +    0,                       /* CKPT_HDR_OVFL */
          776  +    0,                       /* CKPT_HDR_NWRITE */
          777  +    0, 0, 1234, 5678,        /* The log pointer and initial checksum */
          778  +    0,0,0,0, 0,0,0,0,        /* The append list */
          779  +    0,                       /* The free block list */
          780  +    0, 0                     /* Space for checksum values */
   777    781     };
   778    782     u32 nCkpt = array_size(aCkpt);
   779    783     ShmHeader *pShm = pDb->pShmhdr;
   780    784   
   781    785     aCkpt[CKPT_HDR_NCKPT] = nCkpt;
   782    786     aCkpt[CKPT_HDR_BLKSZ] = pDb->nDfltBlksz;
   783    787     aCkpt[CKPT_HDR_PGSZ] = pDb->nDfltPgsz;
................................................................................
   917    921       }else{
   918    922         return LSM_PROTOCOL;
   919    923       }
   920    924     }
   921    925   
   922    926     rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker);
   923    927     if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase;
          928  +
          929  +  if( rc==LSM_OK && pDb->pWorker->iCmpId!=pDb->compress.iId ){
          930  +    lsmFreeSnapshot(pDb->pEnv, pDb->pWorker);
          931  +    rc = LSM_MISMATCH;
          932  +    pDb->pWorker = 0;
          933  +  }
   924    934   
   925    935   #if 0
   926    936     assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) );
   927    937   #endif
   928    938     return rc;
   929    939   }
   930    940   
................................................................................
   946    956       int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE;
   947    957   
   948    958       pNew->iId = lsmCheckpointId(aCkpt, 0);
   949    959       pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK];
   950    960       pNew->nWrite = aCkpt[CKPT_HDR_NWRITE];
   951    961       rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel);
   952    962       pNew->iLogOff = lsmCheckpointLogOffset(aCkpt);
          963  +
          964  +    pNew->iCmpId = aCkpt[CKPT_HDR_CMPID];
          965  +    if( pNew->iCmpId==LSM_COMPRESSION_EMPTY ){
          966  +      pNew->iCmpId = pDb->compress.iId;
          967  +    }
   953    968   
   954    969       /* Make a copy of the append-list */
   955    970       for(i=0; i<LSM_APPLIST_SZ; i++){
   956    971         u32 *a = &aCkpt[CKPT_HDR_SIZE + CKPT_LOGPTR_SIZE + i*2];
   957    972         pNew->aiAppend[i] = ckptRead64(a);
   958    973       }
   959    974   

Changes to src/lsm_file.c.

   506    506       pFS->zDb = (char *)&pFS[1];
   507    507       pFS->zLog = &pFS->zDb[nDb+1];
   508    508       pFS->nPagesize = LSM_DFLT_PAGE_SIZE;
   509    509       pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE;
   510    510       pFS->nMetasize = 4 * 1024;
   511    511       pFS->pDb = pDb;
   512    512       pFS->pEnv = pDb->pEnv;
   513         -    if( pDb->compress.xCompress ){
   514         -      pFS->pCompress = &pDb->compress;
   515         -    }else{
   516         -      pFS->bUseMmap = pDb->bMmap;
   517         -    }
   518    513   
   519    514       /* Make a copy of the database and log file names. */
   520    515       memcpy(pFS->zDb, zDb, nDb+1);
   521    516       memcpy(pFS->zLog, zDb, nDb);
   522    517       memcpy(&pFS->zLog[nDb], "-log", 5);
   523    518   
   524    519       /* Allocate the hash-table here. At some point, it should be changed
................................................................................
   547    542         pFS->szSector = lsmEnvSectorSize(pFS->pEnv, pFS->fdDb);
   548    543       }
   549    544     }
   550    545   
   551    546     pDb->pFS = pFS;
   552    547     return rc;
   553    548   }
          549  +
          550  +/*
          551  +** Configure the file-system object according to the current values of
          552  +** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options.
          553  +*/
          554  +int lsmFsConfigure(lsm_db *db){
          555  +  FileSystem *pFS = db->pFS;
          556  +  lsm_env *pEnv = pFS->pEnv;
          557  +  Page *pPg;
          558  +
          559  +  assert( pFS->nOut==0 );
          560  +  assert( pFS->pWaiting==0 );
          561  +
          562  +  /* Reset any compression/decompression buffers already allocated */
          563  +  lsmFree(pEnv, pFS->aIBuffer);
          564  +  lsmFree(pEnv, pFS->aOBuffer);
          565  +  pFS->nBuffer = 0;
          566  +
          567  +  /* Unmap the file, if it is currently mapped */
          568  +  if( pFS->pMap ){
          569  +    lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap);
          570  +    pFS->bUseMmap = 0;
          571  +  }
          572  +
          573  +  /* Free all allocate page structures */
          574  +  pPg = pFS->pLruFirst;
          575  +  while( pPg ){
          576  +    Page *pNext = pPg->pLruNext;
          577  +    if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData);
          578  +    lsmFree(pEnv, pPg);
          579  +    pPg = pNext;
          580  +  }
          581  +
          582  +  /* Zero pointers that point to deleted page objects */
          583  +  pFS->nCacheAlloc = 0;
          584  +  pFS->pLruFirst = 0;
          585  +  pFS->pLruLast = 0;
          586  +  pFS->pFree = 0;
          587  +
          588  +  /* Configure the FileSystem object */
          589  +  if( db->compress.xCompress ){
          590  +    pFS->pCompress = &db->compress;
          591  +    pFS->bUseMmap = 0;
          592  +  }else{
          593  +    pFS->pCompress = 0;
          594  +    pFS->bUseMmap = db->bMmap;
          595  +  }
          596  +
          597  +  return LSM_OK;
          598  +}
   554    599   
   555    600   /*
   556    601   ** Close and destroy a FileSystem object.
   557    602   */
   558    603   void lsmFsClose(FileSystem *pFS){
   559    604     if( pFS ){
   560    605       Page *pPg;

Changes to src/lsm_main.c.

    92     92     pDb->nMerge = LSM_DFLT_AUTOMERGE;
    93     93     pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES;
    94     94     pDb->bUseLog = LSM_DFLT_USE_LOG;
    95     95     pDb->iReader = -1;
    96     96     pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES;
    97     97     pDb->bMmap = LSM_DFLT_MMAP;
    98     98     pDb->xLog = xLog;
           99  +  pDb->compress.iId = LSM_COMPRESSION_NONE;
    99    100     return LSM_OK;
   100    101   }
   101    102   
   102    103   lsm_env *lsm_get_env(lsm_db *pDb){
   103    104     assert( pDb->pEnv );
   104    105     return pDb->pEnv;
   105    106   }
................................................................................
   190    191         rc = LSM_MISUSE_BKPT;
   191    192       }else{
   192    193         lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
   193    194         pDb->pClient = 0;
   194    195         lsmDbDatabaseRelease(pDb);
   195    196         lsmLogClose(pDb);
   196    197         lsmFsClose(pDb->pFS);
          198  +      
          199  +      /* Invoke any destructors registered for the compression or 
          200  +      ** compression factory callbacks.  */
          201  +      if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx);
          202  +      if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx);
          203  +
   197    204         lsmFree(pDb->pEnv, pDb->rollback.aArray);
   198    205         lsmFree(pDb->pEnv, pDb->aTrans);
   199    206         lsmFree(pDb->pEnv, pDb->apShm);
   200    207         lsmFree(pDb->pEnv, pDb);
   201    208       }
   202    209     }
   203    210     return rc;
................................................................................
   333    340           pDb->bMultiProc = *piVal = (*piVal!=0);
   334    341         }
   335    342         break;
   336    343       }
   337    344   
   338    345       case LSM_CONFIG_SET_COMPRESSION: {
   339    346         lsm_compress *p = va_arg(ap, lsm_compress *);
   340         -      if( pDb->pDatabase ){
   341         -        /* If lsm_open() has been called, this call is against the rules. */
          347  +      if( pDb->iReader>=0 ){
          348  +        /* May not change compression schemes with an open transaction */
   342    349           rc = LSM_MISUSE_BKPT;
   343    350         }else{
   344         -        memcpy(&pDb->compress, p, sizeof(lsm_compress));
          351  +        if( p->xBound==0 ){
          352  +          memset(&pDb->compress, 0, sizeof(lsm_compress));
          353  +          pDb->compress.iId = LSM_COMPRESSION_NONE;
          354  +        }else{
          355  +          memcpy(&pDb->compress, p, sizeof(lsm_compress));
          356  +        }
          357  +        rc = lsmFsConfigure(pDb);
          358  +      }
          359  +      break;
          360  +    }
          361  +
          362  +    case LSM_CONFIG_SET_COMPRESSION_FACTORY: {
          363  +      lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *);
          364  +      if( pDb->factory.xFree ){
          365  +        /* Invoke any destructor belonging to the current factory. */
          366  +        pDb->factory.xFree(pDb->factory.pCtx);
   345    367         }
          368  +      memcpy(&pDb->factory, p, sizeof(lsm_compress_factory));
   346    369         break;
   347    370       }
   348    371   
   349    372       case LSM_CONFIG_GET_COMPRESSION: {
   350    373         lsm_compress *p = va_arg(ap, lsm_compress *);
   351    374         memcpy(p, &pDb->compress, sizeof(lsm_compress));
   352    375         break;

Changes to src/lsm_shared.c.

   430    430     if( rc==LSM_OK ){
   431    431       assert( p );
   432    432       rc = lsmFsOpen(pDb, zName);
   433    433     }
   434    434     if( rc==LSM_OK ){
   435    435       rc = doDbConnect(pDb);
   436    436     }
          437  +  if( rc==LSM_OK ){
          438  +    rc = lsmFsConfigure(pDb);
          439  +  }
   437    440   
   438    441     return rc;
   439    442   }
   440    443   
   441    444   static void dbDeferClose(lsm_db *pDb){
   442    445     if( pDb->pFS ){
   443    446       LsmFile *pLsmFile = 0;
................................................................................
   971    974             ** lsm_sorted.c is changed to work directly from the serialized
   972    975             ** version of the snapshot.  */
   973    976             if( pDb->pClient==0 ){
   974    977               rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient);
   975    978             }
   976    979             assert( (rc==LSM_OK)==(pDb->pClient!=0) );
   977    980             assert( pDb->iReader>=0 );
          981  +
          982  +          /* Check that the client has the right compression hooks loaded.
          983  +          ** If not, set rc to LSM_MISMATCH.  */
          984  +          assert( rc!=LSM_OK || pDb->pClient->iCmpId!=LSM_COMPRESSION_EMPTY );
          985  +          if( rc==LSM_OK && pDb->pClient->iCmpId!=pDb->compress.iId ){
          986  +            rc = LSM_MISMATCH;
          987  +          }
   978    988           }else{
   979    989             rc = lsmReleaseReadlock(pDb);
   980    990           }
   981    991         }
          992  +
   982    993         if( rc==LSM_BUSY ){
   983    994           rc = LSM_OK;
   984    995         }
   985    996       }
   986    997   #if 0
   987    998   if( rc==LSM_OK && pDb->pClient ){
   988    999     fprintf(stderr, 

Changes to src/lsm_unix.c.

   190    190   
   191    191     if( p->pMap ){
   192    192       munmap(p->pMap, p->nMap);
   193    193       *ppOut = p->pMap = 0;
   194    194       *pnOut = p->nMap = 0;
   195    195     }
   196    196   
   197         -  memset(&buf, 0, sizeof(buf));
   198         -  prc = fstat(p->fd, &buf);
   199         -  if( prc!=0 ) return LSM_IOERR_BKPT;
   200         -  iSz = buf.st_size;
   201         -  if( iSz<iMin ){
   202         -    iSz = ((iMin + (2<<20) - 1) / (2<<20)) * (2<<20);
   203         -    prc = ftruncate(p->fd, iSz);
          197  +  if( iMin>=0 ){
          198  +    memset(&buf, 0, sizeof(buf));
          199  +    prc = fstat(p->fd, &buf);
   204    200       if( prc!=0 ) return LSM_IOERR_BKPT;
          201  +    iSz = buf.st_size;
          202  +    if( iSz<iMin ){
          203  +      iSz = ((iMin + (2<<20) - 1) / (2<<20)) * (2<<20);
          204  +      prc = ftruncate(p->fd, iSz);
          205  +      if( prc!=0 ) return LSM_IOERR_BKPT;
          206  +    }
          207  +
          208  +    p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0);
          209  +    p->nMap = iSz;
   205    210     }
   206    211   
   207         -  p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0);
   208         -  p->nMap = iSz;
   209         -
   210    212     *ppOut = p->pMap;
   211    213     *pnOut = p->nMap;
   212    214     return LSM_OK;
   213    215   }
   214    216   
   215    217   static int lsmPosixOsFullpath(
   216    218     lsm_env *pEnv,

Added test/lsm4.test.

            1  +# 2013 February 06
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +set testprefix lsm4
           16  +db close
           17  +
           18  +proc db_fetch {db key} {
           19  +  db csr_open csr
           20  +  csr seek $key eq
           21  +  set ret [csr value]
           22  +  csr close
           23  +  set ret
           24  +}
           25  +
           26  +do_test 1.1 {
           27  +  lsm_open db test.db 
           28  +  db config {set_compression noop}
           29  +  db write 1 abc
           30  +  db write 2 def
           31  +  db close
           32  +} {}
           33  +
           34  +do_test 1.2 {
           35  +  lsm_open db test.db 
           36  +  db config {set_compression noop}
           37  +  list [db_fetch db 1] [db_fetch db 2]
           38  +} {abc def}
           39  +
           40  +do_test 1.3 {
           41  +  db close
           42  +  lsm_open db test.db 
           43  +  db config {set_compression rle}
           44  +  list [catch {db_fetch db 1} msg] $msg
           45  +} {1 {error in lsm_csr_open() - 50}}
           46  +
           47  +do_test 1.4 {
           48  +  db close
           49  +  lsm_open db test.db 
           50  +breakpoint
           51  +  list [catch {db_fetch db 1} msg] $msg
           52  +} {1 {error in lsm_csr_open() - 50}}
           53  +
           54  +
           55  +
           56  +finish_test
           57  +

Changes to test/test_lsm.c.

     5      5   ** a legal notice, here is a blessing:
     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12         -**
    13     12   */
    14     13   
    15     14   #include <tcl.h>
    16     15   #include "lsm.h"
    17     16   #include "sqlite4.h"
    18     17   #include <assert.h>
    19     18   #include <string.h>
    20     19   
    21     20   extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite4 **ppDb);
    22     21   extern const char *sqlite4TestErrorName(int);
           22  +
           23  +/*************************************************************************
           24  +*/
           25  +#define ENCRYPTION_XOR_MASK 0xa3b2bbb6
           26  +static int testCompressEncBound(void *pCtx, int nSrc){
           27  +  return nSrc;
           28  +}
           29  +static int testCompressEncCompress(
           30  +  void *pCtx, 
           31  +  char *pOut, int *pnOut, 
           32  +  const char *pIn, int nIn
           33  +){
           34  +  int i;
           35  +  unsigned int *aIn = (unsigned int *)pOut;
           36  +  unsigned int *aOut = (unsigned int *)pIn;
           37  +
           38  +  assert( (nIn%4)==0 );
           39  +  for(i=0; i<(nIn/4); i++){
           40  +    aOut[i] = (aIn[i] ^ ENCRYPTION_XOR_MASK);
           41  +  }
           42  +  *pnOut = nIn;
           43  +
           44  +  return LSM_OK;
           45  +}
           46  +static int testCompressEncUncompress(
           47  +  void *pCtx, 
           48  +  char *pOut, int *pnOut, 
           49  +  const char *pIn, int nIn
           50  +){
           51  +  return testCompressEncUncompress(pCtx, pOut, pnOut, pIn, nIn);
           52  +}
           53  +static void testCompressEncFree(void *pCtx){
           54  +  /* no-op */
           55  +}
           56  +/* 
           57  +** End of compression routines "encrypt".
           58  +*************************************************************************/
           59  +
           60  +/*************************************************************************
           61  +*/
           62  +static int testCompressRleBound(void *pCtx, int nSrc){
           63  +  return nSrc*2;
           64  +}
           65  +static int testCompressRleCompress(
           66  +  void *pCtx, 
           67  +  char *pOut, int *pnOut, 
           68  +  const char *pIn, int nIn
           69  +){
           70  +  int iOut = 0;
           71  +  int i;
           72  +  char c;
           73  +  int n;
           74  +
           75  +  c = pIn[0];
           76  +  n = 1;
           77  +  for(i=1; i<nIn; i++){
           78  +    if( pIn[i]==c && n<127 ){
           79  +      n++;
           80  +    }else{
           81  +      pOut[iOut++] = c;
           82  +      pOut[iOut++] = (char)n;
           83  +      c = pIn[i];
           84  +      n = 1;
           85  +    }
           86  +  }
           87  +
           88  +  pOut[iOut++] = c;
           89  +  pOut[iOut++] = (char)n;
           90  +  *pnOut = iOut;
           91  +
           92  +  return LSM_OK;
           93  +}
           94  +static int testCompressRleUncompress(
           95  +  void *pCtx, 
           96  +  char *pOut, int *pnOut, 
           97  +  const char *pIn, int nIn
           98  +){
           99  +  int i;
          100  +  int iOut = 0;
          101  +
          102  +  for(i=0; i<nIn; i+=2){
          103  +    int iRep;
          104  +    char c = pIn[i];
          105  +    int n = (int)(pIn[i+1]);
          106  +
          107  +    for(iRep=0; iRep<n; iRep++){
          108  +      pOut[iOut++] = c;
          109  +    }
          110  +  }
          111  +
          112  +  *pnOut = iOut;
          113  +  return LSM_OK;
          114  +}
          115  +static void testCompressRleFree(void *pCtx){
          116  +}
          117  +/* 
          118  +** End of compression routines "rle".
          119  +*************************************************************************/
          120  +
          121  +/*************************************************************************
          122  +*/
          123  +static int testCompressNoopBound(void *pCtx, int nSrc){
          124  +  return nSrc;
          125  +}
          126  +static int testCompressNoopCompress(
          127  +  void *pCtx, 
          128  +  char *pOut, int *pnOut, 
          129  +  const char *pIn, int nIn
          130  +){
          131  +  *pnOut = nIn;
          132  +  memcpy(pOut, pIn, nIn);
          133  +  return LSM_OK;
          134  +}
          135  +static int testCompressNoopUncompress(
          136  +  void *pCtx, 
          137  +  char *pOut, int *pnOut, 
          138  +  const char *pIn, int nIn
          139  +){
          140  +  *pnOut = nIn;
          141  +  memcpy(pOut, pIn, nIn);
          142  +  return LSM_OK;
          143  +}
          144  +static void testCompressNoopFree(void *pCtx){
          145  +}
          146  +/* 
          147  +** End of compression routines "noop".
          148  +*************************************************************************/
    23    149   
    24    150   /*
    25    151   ** TCLCMD:    sqlite4_lsm_config DB DBNAME PARAM ...
    26    152   */
    27    153   static int test_sqlite4_lsm_config(
    28    154     void * clientData,
    29    155     Tcl_Interp *interp,
................................................................................
    30    156     int objc,
    31    157     Tcl_Obj *CONST objv[]
    32    158   ){
    33    159     struct Switch {
    34    160       const char *zSwitch;
    35    161       int iVal;
    36    162     } aParam[] = {
    37         -    { "log-size",       LSM_CONFIG_LOG_SIZE }, 
    38    163       { "safety",         LSM_CONFIG_SAFETY }, 
    39    164       { "autoflush",      LSM_CONFIG_AUTOFLUSH }, 
    40    165       { "mmap",           LSM_CONFIG_MMAP }, 
    41    166       { "page-size",      LSM_CONFIG_PAGE_SIZE }, 
    42    167       { "autowork",       LSM_CONFIG_AUTOWORK }, 
    43    168       { 0, 0 }
    44    169     };
................................................................................
   282    407       Tcl_SetResult(interp, (char *)sqlite4TestErrorName(rc), TCL_STATIC);
   283    408       return TCL_ERROR;
   284    409     }
   285    410   
   286    411     Tcl_ResetResult(interp);
   287    412     return TCL_OK;
   288    413   }
          414  +
          415  +static int testConfigureSetCompression(
          416  +  Tcl_Interp *interp, 
          417  +  lsm_db *db, 
          418  +  Tcl_Obj *pCmp
          419  +){
          420  +  struct CompressionScheme {
          421  +    const char *zName;
          422  +    lsm_compress cmp;
          423  +  } aCmp[] = {
          424  +    { "encrypt", { 0, 43, 
          425  +        testCompressEncBound, testCompressEncCompress,
          426  +        testCompressEncUncompress, testCompressEncFree
          427  +    } },
          428  +    { "rle", { 0, 44, 
          429  +        testCompressRleBound, testCompressRleCompress,
          430  +        testCompressRleUncompress, testCompressRleFree
          431  +    } },
          432  +    { "noop", { 0, 45, 
          433  +        testCompressNoopBound, testCompressNoopCompress,
          434  +        testCompressNoopUncompress, testCompressNoopFree
          435  +    } },
          436  +    { 0, {0, 0, 0, 0, 0, 0} }
          437  +  };
          438  +  int iOpt;
          439  +  int rc;
          440  +
          441  +  rc = Tcl_GetIndexFromObjStruct(
          442  +      interp, pCmp, aCmp, sizeof(aCmp[0]), "scheme", 0, &iOpt
          443  +  );
          444  +  if( rc!=TCL_OK ) return rc;
          445  +
          446  +  rc = lsm_config(db, LSM_CONFIG_SET_COMPRESSION, &aCmp[iOpt].cmp);
          447  +
          448  +  return rc;
          449  +}
   289    450   
   290    451   static int testConfigureLsm(Tcl_Interp *interp, lsm_db *db, Tcl_Obj *pObj){
   291    452     struct Lsmconfig {
   292    453       const char *zOpt;
   293    454       int eOpt;
   294    455     } aConfig[] = {
   295    456       { "autoflush",        LSM_CONFIG_AUTOFLUSH },
   296    457       { "page_size",        LSM_CONFIG_PAGE_SIZE },
   297    458       { "block_size",       LSM_CONFIG_BLOCK_SIZE },
   298    459       { "safety",           LSM_CONFIG_SAFETY },
   299    460       { "autowork",         LSM_CONFIG_AUTOWORK },
   300    461       { "autocheckpoint",   LSM_CONFIG_AUTOCHECKPOINT },
   301         -    { "log_size",         LSM_CONFIG_LOG_SIZE },
   302    462       { "mmap",             LSM_CONFIG_MMAP },
   303    463       { "use_log",          LSM_CONFIG_USE_LOG },
   304    464       { "automerge",        LSM_CONFIG_AUTOMERGE },
   305    465       { "max_freelist",     LSM_CONFIG_MAX_FREELIST },
   306    466       { "multi_proc",       LSM_CONFIG_MULTIPLE_PROCESSES },
          467  +    { "set_compression",  LSM_CONFIG_SET_COMPRESSION },
   307    468       { 0, 0 }
   308    469     };
   309    470     int nElem;
   310    471     int i;
   311    472     Tcl_Obj **apElem;
   312    473     int rc;
   313    474   
................................................................................
   321    482         if( i==(nElem-1) ){
   322    483           Tcl_ResetResult(interp);
   323    484           Tcl_AppendResult(interp, "option \"", Tcl_GetString(apElem[i]), 
   324    485               "\" requires an argument", 0
   325    486               );
   326    487           rc = TCL_ERROR;
   327    488         }else{
   328         -        int iVal;
   329         -        rc = Tcl_GetIntFromObj(interp, apElem[i+1], &iVal);
   330         -        if( rc==TCL_OK ){
   331         -          lsm_config(db, aConfig[iOpt].eOpt, &iVal);
          489  +        if( aConfig[iOpt].eOpt==LSM_CONFIG_SET_COMPRESSION ){
          490  +          rc = testConfigureSetCompression(interp, db, apElem[i+1]);
          491  +        }
          492  +        else {
          493  +          int iVal;
          494  +          rc = Tcl_GetIntFromObj(interp, apElem[i+1], &iVal);
          495  +          if( rc==TCL_OK ){
          496  +            lsm_config(db, aConfig[iOpt].eOpt, &iVal);
          497  +          }
          498  +          Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
   332    499           }
   333    500         }
   334    501       }
   335    502     }
   336    503   
   337    504     return rc;
   338    505   }
................................................................................
   619    786         return TCL_OK;
   620    787       }
   621    788   
   622    789       case 8: assert( 0==strcmp(aCmd[8].zCmd, "work") ); {
   623    790         int nWork = 0;
   624    791         int nMerge = 1;
   625    792         int nWrite = 0;
   626         -      int i;
   627    793   
   628    794         if( objc==3 ){
   629    795           rc = Tcl_GetIntFromObj(interp, objv[2], &nWork);
   630    796         }else if( objc==4 ){
   631    797           rc = Tcl_GetIntFromObj(interp, objv[2], &nMerge);
   632    798           if( rc!=TCL_OK ) return rc;
   633    799           rc = Tcl_GetIntFromObj(interp, objv[3], &nWork);

Changes to www/lsmapi.wiki.

    17     17   
    18     18   <h1>LSM API Topics</h1>
    19     19   <ol>
    20     20   <li><a href="#database" style=text-decoration:none>Database Runtime Environment</a>
    21     21   <li><a href="#lsm" style=text-decoration:none>LSM Error Codes</a>
    22     22   <li><a href="#creating" style=text-decoration:none>Creating and Destroying Database Connection Handles</a>
    23     23   <li><a href="#connecting" style=text-decoration:none>Connecting to a Database</a>
    24         -<li><a href="#obtaining" style=text-decoration:none>Obtaining pointers to databases environments</a>
           24  +<li><a href="#obtaining" style=text-decoration:none>Obtaining pointers to database environments</a>
    25     25   <li><a href="#configuring" style=text-decoration:none>Configuring a database connection.</a>
    26     26   <li><a href="#compression" style=text-decoration:none>Compression and/or Encryption Hooks</a>
    27     27   <li><a href="#allocating" style=text-decoration:none>Allocating and Freeing Memory</a>
    28     28   <li><a href="#querying" style=text-decoration:none>Querying a Connection For Operational Data</a>
    29     29   <li><a href="#opening" style=text-decoration:none>Opening and Closing Write Transactions</a>
    30     30   <li><a href="#writing" style=text-decoration:none>Writing to a Database</a>
    31     31   <li><a href="#explicit" style=text-decoration:none>Explicit Database Work and Checkpointing</a>
................................................................................
    63     63   <span style=display:block;float:left;width:35ex><a href=#lsm_new>lsm_new</a></span>
    64     64   <span style=display:block;float:left;width:35ex><a href=#lsm_open>lsm_open</a></span>
    65     65   <span style=display:block;float:left;width:35ex><a href=#lsm_rollback>lsm_rollback</a></span>
    66     66   <span style=display:block;float:left;width:35ex><a href=#lsm_work>lsm_work</a></span>
    67     67   <br style=clear:both>
    68     68   <h1 style=clear:both>All LSM API Types</h1>
    69     69   <span style=display:block;float:left;width:35ex><a href=#lsm_compress>lsm_compress</a></span>
           70  +<span style=display:block;float:left;width:35ex><a href=#lsm_compress>lsm_compress</a></span>
    70     71   <span style=display:block;float:left;width:35ex><a href=#lsm_env>lsm_env</a></span>
    71     72   <br style=clear:both>
    72     73   <h1>All LSM API Constants</h1>
    73     74   <span style=display:block;float:left;width:35ex><a href=#LSM_BUSY>LSM_BUSY</a></span>
    74     75   <span style=display:block;float:left;width:35ex><a href=#LSM_CANTOPEN>LSM_CANTOPEN</a></span>
    75     76   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOCHECKPOINT>LSM_CONFIG_AUTOCHECKPOINT</a></span>
    76     77   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOFLUSH>LSM_CONFIG_AUTOFLUSH</a></span>
    77     78   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOMERGE>LSM_CONFIG_AUTOMERGE</a></span>
    78     79   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_AUTOWORK>LSM_CONFIG_AUTOWORK</a></span>
    79     80   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_BLOCK_SIZE>LSM_CONFIG_BLOCK_SIZE</a></span>
    80     81   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_GET_COMPRESSION>LSM_CONFIG_GET_COMPRESSION</a></span>
    81         -<span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_LOG_SIZE>LSM_CONFIG_LOG_SIZE</a></span>
    82     82   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_MAX_FREELIST>LSM_CONFIG_MAX_FREELIST</a></span>
    83     83   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_MMAP>LSM_CONFIG_MMAP</a></span>
    84     84   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_MULTIPLE_PROCESSES>LSM_CONFIG_MULTIPLE_PROCESSES</a></span>
    85     85   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_PAGE_SIZE>LSM_CONFIG_PAGE_SIZE</a></span>
    86     86   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_SAFETY>LSM_CONFIG_SAFETY</a></span>
    87     87   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_SET_COMPRESSION>LSM_CONFIG_SET_COMPRESSION</a></span>
           88  +<span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_SET_COMPRESSION_FACTORY>LSM_CONFIG_SET_COMPRESSION_FACTORY</a></span>
    88     89   <span style=display:block;float:left;width:35ex><a href=#LSM_CONFIG_USE_LOG>LSM_CONFIG_USE_LOG</a></span>
    89     90   <span style=display:block;float:left;width:35ex><a href=#LSM_CORRUPT>LSM_CORRUPT</a></span>
    90     91   <span style=display:block;float:left;width:35ex><a href=#LSM_ERROR>LSM_ERROR</a></span>
    91     92   <span style=display:block;float:left;width:35ex><a href=#LSM_FULL>LSM_FULL</a></span>
    92     93   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_ARRAY_PAGES>LSM_INFO_ARRAY_PAGES</a></span>
    93     94   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_ARRAY_STRUCTURE>LSM_INFO_ARRAY_STRUCTURE</a></span>
    94     95   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_CHECKPOINT_SIZE>LSM_INFO_CHECKPOINT_SIZE</a></span>
    95     96   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_DB_STRUCTURE>LSM_INFO_DB_STRUCTURE</a></span>
    96     97   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_FREELIST>LSM_INFO_FREELIST</a></span>
           98  +<span style=display:block;float:left;width:35ex><a href=#LSM_INFO_FREELIST_SIZE>LSM_INFO_FREELIST_SIZE</a></span>
    97     99   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_LOG_STRUCTURE>LSM_INFO_LOG_STRUCTURE</a></span>
    98    100   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_NREAD>LSM_INFO_NREAD</a></span>
    99    101   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_NWRITE>LSM_INFO_NWRITE</a></span>
   100    102   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_PAGE_ASCII_DUMP>LSM_INFO_PAGE_ASCII_DUMP</a></span>
   101    103   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_PAGE_HEX_DUMP>LSM_INFO_PAGE_HEX_DUMP</a></span>
   102    104   <span style=display:block;float:left;width:35ex><a href=#LSM_INFO_TREE_SIZE>LSM_INFO_TREE_SIZE</a></span>
   103    105   <span style=display:block;float:left;width:35ex><a href=#LSM_IOERR>LSM_IOERR</a></span>
................................................................................
   178    180   <verbatim>int lsm_new(lsm_env*, lsm_db **ppDb);
   179    181   int lsm_close(lsm_db *pDb);
   180    182   </verbatim>
   181    183   <p>Open and close a database connection handle.
   182    184   <h2 id=connecting>Connecting to a Database<a id=lsm_open></a></h2>
   183    185   <verbatim>int lsm_open(lsm_db *pDb, const char *zFilename);
   184    186   </verbatim>
   185         -<h2 id=obtaining>Obtaining pointers to databases environments</h2>
          187  +<h2 id=obtaining>Obtaining pointers to database environments</h2>
   186    188   <verbatim>lsm_env *lsm_get_env(lsm_db *pDb);
   187    189   lsm_env *lsm_default_env(void);
   188    190   </verbatim>
   189    191   <p>Return a pointer to the environment used by the database connection 
   190    192   passed as the first argument. Assuming the argument is valid, this 
   191    193   function always returns a valid environment pointer - it cannot fail.
   192    194   The lsm_default_env() function returns a pointer to the default LSM
   193    195   environment for the current platform.
   194         -<h2 id=configuring>Configuring a database connection.<a id=lsm_config></a><a id=LSM_CONFIG_AUTOFLUSH></a><a id=LSM_CONFIG_PAGE_SIZE></a><a id=LSM_CONFIG_SAFETY></a><a id=LSM_CONFIG_BLOCK_SIZE></a><a id=LSM_CONFIG_AUTOWORK></a><a id=LSM_CONFIG_LOG_SIZE></a><a id=LSM_CONFIG_MMAP></a><a id=LSM_CONFIG_USE_LOG></a><a id=LSM_CONFIG_AUTOMERGE></a><a id=LSM_CONFIG_MAX_FREELIST></a><a id=LSM_CONFIG_MULTIPLE_PROCESSES></a><a id=LSM_CONFIG_AUTOCHECKPOINT></a><a id=LSM_CONFIG_SET_COMPRESSION></a><a id=LSM_CONFIG_GET_COMPRESSION></a><a id=LSM_SAFETY_OFF></a><a id=LSM_SAFETY_NORMAL></a><a id=LSM_SAFETY_FULL></a></h2>
          196  +<h2 id=configuring>Configuring a database connection.<a id=lsm_config></a><a id=LSM_CONFIG_AUTOFLUSH></a><a id=LSM_CONFIG_PAGE_SIZE></a><a id=LSM_CONFIG_SAFETY></a><a id=LSM_CONFIG_BLOCK_SIZE></a><a id=LSM_CONFIG_AUTOWORK></a><a id=LSM_CONFIG_MMAP></a><a id=LSM_CONFIG_USE_LOG></a><a id=LSM_CONFIG_AUTOMERGE></a><a id=LSM_CONFIG_MAX_FREELIST></a><a id=LSM_CONFIG_MULTIPLE_PROCESSES></a><a id=LSM_CONFIG_AUTOCHECKPOINT></a><a id=LSM_CONFIG_SET_COMPRESSION></a><a id=LSM_CONFIG_GET_COMPRESSION></a><a id=LSM_CONFIG_SET_COMPRESSION_FACTORY></a><a id=LSM_SAFETY_OFF></a><a id=LSM_SAFETY_NORMAL></a><a id=LSM_SAFETY_FULL></a></h2>
   195    197   <verbatim>int lsm_config(lsm_db *, int, ...);
   196         -#define LSM_CONFIG_AUTOFLUSH           1
   197         -#define LSM_CONFIG_PAGE_SIZE           2
   198         -#define LSM_CONFIG_SAFETY              3
   199         -#define LSM_CONFIG_BLOCK_SIZE          4
   200         -#define LSM_CONFIG_AUTOWORK            5
   201         -#define LSM_CONFIG_LOG_SIZE            6
   202         -#define LSM_CONFIG_MMAP                7
   203         -#define LSM_CONFIG_USE_LOG             8
   204         -#define LSM_CONFIG_AUTOMERGE           9
   205         -#define LSM_CONFIG_MAX_FREELIST       10
   206         -#define LSM_CONFIG_MULTIPLE_PROCESSES 11
   207         -#define LSM_CONFIG_AUTOCHECKPOINT     12
   208         -#define LSM_CONFIG_SET_COMPRESSION    13
   209         -#define LSM_CONFIG_GET_COMPRESSION    14
          198  +#define LSM_CONFIG_AUTOFLUSH                1
          199  +#define LSM_CONFIG_PAGE_SIZE                2
          200  +#define LSM_CONFIG_SAFETY                   3
          201  +#define LSM_CONFIG_BLOCK_SIZE               4
          202  +#define LSM_CONFIG_AUTOWORK                 5
          203  +#define LSM_CONFIG_MMAP                     7
          204  +#define LSM_CONFIG_USE_LOG                  8
          205  +#define LSM_CONFIG_AUTOMERGE                9
          206  +#define LSM_CONFIG_MAX_FREELIST            10
          207  +#define LSM_CONFIG_MULTIPLE_PROCESSES      11
          208  +#define LSM_CONFIG_AUTOCHECKPOINT          12
          209  +#define LSM_CONFIG_SET_COMPRESSION         13
          210  +#define LSM_CONFIG_GET_COMPRESSION         14
          211  +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15
   210    212   #define LSM_SAFETY_OFF    0
   211    213   #define LSM_SAFETY_NORMAL 1
   212    214   #define LSM_SAFETY_FULL   2
   213    215   </verbatim>
   214    216   <p>The lsm_config() function is used to configure a database connection.
   215    217   The following values may be passed as the second argument to lsm_config().
   216         -<p><dl><dt>LSM_CONFIG_AUTOFLUSH<dd>A read/write integer parameter. This value determines the maximum amount
   217         -of space (in bytes) used to accumulate writes in main-memory before 
   218         -they are flushed to a level 0 segment.
          218  +<p><dl><dt>LSM_CONFIG_AUTOFLUSH<dd>A read/write integer parameter. 
          219  +<p>This value determines the amount of data allowed to accumulate in a
          220  +live in-memory tree before it is marked as old. After committing a
          221  +transaction, a connection checks if the size of the live in-memory tree,
          222  +including data structure overhead, is greater than the value of this
          223  +option in KB. If it is, and there is not already an old in-memory tree,
          224  +the live in-memory tree is marked as old.
          225  +<p>The maximum allowable value is 1048576 (1GB). There is no minimum 
          226  +value. If this parameter is set to zero, then an attempt is made to
          227  +mark the live in-memory tree as old after each transaction is committed.
          228  +<p>The default value is 1024 (1MB).
   219    229   <p><dt>LSM_CONFIG_PAGE_SIZE<dd>A read/write integer parameter. This parameter may only be set before
   220    230   lsm_open() has been called.
   221         -<p><dt>LSM_CONFIG_BLOCK_SIZE<dd>A read/write integer parameter. This parameter may only be set before
   222         -lsm_open() has been called.
   223         -<p><dt>LSM_CONFIG_LOG_SIZE<dd>A read/write integer parameter.
          231  +<p><dt>LSM_CONFIG_BLOCK_SIZE<dd>A read/write integer parameter. 
          232  +<p>This parameter may only be set before lsm_open() has been called. It
          233  +must be set to a power of two between 64 and 65536, inclusive (block 
          234  +sizes between 64KB and 64MB).
          235  +<p>If the connection creates a new database, the block size of the new
          236  +database is set to the value of this option in KB. After lsm_open()
          237  +has been called, querying this parameter returns the actual block
          238  +size of the opened database.
          239  +<p>The default value is 1024 (1MB blocks).
   224    240   <p><dt>LSM_CONFIG_SAFETY<dd>A read/write integer parameter. Valid values are 0, 1 (the default) 
   225    241   and 2. This parameter determines how robust the database is in the
   226    242   face of a system crash (e.g. a power failure or operating system 
   227    243   crash). As follows:
   228    244   <p>0 (off):    No robustness. A system crash may corrupt the database.
   229    245   <p>1 (normal): Some robustness. A system crash may not corrupt the
   230    246   database file, but recently committed transactions may
   231    247   be lost following recovery.
   232    248   <p>2 (full):   Full robustness. A system crash may not corrupt the
   233    249   database file. Following recovery the database file
   234    250   contains all successfully committed transactions.
   235    251   <p><dt>LSM_CONFIG_AUTOWORK<dd>A read/write integer parameter.
   236    252   <p><dt>LSM_CONFIG_AUTOCHECKPOINT<dd>A read/write integer parameter.
          253  +<p>If this option is set to non-zero value N, then a checkpoint is
          254  +automatically attempted after each N KB of data have been written to 
          255  +the database file.
          256  +<p>The amount of uncheckpointed data already written to the database file
          257  +is a global parameter. After performing database work (writing to the
          258  +database file), the process checks if the total amount of uncheckpointed 
          259  +data exceeds the value of this paramter. If so, a checkpoint is performed.
          260  +This means that this option may cause the connection to perform a 
          261  +checkpoint even if the current connection has itself written very little
          262  +data into the database file.
          263  +<p>The default value is 2048 (checkpoint every 2MB).
   237    264   <p><dt>LSM_CONFIG_MMAP<dd>A read/write integer parameter. True to use mmap() to access the 
   238    265   database file. False otherwise.
   239    266   <p><dt>LSM_CONFIG_USE_LOG<dd>A read/write boolean parameter. True (the default) to use the log
   240    267   file normally. False otherwise.
   241    268   <p><dt>LSM_CONFIG_AUTOMERGE<dd>A read/write integer parameter. The minimum number of segments to
   242    269   merge together at a time. Default value 4.
   243    270   <p><dt>LSM_CONFIG_MAX_FREELIST<dd>A read/write integer parameter. The maximum number of free-list 
................................................................................
   255    282   content. The argument to this option should be a pointer to a structure
   256    283   of type lsm_compress. The lsm_config() method takes a copy of the 
   257    284   structures contents.
   258    285   <p>This option may only be used before lsm_open() is called. Invoking it
   259    286   after lsm_open() has been called results in an LSM_MISUSE error.
   260    287   <p><dt>LSM_CONFIG_GET_COMPRESSION<dd>Query the compression methods used to compress and decompress database
   261    288   content.
   262         -</dl><h2 id=compression>Compression and/or Encryption Hooks<a id=lsm_compress></a></h2>
          289  +<p><dt>LSM_CONFIG_SET_COMPRESSION_FACTORY<dd>Configure a factory method to be invoked in case of an LSM_MISMATCH
          290  +error.
          291  +</dl><h2 id=compression>Compression and/or Encryption Hooks<a id=lsm_compress></a><a id=lsm_compress></a></h2>
   263    292   <verbatim>struct lsm_compress {
   264    293     void *pCtx;
   265    294     unsigned int iId;
   266    295     int (*xBound)(void *, int nSrc);
   267    296     int (*xCompress)(void *, char *, int *, const char *, int);
   268    297     int (*xUncompress)(void *, char *, int *, const char *, int);
          298  +  void (*xFree)(void *pCtx);
          299  +};
          300  +struct lsm_compress_factory {
          301  +  void *pCtx;
          302  +  int (*xFactory)(void *, lsm_db *, u32);
          303  +  void (*xFree)(void *pCtx);
   269    304   };
   270    305   </verbatim>
   271    306   <h2 id=allocating>Allocating and Freeing Memory<a id=lsm_free></a></h2>
   272    307   <verbatim>void *lsm_malloc(lsm_env*, size_t);
   273    308   void *lsm_realloc(lsm_env*, void *, size_t);
   274    309   void lsm_free(lsm_env*, void *);
   275    310   </verbatim>
   276    311   <p>Invoke the memory allocation functions that belong to environment
   277    312   pEnv. Or the system defaults if no memory allocation functions have 
   278    313   been registered.
   279         -<h2 id=querying>Querying a Connection For Operational Data<a id=lsm_info></a><a id=LSM_INFO_NWRITE></a><a id=LSM_INFO_NREAD></a><a id=LSM_INFO_DB_STRUCTURE></a><a id=LSM_INFO_LOG_STRUCTURE></a><a id=LSM_INFO_ARRAY_STRUCTURE></a><a id=LSM_INFO_PAGE_ASCII_DUMP></a><a id=LSM_INFO_PAGE_HEX_DUMP></a><a id=LSM_INFO_FREELIST></a><a id=LSM_INFO_ARRAY_PAGES></a><a id=LSM_INFO_CHECKPOINT_SIZE></a><a id=LSM_INFO_TREE_SIZE></a></h2>
          314  +<h2 id=querying>Querying a Connection For Operational Data<a id=lsm_info></a><a id=LSM_INFO_NWRITE></a><a id=LSM_INFO_NREAD></a><a id=LSM_INFO_DB_STRUCTURE></a><a id=LSM_INFO_LOG_STRUCTURE></a><a id=LSM_INFO_ARRAY_STRUCTURE></a><a id=LSM_INFO_PAGE_ASCII_DUMP></a><a id=LSM_INFO_PAGE_HEX_DUMP></a><a id=LSM_INFO_FREELIST></a><a id=LSM_INFO_ARRAY_PAGES></a><a id=LSM_INFO_CHECKPOINT_SIZE></a><a id=LSM_INFO_TREE_SIZE></a><a id=LSM_INFO_FREELIST_SIZE></a></h2>
   280    315   <verbatim>int lsm_info(lsm_db *, int, ...);
   281    316   #define LSM_INFO_NWRITE           1
   282    317   #define LSM_INFO_NREAD            2
   283    318   #define LSM_INFO_DB_STRUCTURE     3
   284    319   #define LSM_INFO_LOG_STRUCTURE    4
   285    320   #define LSM_INFO_ARRAY_STRUCTURE  5
   286    321   #define LSM_INFO_PAGE_ASCII_DUMP  6
   287    322   #define LSM_INFO_PAGE_HEX_DUMP    7
   288    323   #define LSM_INFO_FREELIST         8
   289    324   #define LSM_INFO_ARRAY_PAGES      9
   290    325   #define LSM_INFO_CHECKPOINT_SIZE 10
   291    326   #define LSM_INFO_TREE_SIZE       11
          327  +#define LSM_INFO_FREELIST_SIZE   12
   292    328   </verbatim>
   293    329   <p>Query a database connection for operational statistics or data.
   294    330   The following values may be passed as the second argument to lsm_info().
   295    331   <p><dl><dt>LSM_INFO_NWRITE<dd>The third parameter should be of type (int *). The location pointed
   296    332   to by the third parameter is set to the number of 4KB pages written to
   297    333   the database file during the lifetime of this connection. 
   298    334   <p><dt>LSM_INFO_NREAD<dd>The third parameter should be of type (int *). The location pointed
................................................................................
   350    386   to is populated with a pointer to a nul-terminated string containing
   351    387   the string representation of a Tcl data-structure. The returned 
   352    388   string should be eventually freed by the caller using lsm_free().
   353    389   <p>The Tcl structure returned is a list containing one element for each
   354    390   free block in the database. The element itself consists of two 
   355    391   integers - the block number and the id of the snapshot that freed it.
   356    392   <p><dt>LSM_INFO_CHECKPOINT_SIZE<dd>The third argument should be of type (int *). The location pointed to
   357         -by this argument is populated with the number of bytes written to the
          393  +by this argument is populated with the number of KB written to the
   358    394   database file since the most recent checkpoint.
   359    395   <p><dt>LSM_INFO_TREE_SIZE<dd>If this value is passed as the second argument to an lsm_info() call, it
   360    396   should be followed by two arguments of type (int *) (for a total of four
   361    397   arguments).
   362    398   <p>At any time, there are either one or two tree structures held in shared
   363    399   memory that new database clients will access (there may also be additional
   364    400   tree structures being used by older clients - this API does not provide
   365    401   information on them). One tree structure - the current tree - is used to
   366    402   accumulate new data written to the database. The other tree structure -
   367    403   the old tree - is a read-only tree holding older data and may be flushed 
   368    404   to disk at any time.
   369    405   <p>Assuming no error occurs, the location pointed to by the first of the two
   370         -(int *) arguments is set to the size of the old in-memory tree in bytes.
          406  +(int *) arguments is set to the size of the old in-memory tree in KB.
   371    407   The second is set to the size of the current, or live in-memory tree.
   372    408   </dl><h2 id=opening>Opening and Closing Write Transactions<a id=lsm_begin></a><a id=lsm_commit></a><a id=lsm_rollback></a></h2>
   373    409   <verbatim>int lsm_begin(lsm_db *pDb, int iLevel);
   374    410   int lsm_commit(lsm_db *pDb, int iLevel);
   375    411   int lsm_rollback(lsm_db *pDb, int iLevel);
   376    412   </verbatim>
   377    413   <p>These functions are used to open and close transactions and nested 
................................................................................
   405    441   Delete a value from the database. No error is returned if the specified
   406    442   key value does not exist in the database.
   407    443   Delete all database entries with keys that are greater than (pKey1/nKey1) 
   408    444   and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and
   409    445   (pKey2/nKey2) themselves, if they exist in the database, are not deleted.
   410    446   <p>Return LSM_OK if successful, or an LSM error code otherwise.
   411    447   <h2 id=explicit>Explicit Database Work and Checkpointing<a id=lsm_work></a><a id=lsm_flush></a><a id=lsm_checkpoint></a></h2>
   412         -<verbatim>int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite);
          448  +<verbatim>int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite);
   413    449   int lsm_flush(lsm_db *pDb);
   414         -int lsm_checkpoint(lsm_db *pDb, int *pnByte);
          450  +int lsm_checkpoint(lsm_db *pDb, int *pnKB);
   415    451   </verbatim>
   416    452   <p>This function is called by a thread to work on the database structure.
   417    453   Attempt to checkpoint the current database snapshot. Return an LSM
   418    454   error code if an error occurs or LSM_OK otherwise.
   419    455   <p>If the current snapshot has already been checkpointed, calling this 
   420         -function is a no-op. In this case if pnByte is not NULL, *pnByte is
          456  +function is a no-op. In this case if pnKB is not NULL, *pnKB is
   421    457   set to 0. Or, if the current snapshot is successfully checkpointed
   422         -by this function and pbCkpt is not NULL, *pnByte is set to the number
          458  +by this function and pbKB is not NULL, *pnKB is set to the number
   423    459   of bytes written to the database file since the previous checkpoint
   424    460   (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query).
   425    461   <h2 id=opening>Opening and Closing Database Cursors<a id=lsm_csr_open></a><a id=lsm_csr_close></a></h2>
   426    462   <verbatim>int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr);
   427    463   int lsm_csr_close(lsm_cursor *pCsr);
   428    464   </verbatim>
   429    465   <p>Open and close a database cursor.

Changes to www/lsmusr.wiki.

  1146   1146   <p>The example code below might be executed in a background thread or process
  1147   1147   in order to perform database work and checkpointing. In this case all other
  1148   1148   clients should set the LSM_CONFIG_AUTOWORK parameter to zero.
  1149   1149   
  1150   1150   <verbatim>
  1151   1151     int rc;
  1152   1152     lsm_db *db;
  1153         -  int nCkpt = 4*1024*1024;
         1153  +  int nCkpt = 4*1024;             /* 4096KB == 4MB */
  1154   1154   
  1155   1155     /* Open a database connection to database "test.db". 
  1156   1156     **
  1157   1157     ** Configure the connection to automatically checkpoint the database after
  1158   1158     ** writing each 4MB of data to it (instead of the default 2MB). As well
  1159   1159     ** as to auto-work, the LSM_CONFIG_AUTOCHECKPOINT parameter applies to data
  1160   1160     ** written by explicit calls to lsm_work().
................................................................................
  1164   1164     lsm_open(db, "test.db");
  1165   1165   
  1166   1166     while( 1 ){
  1167   1167       int nWrite;
  1168   1168   
  1169   1169       /* Attempt up to 512KB of work. Set nWrite to the number of bytes
  1170   1170       ** actually written to disk.  */
  1171         -    rc = lsm_work(db, 2, 512*1024, &nWrite);
         1171  +    rc = lsm_work(db, 2, 512, &nWrite);
  1172   1172       if( rc!=LSM_OK && rc!=LSM_BUSY ){
  1173   1173         /* Anything other than LSM_OK or LSM_BUSY is a problem. LSM_BUSY
  1174   1174         ** indicates that some other client has taken the WORKER lock. Any
  1175   1175         ** other error indicates something has gone quite wrong.  */
  1176   1176         lsm_close(db);
  1177   1177         return rc;
  1178   1178       }