SQLite4
Check-in [1f0129ac39]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
SHA1 Hash:1f0129ac398e94d48064a6903016a9fe0cd42ec2
Date: 2013-03-04 10:24:54
User: dan
Comment:When in non-mmap mode, cache some page-data in memory. As sqlite does.
Tags And Properties
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to lsm-test/sqltest.c

30
31
32
33
34
35
36

37
38





39
40
41
42
43
44



45
46
47
48
49
50
51
...
180
181
182
183
184
185
186

187
188
189
190
191
192
193
...
194
195
196
197
198
199
200
201

202
203
204
205
206
207
208
209
...
235
236
237
238
239
240
241

242
243
244
245
246
247
248
249
250
251
252
253
254
255

256
257
258
259
260
261
262
263
...
302
303
304
305
306
307
308

309
310
311
312
313
314
315
316
317

318
319
320
321
322
323
324
325
326
327
328
329



330
331
332
333
334
335
336
337
338
339
340
341
342
343

344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360

361
362
363
364
365
366
367
368
369
370
371

372
373
374
375
376
377
378
379
...
395
396
397
398
399
400
401

402
403
404
405
406
407
408
409
410
411
412


413
414
415
416
417
418
419
420
...
446
447
448
449
450
451
452

453
454
455
456
457
458
459
460

461
462
463
464
465
466
467
468
469
470
471
472



473
474
475
476
477
478
479
480
481
482
483
484
485

486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503

/*
** Unlink database zDb and its supporting files (wal, shm, journal, and log).
** This function works with both lsm and sqlite3 databases.
*/
static int unlink_db(const char *zDb){
  int i;

  const char *azExt[] = { "", "-shm", "-wal", "-journal", "-log", 0 };






  for(i=0; azExt[i]; i++){
    char *zFile = sqlite4_mprintf(0, "%s%s", zDb, azExt[i]);
    unlink(zFile);
    sqlite4_free(0, zFile);
  }




  return 0;
}

static char *create_schema_sql(int nIdx){
  char *zSchema;
  int i;

................................................................................
  return iRet;
}
/*
** End of integer query implementations.
*************************************************************************/

static int do_insert1_test4(

  int nRow,                       /* Number of rows to insert in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int nIdx,                       /* Number of aux indexes (aside from PK) */
  int iSync                       /* PRAGMA synchronous value (0, 1 or 2) */
){
  char *zCreateTbl;               /* Create table statement */
  char *zInsert;                  /* INSERT statement */
................................................................................
  sqlite4_stmt *pInsert;          /* Compiled INSERT statement */
  sqlite4 *db = 0;                /* Database handle */
  int i;                          /* Counter to count nRow rows */
  int nMs;                        /* Test time in ms */

  lsm_db *pLsm;

  unlink_db(SQLITE4_DB_FILE);

  EXPLODE(  sqlite4_open(0, SQLITE4_DB_FILE, &db)  );
  sqlite4_kvstore_control(db, "main", SQLITE4_KVCTRL_LSM_HANDLE, &pLsm);
  i = iSync;
  lsm_config(pLsm, LSM_CONFIG_SAFETY, &i);
  assert( i==iSync );

  install_rblob_function4(db);

................................................................................
  nMs = testTimeGet();

  /* Print out the time taken by the test */
  printf("%.3f seconds\n", (double)nMs / 1000.0);
  return 0;
}
static int do_insert1_test3(

  int nRow,                       /* Number of rows to insert in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int nIdx,                       /* Number of aux indexes (aside from PK) */
  int iSync                       /* PRAGMA synchronous value (0, 1 or 2) */
){
  char *zCreateTbl;               /* Create table statement */
  char *zInsert;                  /* INSERT statement */
  char *zSync;                    /* "PRAGMA synchronous=" statement */
  sqlite3_stmt *pInsert;          /* Compiled INSERT statement */
  sqlite3 *db = 0;                /* Database handle */
  int i;                          /* Counter to count nRow rows */
  int nMs;                        /* Test time in ms */

  unlink_db(SQLITE3_DB_FILE);

  EXPLODE( sqlite3_open(SQLITE3_DB_FILE, &db) );
  EXPLODE( sqlite3_exec(db, "PRAGMA journal_mode=WAL", 0, 0, 0) );
  zSync = sqlite4_mprintf(0, "PRAGMA synchronous=%d", iSync);
  EXPLODE( sqlite3_exec(db, zSync, 0, 0, 0) );
  sqlite4_free(0, zSync);

  install_rblob_function3(db);

................................................................................
    int nMax;
  } aArg[] = { 
    {"-db",           3,    4}, 
    {"-rows",         1,    10000000}, 
    {"-rowspertrans", 1,    10000000}, 
    {"-indexes",      0,    20}, 
    {"-sync",         0,    2}, 

    {0,0,0}
  };
  int i;

  int iDb = 4;                    /* SQLite 3 or 4 */
  int nRow = 50000;               /* Total rows: 50000 */
  int nRowPerTrans = 10;          /* Total rows each transaction: 50000 */
  int nIdx = 3;                   /* Number of auxilliary indexes */
  int iSync = 1;                  /* PRAGMA synchronous setting */


  for(i=0; i<argc; i++){
    int iSel;
    int iVal;
    int rc;

    rc = testArgSelectX(aArg, "argument", sizeof(aArg[0]), argv[i], &iSel);
    if( rc!=0 ) return -1;
    if( i==argc-1 ){
      fprintf(stderr, "option %s requires an argument\n", aArg[iSel].zArg);
      return -1;
    }



    iVal = atoi(argv[++i]);
    if( iVal<aArg[iSel].nMin || iVal>aArg[iSel].nMax ){
      fprintf(stderr, "option %s out of range (%d..%d)\n", 
          aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax 
      );
      return -1;
    }

    switch( iSel ){
      case 0: iDb = iVal;          break;
      case 1: nRow = iVal;         break;
      case 2: nRowPerTrans = iVal; break;
      case 3: nIdx = iVal;         break;
      case 4: iSync = iVal;        break;

    }
  }

  printf("insert1: db=%d rows=%d rowspertrans=%d indexes=%d sync=%d ... ", 
      iDb, nRow, nRowPerTrans, nIdx, iSync
  );
  fflush(stdout);
  if( iDb==3 ){
    do_insert1_test3(nRow, nRowPerTrans, nIdx, iSync);
  }else{
    do_insert1_test4(nRow, nRowPerTrans, nIdx, iSync);
  }

  return 0;
}

static int do_select1_test4(

  int nRow,                       /* Number of rows to read in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int iIdx
){
  int nMs = 0;
  sqlite4_stmt *pSelect = 0;
  char *zSelect;
  sqlite4 *db;
  int i;
  int nTblRow;


  EXPLODE( sqlite4_open(0, SQLITE4_DB_FILE, &db) );
  install_rblob_function4(db);

  nTblRow = integer_query4(db, "SELECT count(*) FROM t1");

  /* Create the db schema and prepare the INSERT statement */
  zSelect = create_select_sql(iIdx);
  EXPLODE(  sqlite4_prepare(db, zSelect, -1, &pSelect, 0)  );
................................................................................
  sqlite4_close(db);
  sqlite4_free(0, zSelect);

  printf("%.3f seconds\n", (double)nMs / 1000.0);
  return 0;
}
static int do_select1_test3(

  int nRow,                       /* Number of rows to read in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int iIdx
){
  int nMs = 0;
  sqlite3_stmt *pSelect = 0;
  char *zSelect;
  sqlite3 *db;
  int i;
  int nTblRow;



  EXPLODE( sqlite3_open(SQLITE3_DB_FILE, &db) );
  install_rblob_function3(db);

  nTblRow = integer_query3(db, "SELECT count(*) FROM t1");

  /* Create the db schema and prepare the INSERT statement */
  zSelect = create_select_sql(iIdx);
  EXPLODE(  sqlite3_prepare(db, zSelect, -1, &pSelect, 0)  );
................................................................................
    int nMin;
    int nMax;
  } aArg[] = {
    {"-db",           3,    4}, 
    {"-rows",         1,    10000000}, 
    {"-rowspertrans", 1,    10000000}, 
    {"-index",        0,    21}, 

    {0,0,0}
  };
  int i;

  int iDb = 4;                    /* SQLite 3 or 4 */
  int nRow = 50000;               /* Total rows: 50000 */
  int nRowPerTrans = 10;          /* Total rows each transaction: 50000 */
  int iIdx = 0;


  for(i=0; i<argc; i++){
    int iSel;
    int iVal;
    int rc;

    rc = testArgSelectX(aArg, "argument", sizeof(aArg[0]), argv[i], &iSel);
    if( rc!=0 ) return -1;
    if( i==argc-1 ){
      fprintf(stderr, "option %s requires an argument\n", aArg[iSel].zArg);
      return -1;
    }



    iVal = atoi(argv[++i]);
    if( iVal<aArg[iSel].nMin || iVal>aArg[iSel].nMax ){
      fprintf(stderr, "option %s out of range (%d..%d)\n", 
          aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax 
      );
      return -1;
    }

    switch( iSel ){
      case 0: iDb = iVal;          break;
      case 1: nRow = iVal;         break;
      case 2: nRowPerTrans = iVal; break;
      case 3: iIdx = iVal;         break;

    }
  }

  printf("select1: db=%d rows=%d rowspertrans=%d index=%d ... ", 
      iDb, nRow, nRowPerTrans, iIdx
  );
  fflush(stdout);
  if( iDb==3 ){
    do_select1_test3(nRow, nRowPerTrans, iIdx);
  }else{
    do_select1_test4(nRow, nRowPerTrans, iIdx);
  }

  return 0;
}

int main(int argc, char **argv){
  struct SqltestArg {







>


>
>
>
>
>

|




>
>
>







 







>







 







|
>
|







 







>













|
>
|







 







>









>












>
>
>
|
|
|
|
|
|
|

|
|
|
|
|
|
>








|

|






>











>
|







 







>










|
>
>
|







 







>








>



<








>
>
>
|
|
|
|
|
|
|

|
|
|
|
|
>








|

|







30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
...
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
...
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
...
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
...
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
...
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
...
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489

490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532

/*
** Unlink database zDb and its supporting files (wal, shm, journal, and log).
** This function works with both lsm and sqlite3 databases.
*/
static int unlink_db(const char *zDb){
  int i;
  char *zBase = (char *)zDb;
  const char *azExt[] = { "", "-shm", "-wal", "-journal", "-log", 0 };

  if( 0==sqlite4_strnicmp("file:", zDb, 5) ){
    for(i=5; zDb[i] && zDb[i]!='?'; i++);
    zBase = sqlite4_mprintf(0, "%.*s", i-5, &zDb[5]);
  }

  for(i=0; azExt[i]; i++){
    char *zFile = sqlite4_mprintf(0, "%s%s", zBase, azExt[i]);
    unlink(zFile);
    sqlite4_free(0, zFile);
  }

  if( zBase!=zDb ){
    sqlite4_free(0, zBase);
  }
  return 0;
}

static char *create_schema_sql(int nIdx){
  char *zSchema;
  int i;

................................................................................
  return iRet;
}
/*
** End of integer query implementations.
*************************************************************************/

static int do_insert1_test4(
  const char *zFile,
  int nRow,                       /* Number of rows to insert in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int nIdx,                       /* Number of aux indexes (aside from PK) */
  int iSync                       /* PRAGMA synchronous value (0, 1 or 2) */
){
  char *zCreateTbl;               /* Create table statement */
  char *zInsert;                  /* INSERT statement */
................................................................................
  sqlite4_stmt *pInsert;          /* Compiled INSERT statement */
  sqlite4 *db = 0;                /* Database handle */
  int i;                          /* Counter to count nRow rows */
  int nMs;                        /* Test time in ms */

  lsm_db *pLsm;

  if( zFile==0 ) zFile = SQLITE4_DB_FILE;
  unlink_db(zFile);
  EXPLODE(  sqlite4_open(0, zFile, &db)  );
  sqlite4_kvstore_control(db, "main", SQLITE4_KVCTRL_LSM_HANDLE, &pLsm);
  i = iSync;
  lsm_config(pLsm, LSM_CONFIG_SAFETY, &i);
  assert( i==iSync );

  install_rblob_function4(db);

................................................................................
  nMs = testTimeGet();

  /* Print out the time taken by the test */
  printf("%.3f seconds\n", (double)nMs / 1000.0);
  return 0;
}
static int do_insert1_test3(
  const char *zFile,
  int nRow,                       /* Number of rows to insert in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int nIdx,                       /* Number of aux indexes (aside from PK) */
  int iSync                       /* PRAGMA synchronous value (0, 1 or 2) */
){
  char *zCreateTbl;               /* Create table statement */
  char *zInsert;                  /* INSERT statement */
  char *zSync;                    /* "PRAGMA synchronous=" statement */
  sqlite3_stmt *pInsert;          /* Compiled INSERT statement */
  sqlite3 *db = 0;                /* Database handle */
  int i;                          /* Counter to count nRow rows */
  int nMs;                        /* Test time in ms */

  if( zFile==0 ) zFile = SQLITE3_DB_FILE;
  unlink_db(zFile);
  EXPLODE( sqlite3_open(zFile, &db) );
  EXPLODE( sqlite3_exec(db, "PRAGMA journal_mode=WAL", 0, 0, 0) );
  zSync = sqlite4_mprintf(0, "PRAGMA synchronous=%d", iSync);
  EXPLODE( sqlite3_exec(db, zSync, 0, 0, 0) );
  sqlite4_free(0, zSync);

  install_rblob_function3(db);

................................................................................
    int nMax;
  } aArg[] = { 
    {"-db",           3,    4}, 
    {"-rows",         1,    10000000}, 
    {"-rowspertrans", 1,    10000000}, 
    {"-indexes",      0,    20}, 
    {"-sync",         0,    2}, 
    {"-file",         -1,    -1}, 
    {0,0,0}
  };
  int i;

  int iDb = 4;                    /* SQLite 3 or 4 */
  int nRow = 50000;               /* Total rows: 50000 */
  int nRowPerTrans = 10;          /* Total rows each transaction: 50000 */
  int nIdx = 3;                   /* Number of auxilliary indexes */
  int iSync = 1;                  /* PRAGMA synchronous setting */
  const char *zFile = 0;

  for(i=0; i<argc; i++){
    int iSel;
    int iVal;
    int rc;

    rc = testArgSelectX(aArg, "argument", sizeof(aArg[0]), argv[i], &iSel);
    if( rc!=0 ) return -1;
    if( i==argc-1 ){
      fprintf(stderr, "option %s requires an argument\n", aArg[iSel].zArg);
      return -1;
    }
    if( 0==strcmp("-file", aArg[iSel].zArg) ){
      zFile = argv[++i];
    }else{
      iVal = atoi(argv[++i]);
      if( iVal<aArg[iSel].nMin || iVal>aArg[iSel].nMax ){
        fprintf(stderr, "option %s out of range (%d..%d)\n", 
            aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax 
            );
        return -1;
      }

      switch( iSel ){
        case 0: iDb = iVal;          break;
        case 1: nRow = iVal;         break;
        case 2: nRowPerTrans = iVal; break;
        case 3: nIdx = iVal;         break;
        case 4: iSync = iVal;        break;
      }
    }
  }

  printf("insert1: db=%d rows=%d rowspertrans=%d indexes=%d sync=%d ... ", 
      iDb, nRow, nRowPerTrans, nIdx, iSync
  );
  fflush(stdout);
  if( iDb==3 ){
    do_insert1_test3(zFile, nRow, nRowPerTrans, nIdx, iSync);
  }else{
    do_insert1_test4(zFile, nRow, nRowPerTrans, nIdx, iSync);
  }

  return 0;
}

static int do_select1_test4(
  const char *zFile,
  int nRow,                       /* Number of rows to read in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int iIdx
){
  int nMs = 0;
  sqlite4_stmt *pSelect = 0;
  char *zSelect;
  sqlite4 *db;
  int i;
  int nTblRow;

  if( zFile==0 ) zFile = SQLITE4_DB_FILE;
  EXPLODE( sqlite4_open(0, zFile, &db) );
  install_rblob_function4(db);

  nTblRow = integer_query4(db, "SELECT count(*) FROM t1");

  /* Create the db schema and prepare the INSERT statement */
  zSelect = create_select_sql(iIdx);
  EXPLODE(  sqlite4_prepare(db, zSelect, -1, &pSelect, 0)  );
................................................................................
  sqlite4_close(db);
  sqlite4_free(0, zSelect);

  printf("%.3f seconds\n", (double)nMs / 1000.0);
  return 0;
}
static int do_select1_test3(
  const char *zFile,
  int nRow,                       /* Number of rows to read in total */
  int nRowPerTrans,               /* Number of rows per transaction */
  int iIdx
){
  int nMs = 0;
  sqlite3_stmt *pSelect = 0;
  char *zSelect;
  sqlite3 *db;
  int i;
  int nTblRow;
  

  if( zFile==0 ) zFile = SQLITE3_DB_FILE;
  EXPLODE( sqlite3_open(zFile, &db) );
  install_rblob_function3(db);

  nTblRow = integer_query3(db, "SELECT count(*) FROM t1");

  /* Create the db schema and prepare the INSERT statement */
  zSelect = create_select_sql(iIdx);
  EXPLODE(  sqlite3_prepare(db, zSelect, -1, &pSelect, 0)  );
................................................................................
    int nMin;
    int nMax;
  } aArg[] = {
    {"-db",           3,    4}, 
    {"-rows",         1,    10000000}, 
    {"-rowspertrans", 1,    10000000}, 
    {"-index",        0,    21}, 
    {"-file",        -1,    -1}, 
    {0,0,0}
  };
  int i;

  int iDb = 4;                    /* SQLite 3 or 4 */
  int nRow = 50000;               /* Total rows: 50000 */
  int nRowPerTrans = 10;          /* Total rows each transaction: 50000 */
  int iIdx = 0;
  const char *zFile = 0;

  for(i=0; i<argc; i++){
    int iSel;

    int rc;

    rc = testArgSelectX(aArg, "argument", sizeof(aArg[0]), argv[i], &iSel);
    if( rc!=0 ) return -1;
    if( i==argc-1 ){
      fprintf(stderr, "option %s requires an argument\n", aArg[iSel].zArg);
      return -1;
    }
    if( 0==strcmp("-file", aArg[iSel].zArg) ){
      zFile = argv[++i];
    }else{
      int iVal = atoi(argv[++i]);
      if( iVal<aArg[iSel].nMin || iVal>aArg[iSel].nMax ){
        fprintf(stderr, "option %s out of range (%d..%d)\n", 
            aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax 
            );
        return -1;
      }

      switch( iSel ){
        case 0: iDb = iVal;          break;
        case 1: nRow = iVal;         break;
        case 2: nRowPerTrans = iVal; break;
        case 3: iIdx = iVal;         break;
      }
    }
  }

  printf("select1: db=%d rows=%d rowspertrans=%d index=%d ... ", 
      iDb, nRow, nRowPerTrans, iIdx
  );
  fflush(stdout);
  if( iDb==3 ){
    do_select1_test3(zFile, nRow, nRowPerTrans, iIdx);
  }else{
    do_select1_test4(zFile, nRow, nRowPerTrans, iIdx);
  }

  return 0;
}

int main(int argc, char **argv){
  struct SqltestArg {

Changes to src/kvlsm.c

449
450
451
452
453
454
455


456
457
458
459
460
461
462
463
464
465
466
467

468
469
470
471
472
473
474
  if( pNew==0 ){
    rc = SQLITE4_NOMEM;
  }else{
    struct Config {
      const char *zParam;
      int eParam;
    } aConfig[] = {


      { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE },
      { "lsm_multiple_processes", LSM_CONFIG_MULTIPLE_PROCESSES }
    };

    memset(pNew, 0, sizeof(KVLsm));
    pNew->base.pStoreVfunc = &kvlsmMethods;
    pNew->base.pEnv = pEnv;
    rc = lsm_new(0, &pNew->pDb);
    if( rc==SQLITE4_OK ){
      int i;
      int bMmap = 0;
      lsm_config(pNew->pDb, LSM_CONFIG_MMAP, &bMmap);

      for(i=0; i<ArraySize(aConfig); i++){
        const char *zVal = sqlite4_uri_parameter(zName, aConfig[i].zParam);
        if( zVal ){
          int nVal = sqlite4Atoi(zVal);
          lsm_config(pNew->pDb, aConfig[i].eParam, &nVal);
        }
      }







>
>












>







449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
  if( pNew==0 ){
    rc = SQLITE4_NOMEM;
  }else{
    struct Config {
      const char *zParam;
      int eParam;
    } aConfig[] = {
      { "lsm_mmap", LSM_CONFIG_MMAP },
      { "lsm_page_size", LSM_CONFIG_PAGE_SIZE },
      { "lsm_block_size", LSM_CONFIG_BLOCK_SIZE },
      { "lsm_multiple_processes", LSM_CONFIG_MULTIPLE_PROCESSES }
    };

    memset(pNew, 0, sizeof(KVLsm));
    pNew->base.pStoreVfunc = &kvlsmMethods;
    pNew->base.pEnv = pEnv;
    rc = lsm_new(0, &pNew->pDb);
    if( rc==SQLITE4_OK ){
      int i;
      int bMmap = 0;
      lsm_config(pNew->pDb, LSM_CONFIG_MMAP, &bMmap);

      for(i=0; i<ArraySize(aConfig); i++){
        const char *zVal = sqlite4_uri_parameter(zName, aConfig[i].zParam);
        if( zVal ){
          int nVal = sqlite4Atoi(zVal);
          lsm_config(pNew->pDb, aConfig[i].eParam, &nVal);
        }
      }

Changes to src/lsmInt.h

764
765
766
767
768
769
770


771
772
773
774
775
776
777

void lsmEnvSleep(lsm_env *, int);

int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);

int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);



/*
** End of functions from "lsm_file.c".
**************************************************************************/

/* 
** Functions from file "lsm_sorted.c".







>
>







764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779

void lsmEnvSleep(lsm_env *, int);

int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);

int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);

void lsmFsPurgeCache(FileSystem *);

/*
** End of functions from "lsm_file.c".
**************************************************************************/

/* 
** Functions from file "lsm_sorted.c".

Changes to src/lsm_file.c

558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
...
725
726
727
728
729
730
731

732
733
734
735
736
737
738
...
883
884
885
886
887
888
889










































890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913

914

915
916
917



918
919

920
921
922
923
924
925
926
....
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
....
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
....
2107
2108
2109
2110
2111
2112
2113

2114
2115
2116
2117
2118
2119
2120
....
2303
2304
2305
2306
2307
2308
2309
















2310
2311
2312
2313
2314
2315
2316
....
2366
2367
2368
2369
2370
2371
2372

2373
2374
2375
2376
2377
2378
2379
....
2511
2512
2513
2514
2515
2516
2517

2518
2519
2520
2521



2522
2523
2524
2525
2526
2527
2528
    /* Make a copy of the database and log file names. */
    memcpy(pFS->zDb, zDb, nDb+1);
    memcpy(pFS->zLog, zDb, nDb);
    memcpy(&pFS->zLog[nDb], "-log", 5);

    /* Allocate the hash-table here. At some point, it should be changed
    ** so that it can grow dynamicly. */
    pFS->nCacheMax = 2048;
    pFS->nHash = 4096;
    pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc);

    /* Open the database file */
    pLsmFile = lsmDbRecycleFd(pDb);
    if( pLsmFile ){
      pFS->pLsmFile = pLsmFile;
................................................................................

/*
** Configure the nominal page-size used by this file-system. Actual 
** pages may be smaller or larger than this value.
*/
void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){
  pFS->nPagesize = nPgsz;

}

/*
** Configure the block-size used by this file-system. Actual pages may be 
** smaller or larger than this value.
*/
void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){
................................................................................
  Page **pp;

  iHash = fsHashKey(pFS->nHash, pPg->iPg);
  for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext);
  *pp = pPg->pHashNext;
}











































static int fsPageBuffer(
  FileSystem *pFS, 
  int bRequireData,               /* True to allocate buffer as well */
  Page **ppOut
){
  int rc = LSM_OK;
  Page *pPage = 0;
  if( pFS->bUseMmap || pFS->pLruFirst==0 || pFS->nCacheAlloc<pFS->nCacheMax ){
    pPage = lsmMallocZero(pFS->pEnv, sizeof(Page));
    if( !pPage ){
      rc = LSM_NOMEM_BKPT;
    }else if( bRequireData ){
      pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize);
      pPage->flags = PAGE_FREE;
      if( !pPage->aData ){
        lsmFree(pFS->pEnv, pPage);
        rc = LSM_NOMEM_BKPT;
        pPage = 0;
      }
      pFS->nCacheAlloc++;
    }else{
      fsPageAddToLru(pFS, pPage);
    }
  }else{

    pPage = pFS->pLruFirst;

    fsPageRemoveFromLru(pFS, pPage);
    fsPageRemoveFromHash(pFS, pPage);
  }




  assert( pPage==0 || (pPage->flags & PAGE_DIRTY)==0 );

  *ppOut = pPage;
  return rc;
}

static void fsPageBufferFree(Page *pPg){
  if( pPg->flags & PAGE_FREE ){
    lsmFree(pPg->pFS->pEnv, pPg->aData);
................................................................................
    /* Search the hash-table for the page */
    iHash = fsHashKey(pFS->nHash, iReal);
    for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
      if( p->iPg==iReal) break;
    }

    if( p==0 ){
      rc = fsPageBuffer(pFS, 1, &p);
      if( rc==LSM_OK ){
        int nSpace = 0;
        p->iPg = iReal;
        p->nRef = 0;
        p->pFS = pFS;
        assert( p->flags==0 || p->flags==PAGE_FREE );

................................................................................

  assert( p->pRedirect==0 );

  if( pFS->pCompress || bDefer ){
    /* In compressed database mode the page is not assigned a page number
    ** or location in the database file at this point. This will be done
    ** by the lsmFsPagePersist() call.  */
    rc = fsPageBuffer(pFS, 1, &pPg);
    if( rc==LSM_OK ){
      pPg->pFS = pFS;
      pPg->pSeg = p;
      pPg->iPg = 0;
      pPg->flags |= PAGE_DIRTY;
      pPg->nData = pFS->nPagesize;
      assert( pPg->aData );
................................................................................
        i64 iOff = iFromOff + i*nSz;
        rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz);
        if( rc==LSM_OK ){
          iOff = iToOff + i*nSz;
          rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz);
        }
      }

    }
  }

  /* Update append-point list if necessary */
  if( rc==LSM_OK ){
    int i;
    for(i=0; i<LSM_APPLIST_SZ; i++){
................................................................................
    if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg);
    assert( pPg->nRef==1 );
    lsmFsPageRelease(pPg);
    pPg = pNext;
  }
  *pRc = rc;
}

















/*
** If the page passed as an argument is dirty, update the database file
** (or mapping of the database file) with its current contents and mark
** the page as clean.
**
** Return LSM_OK if the operation is a success, or an LSM error code
................................................................................
        assert( pPg->nData==pFS->nPagesize-4 );

        rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext);
        if( rc!=LSM_OK ) return rc;

        if( pFS->bUseMmap==0 ){
          int iHash = fsHashKey(pFS->nHash, pPg->iPg);

          pPg->pHashNext = pFS->apHash[iHash];
          pFS->apHash[iHash] = pPg;
          assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg );
        }

        if( iPrev ){
          assert( iNext==0 );
................................................................................
      pPg->aData -= (pPg->flags & PAGE_HASPREV);
      pPg->flags &= ~PAGE_HASPREV;

      if( pFS->bUseMmap ){
        pPg->pHashNext = pFS->pFree;
        pFS->pFree = pPg;
      }else{

        assert( pPg->pLruNext==0 );
        assert( pPg->pLruPrev==0 );
        fsPageRemoveFromHash(pFS, pPg);
        fsPageBufferFree(pPg);



      }
    }
  }

  return rc;
}








|







 







>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


<








|








<
<


>

>


|
>
>
>
|
<
>







 







|







 







|







 







>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>




>
>
>







558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
...
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
...
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934

935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951


952
953
954
955
956
957
958
959
960
961
962
963

964
965
966
967
968
969
970
971
....
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
....
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
....
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
....
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
....
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
....
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
    /* Make a copy of the database and log file names. */
    memcpy(pFS->zDb, zDb, nDb+1);
    memcpy(pFS->zLog, zDb, nDb);
    memcpy(&pFS->zLog[nDb], "-log", 5);

    /* Allocate the hash-table here. At some point, it should be changed
    ** so that it can grow dynamicly. */
    pFS->nCacheMax = 2048*1024 / pFS->nPagesize;
    pFS->nHash = 4096;
    pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc);

    /* Open the database file */
    pLsmFile = lsmDbRecycleFd(pDb);
    if( pLsmFile ){
      pFS->pLsmFile = pLsmFile;
................................................................................

/*
** Configure the nominal page-size used by this file-system. Actual 
** pages may be smaller or larger than this value.
*/
void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){
  pFS->nPagesize = nPgsz;
  pFS->nCacheMax = 2048*1024 / pFS->nPagesize;
}

/*
** Configure the block-size used by this file-system. Actual pages may be 
** smaller or larger than this value.
*/
void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){
................................................................................
  Page **pp;

  iHash = fsHashKey(pFS->nHash, pPg->iPg);
  for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext);
  *pp = pPg->pHashNext;
}


/*
** Purge the page cache of all entries with nRef==0.
*/
void lsmFsPurgeCache(FileSystem *pFS){
  Page *pPg;

  pPg = pFS->pLruFirst;
  while( pPg ){
    Page *pNext = pPg->pLruNext;
    fsPageRemoveFromHash(pFS, pPg);
    if( pPg->flags & PAGE_FREE ){
      lsmFree(pFS->pEnv, pPg->aData);
    }
    lsmFree(pFS->pEnv, pPg);
    pPg = pNext;
    pFS->nCacheAlloc--;
  }
  pFS->pLruFirst = 0;
  pFS->pLruLast = 0;

  assert( pFS->nCacheAlloc<=pFS->nOut && pFS->nCacheAlloc>=0 );
}

/*
** Search the hash-table for page iPg. If an entry is round, return a pointer
** to it. Otherwise, return NULL.
**
** Either way, if argument piHash is not NULL set *piHash to the hash slot
** number that page iPg would be stored in before returning.
*/
static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
  Page *p;                        /* Return value */
  int iHash = fsHashKey(pFS->nHash, iPg);

  if( piHash ) *piHash = iHash;
  for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
    if( p->iPg==iPg) break;
  }
  return p;
}

static int fsPageBuffer(
  FileSystem *pFS, 

  Page **ppOut
){
  int rc = LSM_OK;
  Page *pPage = 0;
  if( pFS->bUseMmap || pFS->pLruFirst==0 || pFS->nCacheAlloc<pFS->nCacheMax ){
    pPage = lsmMallocZero(pFS->pEnv, sizeof(Page));
    if( !pPage ){
      rc = LSM_NOMEM_BKPT;
    }else{
      pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize);
      pPage->flags = PAGE_FREE;
      if( !pPage->aData ){
        lsmFree(pFS->pEnv, pPage);
        rc = LSM_NOMEM_BKPT;
        pPage = 0;
      }
      pFS->nCacheAlloc++;


    }
  }else{
    u8 *aData;
    pPage = pFS->pLruFirst;
    aData = pPage->aData;
    fsPageRemoveFromLru(pFS, pPage);
    fsPageRemoveFromHash(pFS, pPage);

    memset(pPage, 0, sizeof(Page));
    pPage->flags = PAGE_FREE;
    pPage->aData = aData;
  }


  *ppOut = pPage;
  return rc;
}

static void fsPageBufferFree(Page *pPg){
  if( pPg->flags & PAGE_FREE ){
    lsmFree(pPg->pFS->pEnv, pPg->aData);
................................................................................
    /* Search the hash-table for the page */
    iHash = fsHashKey(pFS->nHash, iReal);
    for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
      if( p->iPg==iReal) break;
    }

    if( p==0 ){
      rc = fsPageBuffer(pFS, &p);
      if( rc==LSM_OK ){
        int nSpace = 0;
        p->iPg = iReal;
        p->nRef = 0;
        p->pFS = pFS;
        assert( p->flags==0 || p->flags==PAGE_FREE );

................................................................................

  assert( p->pRedirect==0 );

  if( pFS->pCompress || bDefer ){
    /* In compressed database mode the page is not assigned a page number
    ** or location in the database file at this point. This will be done
    ** by the lsmFsPagePersist() call.  */
    rc = fsPageBuffer(pFS, &pPg);
    if( rc==LSM_OK ){
      pPg->pFS = pFS;
      pPg->pSeg = p;
      pPg->iPg = 0;
      pPg->flags |= PAGE_DIRTY;
      pPg->nData = pFS->nPagesize;
      assert( pPg->aData );
................................................................................
        i64 iOff = iFromOff + i*nSz;
        rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz);
        if( rc==LSM_OK ){
          iOff = iToOff + i*nSz;
          rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz);
        }
      }
      lsmFsPurgeCache(pFS);
    }
  }

  /* Update append-point list if necessary */
  if( rc==LSM_OK ){
    int i;
    for(i=0; i<LSM_APPLIST_SZ; i++){
................................................................................
    if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg);
    assert( pPg->nRef==1 );
    lsmFsPageRelease(pPg);
    pPg = pNext;
  }
  *pRc = rc;
}

static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
  Page *p;
  int iHash = fsHashKey(pFS->nHash, iPg);

  for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext);

  if( p ){
    assert( p->nRef==0 );
    fsPageRemoveFromHash(pFS, p);
    p->iPg = 0;
    iHash = fsHashKey(pFS->nHash, 0);
    p->pHashNext = pFS->apHash[iHash];
    pFS->apHash[iHash] = p;
  }
}

/*
** If the page passed as an argument is dirty, update the database file
** (or mapping of the database file) with its current contents and mark
** the page as clean.
**
** Return LSM_OK if the operation is a success, or an LSM error code
................................................................................
        assert( pPg->nData==pFS->nPagesize-4 );

        rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext);
        if( rc!=LSM_OK ) return rc;

        if( pFS->bUseMmap==0 ){
          int iHash = fsHashKey(pFS->nHash, pPg->iPg);
          fsRemoveHashEntry(pFS, pPg->iPg);
          pPg->pHashNext = pFS->apHash[iHash];
          pFS->apHash[iHash] = pPg;
          assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg );
        }

        if( iPrev ){
          assert( iNext==0 );
................................................................................
      pPg->aData -= (pPg->flags & PAGE_HASPREV);
      pPg->flags &= ~PAGE_HASPREV;

      if( pFS->bUseMmap ){
        pPg->pHashNext = pFS->pFree;
        pFS->pFree = pPg;
      }else{
#if 0
        assert( pPg->pLruNext==0 );
        assert( pPg->pLruPrev==0 );
        fsPageRemoveFromHash(pFS, pPg);
        fsPageBufferFree(pPg);
#else
        fsPageAddToLru(pFS, pPg);
#endif
      }
    }
  }

  return rc;
}

Changes to src/lsm_shared.c

1148
1149
1150
1151
1152
1153
1154

1155
1156
1157
1158
1159
1160
1161

    /* Load the database snapshot */
    if( rc==LSM_OK ){
      if( lsmCheckpointClientCacheOk(pDb)==0 ){
        lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
        pDb->pClient = 0;
        lsmMCursorFreeCache(pDb);

        rc = lsmCheckpointLoad(pDb, &iSnap);
      }else{
        iSnap = 1;
      }
    }

    /* Take a read-lock on the tree and snapshot just loaded. Then check







>







1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162

    /* Load the database snapshot */
    if( rc==LSM_OK ){
      if( lsmCheckpointClientCacheOk(pDb)==0 ){
        lsmFreeSnapshot(pDb->pEnv, pDb->pClient);
        pDb->pClient = 0;
        lsmMCursorFreeCache(pDb);
        lsmFsPurgeCache(pDb->pFS);
        rc = lsmCheckpointLoad(pDb, &iSnap);
      }else{
        iSnap = 1;
      }
    }

    /* Take a read-lock on the tree and snapshot just loaded. Then check

Changes to src/lsm_sorted.c

5308
5309
5310
5311
5312
5313
5314


5315
5316
5317
5318
5319
5320
5321
....
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
  int nPage;                      /* Equivalent of nKB in pages */
  int nWrite = 0;                 /* Number of pages written */

  /* This function may not be called if pDb has an open read or write
  ** transaction. Return LSM_MISUSE if an application attempts this.  */
  if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;
  if( nMerge<=0 ) nMerge = pDb->nMerge;



  /* Convert from KB to pages */
  nPgsz = lsmFsPageSize(pDb->pFS);
  if( nKB>=0 ){
    nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz;
  }else{
    nPage = -1;
................................................................................
  int bKeys,                      /* Output the keys from each segment */
  int bVals,                      /* Output the values from each segment */
  const char *zWhy                /* Caption to print near top of dump */
){
  Snapshot *pDump = pSnap;
  Level *pTopLevel;
  char *zFree = 0;


  assert( pSnap );
  pTopLevel = lsmDbSnapshotLevel(pDump);
  if( pDb->xLog && pTopLevel ){
    static int nCall = 0;
    Level *pLevel;
    int iLevel = 0;







>
>







 







<







5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
....
5836
5837
5838
5839
5840
5841
5842

5843
5844
5845
5846
5847
5848
5849
  int nPage;                      /* Equivalent of nKB in pages */
  int nWrite = 0;                 /* Number of pages written */

  /* This function may not be called if pDb has an open read or write
  ** transaction. Return LSM_MISUSE if an application attempts this.  */
  if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT;
  if( nMerge<=0 ) nMerge = pDb->nMerge;

  lsmFsPurgeCache(pDb->pFS);

  /* Convert from KB to pages */
  nPgsz = lsmFsPageSize(pDb->pFS);
  if( nKB>=0 ){
    nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz;
  }else{
    nPage = -1;
................................................................................
  int bKeys,                      /* Output the keys from each segment */
  int bVals,                      /* Output the values from each segment */
  const char *zWhy                /* Caption to print near top of dump */
){
  Snapshot *pDump = pSnap;
  Level *pTopLevel;
  char *zFree = 0;


  assert( pSnap );
  pTopLevel = lsmDbSnapshotLevel(pDump);
  if( pDb->xLog && pTopLevel ){
    static int nCall = 0;
    Level *pLevel;
    int iLevel = 0;