SQLite

Check-in [f6fcac6ae8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improvements to the kvtest.c utility. Added the --cache-size option. Additional reporting of version and settings at the end of "./kvtest run".
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | kvtest
Files: files | file ages | folders
SHA1: f6fcac6ae8b3bffee6bf994eef2064affd301a95
User & Date: drh 2016-12-29 14:44:43.614
Context
2016-12-29
15:26
In kvtest.c: do not show the --cache-size setting on fopen() runs. Show progress for the (potentially slow) "export" command. (check-in: b0f58d2470 user: drh tags: kvtest)
14:44
Improvements to the kvtest.c utility. Added the --cache-size option. Additional reporting of version and settings at the end of "./kvtest run". (check-in: f6fcac6ae8 user: drh tags: kvtest)
03:57
Add the kvtest.c test program for measuring key/value read performance under various scenarios. (check-in: 489e0787c1 user: drh tags: kvtest)
Changes
Unified Diff Ignore Whitespace Patch
Changes to test/kvtest.c.
32
33
34
35
36
37
38
39
40

41
42
43

44
45
46
47
48
49
50
"        files in DIRECTORY.\n"
"\n"
"   kvhelp run DBFILE [options]\n"
"\n"
"        Run a performance test.  DBFILE can be either the name of a\n"
"        database or a directory containing sample files.  Options:\n"
"\n"
"             --count N            Read N blobs\n"
"             --blob-api           Use the BLOB API\n"

"             --random             Read blobs in a random order\n"
"             --desc               Read blobs in descending order\n"
"             --max-id N           Maximum blob key to use\n"

"             --start N            Start reading with this blob key\n"
;

/* Reference resources used */
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>







|

>
|


>







32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
"        files in DIRECTORY.\n"
"\n"
"   kvhelp run DBFILE [options]\n"
"\n"
"        Run a performance test.  DBFILE can be either the name of a\n"
"        database or a directory containing sample files.  Options:\n"
"\n"
"             --asc                Read blobs in ascending order\n"
"             --blob-api           Use the BLOB API\n"
"             --cache-size N       Database cache size\n"
"             --count N            Read N blobs\n"
"             --desc               Read blobs in descending order\n"
"             --max-id N           Maximum blob key to use\n"
"             --random             Read blobs in a random order\n"
"             --start N            Start reading with this blob key\n"
;

/* Reference resources used */
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
310
311
312
313
314
315
316


317
318
319
320
321
322
323
  char *zDb;                  /* Database or directory name */
  int i;                      /* Loop counter */
  int rc;                     /* Return code from SQLite calls */
  int nCount = 1000;          /* Number of blob fetch operations */
  int nExtra = 0;             /* Extra cycles */
  int iKey = 1;               /* Next blob key */
  int iMax = 1000;            /* Largest allowed key */


  int bBlobApi = 0;           /* Use the incremental blob I/O API */
  int eOrder = ORDER_ASC;     /* Access order */
  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
  sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
  sqlite3_int64 tmStart;      /* Start time */
  sqlite3_int64 tmElapsed;    /* Elapsed time */







>
>







312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
  char *zDb;                  /* Database or directory name */
  int i;                      /* Loop counter */
  int rc;                     /* Return code from SQLite calls */
  int nCount = 1000;          /* Number of blob fetch operations */
  int nExtra = 0;             /* Extra cycles */
  int iKey = 1;               /* Next blob key */
  int iMax = 1000;            /* Largest allowed key */
  int iPagesize = 0;          /* Database page size */
  int iCache = 1000;          /* Database cache size in kibibytes */
  int bBlobApi = 0;           /* Use the incremental blob I/O API */
  int eOrder = ORDER_ASC;     /* Access order */
  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
  sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
  sqlite3_int64 tmStart;      /* Start time */
  sqlite3_int64 tmElapsed;    /* Elapsed time */
349
350
351
352
353
354
355





356
357
358
359
360
361
362
      continue;
    }
    if( strcmp(z, "-start")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iKey = atoi(argv[++i]);
      if( iKey<1 ) fatalError("the --start must be positive");
      continue;





    }
    if( strcmp(z, "-random")==0 ){
      eOrder = ORDER_RANDOM;
      continue;
    }
    if( strcmp(z, "-asc")==0 ){
      eOrder = ORDER_ASC;







>
>
>
>
>







353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
      continue;
    }
    if( strcmp(z, "-start")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iKey = atoi(argv[++i]);
      if( iKey<1 ) fatalError("the --start must be positive");
      continue;
    }
    if( strcmp(z, "-cache-size")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iCache = atoi(argv[++i]);
      continue;
    }
    if( strcmp(z, "-random")==0 ){
      eOrder = ORDER_RANDOM;
      continue;
    }
    if( strcmp(z, "-asc")==0 ){
      eOrder = ORDER_ASC;
370
371
372
373
374
375
376

377
378
379
380

















381
382
383
384
385
386
387
      bBlobApi = 1;
      continue;
    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  tmStart = timeOfDay();
  if( eType==PATH_DB ){

    rc = sqlite3_open(zDb, &db);
    if( rc ){
      fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
    }

















    sqlite3_exec(db, "BEGIN", 0, 0, 0);
  }
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR ){
      /* CASE 1: Reading blobs out of separate files */
      char *zKey;
      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);







>




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
      bBlobApi = 1;
      continue;
    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  tmStart = timeOfDay();
  if( eType==PATH_DB ){
    char *zSql;
    rc = sqlite3_open(zDb, &db);
    if( rc ){
      fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
    }
    zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache);
    sqlite3_exec(db, zSql, 0, 0, 0);
    sqlite3_free(zSql);
    pStmt = 0;
    sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0);
    if( sqlite3_step(pStmt)==SQLITE_ROW ){
      iPagesize = sqlite3_column_int(pStmt, 0);
    }
    sqlite3_finalize(pStmt);
    sqlite3_prepare_v2(db, "PRAGMA cache_size", -1, &pStmt, 0);
    if( sqlite3_step(pStmt)==SQLITE_ROW ){
      iCache = sqlite3_column_int(pStmt, 0);
    }else{
      iCache = 0;
    }
    sqlite3_finalize(pStmt);
    pStmt = 0;
    sqlite3_exec(db, "BEGIN", 0, 0, 0);
  }
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR ){
      /* CASE 1: Reading blobs out of separate files */
      char *zKey;
      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
446
447
448
449
450
451
452










453
454
455
456
457
458
459
  if( pStmt ) sqlite3_finalize(pStmt);
  if( pBlob ) sqlite3_blob_close(pBlob);
  if( db ) sqlite3_close(db);
  tmElapsed = timeOfDay() - tmStart;
  if( nExtra ){
    printf("%d cycles due to %d misses\n", nCount, nExtra);
  }










  printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);
  printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
  printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
  return 0;
}









>
>
>
>
>
>
>
>
>
>







473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
  if( pStmt ) sqlite3_finalize(pStmt);
  if( pBlob ) sqlite3_blob_close(pBlob);
  if( db ) sqlite3_close(db);
  tmElapsed = timeOfDay() - tmStart;
  if( nExtra ){
    printf("%d cycles due to %d misses\n", nCount, nExtra);
  }
  if( eType==PATH_DB ){
    printf("SQLite version: %s\n", sqlite3_libversion());
  }
  printf("--count %d --max-id %d --cache-size %d", nCount-nExtra, iMax, iCache);
  switch( eOrder ){
    case ORDER_RANDOM:  printf(" --random\n");  break;
    case ORDER_DESC:    printf(" --desc\n");    break;
    default:            printf(" --asc\n");     break;
  }
  if( iPagesize ) printf("Database page size: %d\n", iPagesize);
  printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);
  printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
  printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
  return 0;
}