Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the "stat" command to kvtest.c. Also add the --variance option to the "init" command. Add the tool/kvtest-speed.sh script used for doing performance testing on key/value access patterns. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
b63deed600b1a457a6960ebad5645f4d |
User & Date: | drh 2017-01-31 15:29:05.242 |
Context
2017-01-31
| ||
19:10 | Very small performance improvements and size reductions in sqlite3VdbeExec() and blobSeekToRow(). (check-in: 85dddf2b45 user: drh tags: trunk) | |
16:34 | Remove an unnecessary initialization of the pOp variable in sqlite3VdbeExec(). (check-in: 02f6293f27 user: drh tags: micro-optimizations) | |
15:29 | Add the "stat" command to kvtest.c. Also add the --variance option to the "init" command. Add the tool/kvtest-speed.sh script used for doing performance testing on key/value access patterns. (check-in: b63deed600 user: drh tags: trunk) | |
15:27 | Fix a typo in a comment. (check-in: bd22bf9cbe user: drh tags: trunk) | |
Changes
Changes to test/kvtest.c.
︙ | ︙ | |||
63 64 65 66 67 68 69 | static const char zHelp[] = "Usage: kvtest COMMAND ARGS...\n" "\n" " kvtest init DBFILE --count N --size M --pagesize X\n" "\n" " Generate a new test database file named DBFILE containing N\n" " BLOBs each of size M bytes. The page size of the new database\n" | | > > > > > > | 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | static const char zHelp[] = "Usage: kvtest COMMAND ARGS...\n" "\n" " kvtest init DBFILE --count N --size M --pagesize X\n" "\n" " Generate a new test database file named DBFILE containing N\n" " BLOBs each of size M bytes. The page size of the new database\n" " file will be X. Additional options:\n" "\n" " --variance V Randomly vary M by plus or minus V\n" "\n" " kvtest export DBFILE DIRECTORY\n" "\n" " Export all the blobs in the kv table of DBFILE into separate\n" " files in DIRECTORY.\n" "\n" " kvtest stat DBFILE\n" "\n" " Display summary information about DBFILE\n" "\n" " kvtest run DBFILE [options]\n" "\n" " Run a performance test. DBFILE can be either the name of a\n" " database or a directory containing sample files. Options:\n" "\n" " --asc Read blobs in ascending order\n" " --blob-api Use the BLOB API\n" |
︙ | ︙ | |||
247 248 249 250 251 252 253 254 255 256 257 258 259 260 | ** Do database initialization. */ static int initMain(int argc, char **argv){ char *zDb; int i, rc; int nCount = 1000; int sz = 10000; int pgsz = 4096; sqlite3 *db; char *zSql; char *zErrMsg = 0; assert( strcmp(argv[1],"init")==0 ); assert( argc>=3 ); | > | 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 | ** Do database initialization. */ static int initMain(int argc, char **argv){ char *zDb; int i, rc; int nCount = 1000; int sz = 10000; int iVariance = 0; int pgsz = 4096; sqlite3 *db; char *zSql; char *zErrMsg = 0; assert( strcmp(argv[1],"init")==0 ); assert( argc>=3 ); |
︙ | ︙ | |||
270 271 272 273 274 275 276 277 278 279 280 281 282 283 | continue; } if( strcmp(z, "-size")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); sz = integerValue(argv[++i]); if( sz<1 ) fatalError("the --size must be positive"); continue; } if( strcmp(z, "-pagesize")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); pgsz = integerValue(argv[++i]); if( pgsz<512 || pgsz>65536 || ((pgsz-1)&pgsz)!=0 ){ fatalError("the --pagesize must be power of 2 between 512 and 65536"); } | > > > > > | 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 | continue; } if( strcmp(z, "-size")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); sz = integerValue(argv[++i]); if( sz<1 ) fatalError("the --size must be positive"); continue; } if( strcmp(z, "-variance")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); iVariance = integerValue(argv[++i]); continue; } if( strcmp(z, "-pagesize")==0 ){ if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]); pgsz = integerValue(argv[++i]); if( pgsz<512 || pgsz>65536 || ((pgsz-1)&pgsz)!=0 ){ fatalError("the --pagesize must be power of 2 between 512 and 65536"); } |
︙ | ︙ | |||
292 293 294 295 296 297 298 | zSql = sqlite3_mprintf( "DROP TABLE IF EXISTS kv;\n" "PRAGMA page_size=%d;\n" "VACUUM;\n" "BEGIN;\n" "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n" "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 | zSql = sqlite3_mprintf( "DROP TABLE IF EXISTS kv;\n" "PRAGMA page_size=%d;\n" "VACUUM;\n" "BEGIN;\n" "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n" "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n" "COMMIT;\n", pgsz, nCount, sz, iVariance ); rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); if( rc ) fatalError("database create failed: %s", zErrMsg); sqlite3_free(zSql); sqlite3_close(db); return 0; } /* ** Analyze an existing database file. Report its content. */ static int statMain(int argc, char **argv){ char *zDb; int i, rc; sqlite3 *db; char *zSql; sqlite3_stmt *pStmt; assert( strcmp(argv[1],"stat")==0 ); assert( argc>=3 ); zDb = argv[2]; for(i=3; i<argc; i++){ char *z = argv[i]; if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z); if( z[1]=='-' ) z++; fatalError("unknown option: \"%s\"", argv[i]); } rc = sqlite3_open(zDb, &db); if( rc ){ fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); } zSql = sqlite3_mprintf( "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))" " FROM kv" ); rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); sqlite3_free(zSql); if( sqlite3_step(pStmt)==SQLITE_ROW ){ printf("Number of entries: %8d\n", sqlite3_column_int(pStmt, 0)); printf("Average value size: %8d\n", sqlite3_column_int(pStmt, 3)); printf("Minimum value size: %8d\n", sqlite3_column_int(pStmt, 1)); printf("Maximum value size: %8d\n", sqlite3_column_int(pStmt, 2)); }else{ printf("No rows\n"); } sqlite3_finalize(pStmt); zSql = sqlite3_mprintf("PRAGMA page_size"); rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); sqlite3_free(zSql); if( sqlite3_step(pStmt)==SQLITE_ROW ){ printf("Page-size: %8d\n", sqlite3_column_int(pStmt, 0)); } sqlite3_finalize(pStmt); zSql = sqlite3_mprintf("PRAGMA page_count"); rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db)); sqlite3_free(zSql); if( sqlite3_step(pStmt)==SQLITE_ROW ){ printf("Page-count: %8d\n", sqlite3_column_int(pStmt, 0)); } sqlite3_finalize(pStmt); sqlite3_close(db); return 0; } /* ** Implementation of the "writefile(X,Y)" SQL function. The argument Y ** is written into file X. The number of bytes written is returned. Or ** NULL is returned if something goes wrong, such as being unable to open ** file X for writing. */ |
︙ | ︙ | |||
796 797 798 799 800 801 802 803 804 805 806 | return initMain(argc, argv); } if( strcmp(argv[1],"export")==0 ){ return exportMain(argc, argv); } if( strcmp(argv[1],"run")==0 ){ return runMain(argc, argv); } showHelp(); return 0; } | > > > | 867 868 869 870 871 872 873 874 875 876 877 878 879 880 | return initMain(argc, argv); } if( strcmp(argv[1],"export")==0 ){ return exportMain(argc, argv); } if( strcmp(argv[1],"run")==0 ){ return runMain(argc, argv); } if( strcmp(argv[1],"stat")==0 ){ return statMain(argc, argv); } showHelp(); return 0; } |
Added tool/kvtest-speed.sh.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | #!/bin/bash # # A script for running speed tests using kvtest. # # The test database must be set up first. Recommended # command-line: # # ./kvtest init kvtest.db --count 100K --size 12K --variance 5K if test "$1" = "" then echo "Usage: $0 OUTPUTFILE [OPTIONS]" exit fi NAME=$1 shift OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD" KVARGS="--count 100K --stats" gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest # First run using SQL rm cachegrind.out.[1-9][0-9]* valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-sql-$NAME.txt # Second run using the sqlite3_blob object rm cachegrind.out.[1-9][0-9]* valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt # Diff the sqlite3_blob API analysis for non-trunk runs. if test "$NAME" != "trunk"; then fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt & fi |