/ Check-in [41c95439]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Many new configuration options for fuzzershell.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 41c954391690833bf2ece4ea14457ee4bdd74e90
User & Date: drh 2015-04-20 18:58:38
Context
2015-04-20
22:36
Add the --mode option to fuzzershell. check-in: b940b0fa user: drh tags: trunk
18:58
Many new configuration options for fuzzershell. check-in: 41c95439 user: drh tags: trunk
15:13
Fix a memory leak caused by duplicate entries in the sqlite_stat1 table. check-in: 2f58c8c9 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to tool/fuzzershell.c.

43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

62
63
64
65
66
67
68
..
92
93
94
95
96
97
98
















99
100
101
102
103
104
105
...
222
223
224
225
226
227
228

229

230
231






232
233


























































234
235
236
237
238
239
240
...
242
243
244
245
246
247
248












249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264










265
266
267




























268
269
270
271
272
273
274
275
276



























277
278
279
280
281
282
283
...
304
305
306
307
308
309
310




311
312
313
314
315
316
317
...
318
319
320
321
322
323
324
325


326
327
328
329
330
331
332
...
340
341
342
343
344
345
346




347
348
** in-memory SQLite database is created to run each chunk of SQL.  This
** feature allows the "queue" of AFL to be captured into a single big
** file using a command like this:
**
**    (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt
**
** (Once again, change the "|" to "/") Then all elements of the AFL queue
** can be run in a single go (for regression testing, for example, by typing:
**
**    fuzzershell -f ~/all-queue.txt >out.txt
**
** After running each chunk of SQL, the database connection is closed.  The
** program aborts if the close fails or if there is any unfreed memory after
** the close.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>

#include "sqlite3.h"

/*
** All global variables are gathered into the "g" singleton.
*/
struct GlobalVars {
  const char *zArgv0;         /* Name of program */
................................................................................
  fprintf(stderr, "%s: ", g.zArgv0);
  va_start(ap, zFormat);
  vfprintf(stderr, zFormat, ap);
  va_end(ap);
  fprintf(stderr, "\n");
  exit(1);
}

















/*
** This callback is invoked by sqlite3_log().
*/
static void shellLog(void *pNotUsed, int iErrCode, const char *zMsg){
  printf("LOG: (%d) %s\n", iErrCode, zMsg);
}
................................................................................
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
  printf("Usage: %s [options]\n", g.zArgv0);
  printf(
"Read SQL text from standard input and evaluate it.\n"
"Options:\n"

"  -f FILE             Read SQL text from FILE instead of standard input\n"

"  --help              Show this help text\n"    
"  --initdb DBFILE     Initialize the in-memory database using template DBFILE\n"






  );
}




























































int main(int argc, char **argv){
  char *zIn = 0;          /* Input text */
  int nAlloc = 0;         /* Number of bytes allocated for zIn[] */
  int nIn = 0;            /* Number of bytes of zIn[] used */
  size_t got;             /* Bytes read from input */
................................................................................
  int rc = SQLITE_OK;     /* Result codes from API functions */
  int i;                  /* Loop counter */
  int iNext;              /* Next block of SQL */
  sqlite3 *db;            /* Open database */
  sqlite3 *dbInit = 0;    /* On-disk database used to initialize the in-memory db */
  const char *zInitDb = 0;/* Name of the initialization database file */
  char *zErrMsg = 0;      /* Error message returned from sqlite3_exec() */













  g.zArgv0 = argv[0];
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"help")==0 ){
        showHelp();
        return 0;
      }else
      if( strcmp(z, "f")==0 && i+1<argc ){
        if( in!=stdin ) abendError("only one -f allowed");
        in = fopen(argv[++i],"rb");
        if( in==0 )  abendError("cannot open input file \"%s\"", argv[i]);
      }else










      if( strcmp(z, "initdb")==0 && i+1<argc ){
        if( zInitDb!=0 ) abendError("only one --initdb allowed");
        zInitDb = argv[++i];




























      }else
      {
        abendError("unknown option: %s", argv[i]);
      }
    }else{
      abendError("unknown argument: %s", argv[i]);
    }
  }
  sqlite3_config(SQLITE_CONFIG_LOG, shellLog, 0);



























  while( !feof(in) ){
    nAlloc += nAlloc+1000;
    zIn = realloc(zIn, nAlloc);
    if( zIn==0 ) fatalError("out of memory");
    got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); 
    nIn += (int)got;
    zIn[nIn] = 0;
................................................................................
    rc = sqlite3_open_v2(
      "main.db", &db,
      SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY,
      0);
    if( rc!=SQLITE_OK ){
      abendError("Unable to open the in-memory database");
    }




    if( zInitDb ){
      sqlite3_backup *pBackup;
      pBackup = sqlite3_backup_init(db, "main", dbInit, "main");
      rc = sqlite3_backup_step(pBackup, -1);
      if( rc!=SQLITE_DONE ){
        abendError("attempt to initialize the in-memory database failed (rc=%d)",
                   rc);
................................................................................
      }
      sqlite3_backup_finish(pBackup);
    }
    sqlite3_trace(db, traceCallback, 0);
    sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);



    cSaved = zIn[iNext];
    zIn[iNext] = 0;
    printf("INPUT (offset: %d, size: %d): [%s]\n",
            i, (int)strlen(&zIn[i]), &zIn[i]);
    rc = sqlite3_exec(db, &zIn[i], execCallback, 0, &zErrMsg);
    zIn[iNext] = cSaved;

................................................................................
      abendError("sqlite3_close() failed with rc=%d", rc);
    }
    if( sqlite3_memory_used()>0 ){
      abendError("memory in use after close: %lld bytes", sqlite3_memory_used());
    }
  }
  free(zIn);




  return 0;
}







|











>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>

>


>
>
>
>
>
>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>







|
|
<






>
>
>
>
>
>
>
>
>
>



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>







 







|
>
>







 







>
>
>
>


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
..
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
...
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
...
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352

353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
...
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
...
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
...
505
506
507
508
509
510
511
512
513
514
515
516
517
** in-memory SQLite database is created to run each chunk of SQL.  This
** feature allows the "queue" of AFL to be captured into a single big
** file using a command like this:
**
**    (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt
**
** (Once again, change the "|" to "/") Then all elements of the AFL queue
** can be run in a single go (for regression testing, for example) by typing:
**
**    fuzzershell -f ~/all-queue.txt >out.txt
**
** After running each chunk of SQL, the database connection is closed.  The
** program aborts if the close fails or if there is any unfreed memory after
** the close.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <ctype.h>
#include "sqlite3.h"

/*
** All global variables are gathered into the "g" singleton.
*/
struct GlobalVars {
  const char *zArgv0;         /* Name of program */
................................................................................
  fprintf(stderr, "%s: ", g.zArgv0);
  va_start(ap, zFormat);
  vfprintf(stderr, zFormat, ap);
  va_end(ap);
  fprintf(stderr, "\n");
  exit(1);
}

/*
** Evaluate some SQL.  Abort if unable.
*/
static void sqlexec(sqlite3 *db, const char *zFormat, ...){
  va_list ap;
  char *zSql;
  char *zErrMsg = 0;
  int rc;
  va_start(ap, zFormat);
  zSql = sqlite3_vmprintf(zFormat, ap);
  va_end(ap);
  rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
  if( rc ) abendError("failed sql [%s]: %s", zSql, zErrMsg);
  sqlite3_free(zSql);
}

/*
** This callback is invoked by sqlite3_log().
*/
static void shellLog(void *pNotUsed, int iErrCode, const char *zMsg){
  printf("LOG: (%d) %s\n", iErrCode, zMsg);
}
................................................................................
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
  printf("Usage: %s [options]\n", g.zArgv0);
  printf(
"Read SQL text from standard input and evaluate it.\n"
"Options:\n"
"  --autovacuum        Enable AUTOVACUUM mode\n"
"  -f FILE             Read SQL text from FILE instead of standard input\n"
"  --heap SZ MIN       Memory allocator uses SZ bytes & min allocation MIN\n"
"  --help              Show this help text\n"    
"  --initdb DBFILE     Initialize the in-memory database using template DBFILE\n"
"  --lookaside N SZ    Configure lookaside for N slots of SZ bytes each\n"
"  --pagesize N        Set the page size to N\n"
"  --pcache N SZ       Configure N pages of pagecache each of size SZ bytes\n"
"  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"
"  --utf16be           Set text encoding to UTF-16BE\n"
"  --utf16le           Set text encoding to UTF-16LE\n"
  );
}

/*
** Return the value of a hexadecimal digit.  Return -1 if the input
** is not a hex digit.
*/
static int hexDigitValue(char c){
  if( c>='0' && c<='9' ) return c - '0';
  if( c>='a' && c<='f' ) return c - 'a' + 10;
  if( c>='A' && c<='F' ) return c - 'A' + 10;
  return -1;
}

/*
** Interpret zArg as an integer value, possibly with suffixes.
*/
static int integerValue(const char *zArg){
  sqlite3_int64 v = 0;
  static const struct { char *zSuffix; int iMult; } aMult[] = {
    { "KiB", 1024 },
    { "MiB", 1024*1024 },
    { "GiB", 1024*1024*1024 },
    { "KB",  1000 },
    { "MB",  1000000 },
    { "GB",  1000000000 },
    { "K",   1000 },
    { "M",   1000000 },
    { "G",   1000000000 },
  };
  int i;
  int isNeg = 0;
  if( zArg[0]=='-' ){
    isNeg = 1;
    zArg++;
  }else if( zArg[0]=='+' ){
    zArg++;
  }
  if( zArg[0]=='0' && zArg[1]=='x' ){
    int x;
    zArg += 2;
    while( (x = hexDigitValue(zArg[0]))>=0 ){
      v = (v<<4) + x;
      zArg++;
    }
  }else{
    while( isdigit(zArg[0]) ){
      v = v*10 + zArg[0] - '0';
      zArg++;
    }
  }
  for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){
    if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
      v *= aMult[i].iMult;
      break;
    }
  }
  if( v>0x7fffffff ) abendError("parameter too large - max 2147483648");
  return (int)(isNeg? -v : v);
}


int main(int argc, char **argv){
  char *zIn = 0;          /* Input text */
  int nAlloc = 0;         /* Number of bytes allocated for zIn[] */
  int nIn = 0;            /* Number of bytes of zIn[] used */
  size_t got;             /* Bytes read from input */
................................................................................
  int rc = SQLITE_OK;     /* Result codes from API functions */
  int i;                  /* Loop counter */
  int iNext;              /* Next block of SQL */
  sqlite3 *db;            /* Open database */
  sqlite3 *dbInit = 0;    /* On-disk database used to initialize the in-memory db */
  const char *zInitDb = 0;/* Name of the initialization database file */
  char *zErrMsg = 0;      /* Error message returned from sqlite3_exec() */
  const char *zEncoding = 0;    /* --utf16be or --utf16le */
  int nHeap = 0, mnHeap = 0;    /* Heap size from --heap */
  int nLook = 0, szLook = 0;    /* --lookaside configuration */
  int nPCache = 0, szPCache = 0;/* --pcache configuration */
  int nScratch = 0, szScratch=0;/* --scratch configuration */
  int pageSize = 0;             /* Desired page size.  0 means default */
  void *pHeap = 0;              /* Allocated heap space */
  void *pLook = 0;              /* Allocated lookaside space */
  void *pPCache = 0;            /* Allocated storage for pcache */
  void *pScratch = 0;           /* Allocated storage for scratch */
  int doAutovac = 0;            /* True for --autovacuum */


  g.zArgv0 = argv[0];
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"autovacuum")==0 ){
        doAutovac = 1;

      }else
      if( strcmp(z, "f")==0 && i+1<argc ){
        if( in!=stdin ) abendError("only one -f allowed");
        in = fopen(argv[++i],"rb");
        if( in==0 )  abendError("cannot open input file \"%s\"", argv[i]);
      }else
      if( strcmp(z,"heap")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s\n", argv[i]);
        nHeap = integerValue(argv[i+1]);
        mnHeap = integerValue(argv[i+2]);
        i += 2;
      }else
      if( strcmp(z,"help")==0 ){
        showHelp();
        return 0;
      }else
      if( strcmp(z, "initdb")==0 && i+1<argc ){
        if( zInitDb!=0 ) abendError("only one --initdb allowed");
        zInitDb = argv[++i];
      }else
      if( strcmp(z,"lookaside")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nLook = integerValue(argv[i+1]);
        szLook = integerValue(argv[i+2]);
        i += 2;
      }else
      if( strcmp(z,"pagesize")==0 ){
        if( i>=argc-1 ) abendError("missing argument on %s", argv[i]);
        pageSize = integerValue(argv[++i]);
      }else
      if( strcmp(z,"pcache")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nPCache = integerValue(argv[i+1]);
        szPCache = integerValue(argv[i+2]);
        i += 2;
      }else
      if( strcmp(z,"scratch")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nScratch = integerValue(argv[i+1]);
        szScratch = integerValue(argv[i+2]);
        i += 2;
      }else
      if( strcmp(z,"utf16le")==0 ){
        zEncoding = "utf16le";
      }else
      if( strcmp(z,"utf16be")==0 ){
        zEncoding = "utf16be";
      }else
      {
        abendError("unknown option: %s", argv[i]);
      }
    }else{
      abendError("unknown argument: %s", argv[i]);
    }
  }
  sqlite3_config(SQLITE_CONFIG_LOG, shellLog, 0);
  if( nHeap>0 ){
    pHeap = malloc( nHeap );
    if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap);
    rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap);
    if( rc ) abendError("heap configuration failed: %d\n", rc);
  }
  if( nLook>0 ){
    sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0);
    if( szLook>0 ){
      pLook = malloc( nLook*szLook );
      if( pLook==0 ) fatalError("out of memory");
    }
  }
  if( nScratch>0 && szScratch>0 ){
    pScratch = malloc( nScratch*(sqlite3_int64)szScratch );
    if( pScratch==0 ) fatalError("cannot allocate %lld-byte scratch",
                                 nScratch*(sqlite3_int64)szScratch);
    rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch);
    if( rc ) abendError("scratch configuration failed: %d\n", rc);
  }
  if( nPCache>0 && szPCache>0 ){
    pPCache = malloc( nPCache*(sqlite3_int64)szPCache );
    if( pPCache==0 ) fatalError("cannot allocate %lld-byte pcache",
                                 nPCache*(sqlite3_int64)szPCache);
    rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache);
    if( rc ) abendError("pcache configuration failed: %d", rc);
  }
  while( !feof(in) ){
    nAlloc += nAlloc+1000;
    zIn = realloc(zIn, nAlloc);
    if( zIn==0 ) fatalError("out of memory");
    got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); 
    nIn += (int)got;
    zIn[nIn] = 0;
................................................................................
    rc = sqlite3_open_v2(
      "main.db", &db,
      SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY,
      0);
    if( rc!=SQLITE_OK ){
      abendError("Unable to open the in-memory database");
    }
    if( pLook ){
      rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, pLook, szLook, nLook);
      if( rc!=SQLITE_OK ) abendError("lookaside configuration filed: %d", rc);
    }
    if( zInitDb ){
      sqlite3_backup *pBackup;
      pBackup = sqlite3_backup_init(db, "main", dbInit, "main");
      rc = sqlite3_backup_step(pBackup, -1);
      if( rc!=SQLITE_DONE ){
        abendError("attempt to initialize the in-memory database failed (rc=%d)",
                   rc);
................................................................................
      }
      sqlite3_backup_finish(pBackup);
    }
    sqlite3_trace(db, traceCallback, 0);
    sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);
    if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding);
    if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
    if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");
    cSaved = zIn[iNext];
    zIn[iNext] = 0;
    printf("INPUT (offset: %d, size: %d): [%s]\n",
            i, (int)strlen(&zIn[i]), &zIn[i]);
    rc = sqlite3_exec(db, &zIn[i], execCallback, 0, &zErrMsg);
    zIn[iNext] = cSaved;

................................................................................
      abendError("sqlite3_close() failed with rc=%d", rc);
    }
    if( sqlite3_memory_used()>0 ){
      abendError("memory in use after close: %lld bytes", sqlite3_memory_used());
    }
  }
  free(zIn);
  free(pHeap);
  free(pLook);
  free(pScratch);
  free(pPCache);
  return 0;
}