/ Check-in [627ea83c]
Login
☰Home Timeline Branches

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add AFL-generated test cases in the test/fuzzdata1.txt file. Automatically run fuzzershell against those cases on a "make test".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 627ea83c26b420088f101801eb8765127f47d2d0
User & Date: drh 2015-04-24 16:09:12
Context
2015-04-24
16:16
Add the fuzztest target to the MSVC makefile. (check-in: 98edc615 user: drh tags: trunk)
16:09
Add AFL-generated test cases in the test/fuzzdata1.txt file. Automatically run fuzzershell against those cases on a "make test". (check-in: 627ea83c user: drh tags: trunk)
14:47
Add the --unique-cases option to fuzzershell. (check-in: 7cb71849 user: drh tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in. 

947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964




965
966
967
968
969
970
971
972
973
974
975

TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

testfixture$(TEXE):	$(TESTFIXTURE_SRC)
	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not aeverything.
fulltestonly:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/full.test





# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	testfixture$(TEXE) sqlite3$(TEXE)
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind








|










>
>
>
>



|








947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979

TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

testfixture$(TEXE):	$(TESTFIXTURE_SRC)
	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	testfixture$(TEXE) sqlite3$(TEXE) fuzztest
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not aeverything.
fulltestonly:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzershell$(TEXE)
	./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt

# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	testfixture$(TEXE) sqlite3$(TEXE) fuzztest
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	testfixture$(TEXE) sqlite3$(TEXE)
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

Changes to main.mk. 

629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647



648
649
650
651
652
653
654
655


fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner




test:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \








|





|





>
>
>
|








629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658


fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	testfixture$(EXE) sqlite3$(EXE) fuzztest
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	testfixture$(EXE) sqlite3$(EXE) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner

fuzztest:	fuzzershell$(EXE)
	./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt

test:	testfixture$(EXE) sqlite3$(EXE) fuzztest
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \

Added test/fuzzdata1.txt.

cannot compute difference between binary files

Changes to test/releasetest.tcl. 

195
196
197
198
199
200
201

202
203
204
205
206
207
208
...
251
252
253
254
255
256
257

258
259
260
261
262
263
264

  # different names for them all so that they results appear in separate
  # subdirectories.
  #
  Fail0 {-O0}
  Fail2 {-O0}
  Fail3 {-O0}
  Fail4 {-O0}

}]

array set ::Platforms [strip_comments {
  Linux-x86_64 {
    "Check-Symbols"           checksymbols
    "Debug-One"               "mptest test"
    "Have-Not"                test
................................................................................
  #
  Failure-Detection {
    Fail0     "TEST_FAILURE=0 test"
    Sanitize  "TEST_FAILURE=1 test"
    Fail2     "TEST_FAILURE=2 valgrindtest"
    Fail3     "TEST_FAILURE=3 valgrindtest"
    Fail4     "TEST_FAILURE=4 test"

  }
}]


# End of configuration section.
#########################################################################
#########################################################################








>







 







>








195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
...
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266

  # different names for them all so that they results appear in separate
  # subdirectories.
  #
  Fail0 {-O0}
  Fail2 {-O0}
  Fail3 {-O0}
  Fail4 {-O0}
  FuzzFail {-O0}
}]

array set ::Platforms [strip_comments {
  Linux-x86_64 {
    "Check-Symbols"           checksymbols
    "Debug-One"               "mptest test"
    "Have-Not"                test
................................................................................
  #
  Failure-Detection {
    Fail0     "TEST_FAILURE=0 test"
    Sanitize  "TEST_FAILURE=1 test"
    Fail2     "TEST_FAILURE=2 valgrindtest"
    Fail3     "TEST_FAILURE=3 valgrindtest"
    Fail4     "TEST_FAILURE=4 test"
    FuzzFail  "TEST_FAILURE=5 test"
  }
}]


# End of configuration section.
#########################################################################
#########################################################################

Changes to tool/fuzzershell.c. 

369
370
371
372
373
374
375

376
377
378
379
380
381
382
...
520
521
522
523
524
525
526





527
528
529
530
531
532
533
534
...
578
579
580
581
582
583
584
585
586
587

588
589
590
591
592
593
594
...
617
618
619
620
621
622
623





624



625
626
627
628
629
630
631

632

633
634
635
636
637
638
639

  int quietFlag = 0;            /* --quiet or -q flag */
  int nTest = 0;                /* Number of test cases run */
  int multiTest = 0;            /* True if there will be multiple test cases */
  int lastPct = -1;             /* Previous percentage done output */
  sqlite3 *dataDb = 0;          /* Database holding compacted input data */
  sqlite3_stmt *pStmt = 0;      /* Statement to insert testcase into dataDb */
  const char *zDataOut = 0;     /* Write compacted data to this output file */



  g.zArgv0 = argv[0];
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
................................................................................
  }
  if( zInitDb ){
    rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0);
    if( rc!=SQLITE_OK ){
      abendError("unable to open initialization database \"%s\"", zInitDb);
    }
  }





  for(i=nTest=0; i<nIn; i=iNext, nTest++){
    char cSaved;
    if( strncmp(&zIn[i], "/****<",6)==0 ){
      char *z = strstr(&zIn[i], ">****/");
      if( z ){
        z += 6;
        if( verboseFlag ) printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]);
        i += (int)(z-&zIn[i]);
................................................................................
    if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
    if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");
    zSql = &zIn[i];
    if( verboseFlag ){
      printf("INPUT (offset: %d, size: %d): [%s]\n",
              i, (int)strlen(&zIn[i]), &zIn[i]);
    }else if( multiTest && !quietFlag ){
      int pct = 100*(i+strlen(zSql))/nIn;
      if( pct!=lastPct ){
        printf("%d%%\r", pct);

        fflush(stdout);
        lastPct = pct;
      }
    }
    switch( iMode ){
      case FZMODE_Glob:
        zSql = zToFree = sqlite3_mprintf("SELECT glob(%s);", zSql);
................................................................................
    rc = sqlite3_close(db);
    if( rc ){
      abendError("sqlite3_close() failed with rc=%d", rc);
    }
    if( sqlite3_memory_used()>0 ){
      abendError("memory in use after close: %lld bytes", sqlite3_memory_used());
    }





  }



  if( nTest>1 && !quietFlag ){
    printf("%d tests with no errors\nSQLite %s %s\n",
           nTest, sqlite3_libversion(), sqlite3_sourceid());
  }
  if( zDataOut ){
    FILE *out = fopen(zDataOut, "wb");
    int n = 0;

    if( out==0 ) abendError("cannot open %s for writing", zDataOut);

    sqlite3_finalize(pStmt);
    rc = sqlite3_prepare_v2(dataDb, "SELECT sql FROM testcase", -1, &pStmt, 0);
    if( rc ) abendError("%s", sqlite3_errmsg(dataDb));
    while( sqlite3_step(pStmt)==SQLITE_ROW ){
      fprintf(out,"/****<%d>****/", ++n);
      fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out);
    }








>







 







>
>
>
>
>
|







 







|

|
>







 







>
>
>
>
>
|
>
>
>

|



<

>

>








369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
...
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
...
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
...
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644

645
646
647
648
649
650
651
652
653
654
655

  int quietFlag = 0;            /* --quiet or -q flag */
  int nTest = 0;                /* Number of test cases run */
  int multiTest = 0;            /* True if there will be multiple test cases */
  int lastPct = -1;             /* Previous percentage done output */
  sqlite3 *dataDb = 0;          /* Database holding compacted input data */
  sqlite3_stmt *pStmt = 0;      /* Statement to insert testcase into dataDb */
  const char *zDataOut = 0;     /* Write compacted data to this output file */
  int nHeader = 0;              /* Bytes of header comment text on input file */


  g.zArgv0 = argv[0];
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
................................................................................
  }
  if( zInitDb ){
    rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0);
    if( rc!=SQLITE_OK ){
      abendError("unable to open initialization database \"%s\"", zInitDb);
    }
  }
  for(i=0; i<nIn; i=iNext+1){   /* Skip initial lines beginning with '#' */
    if( zIn[i]!='#' ) break;
    for(iNext=i+1; iNext<nIn && zIn[iNext]!='\n'; iNext++){}
  }
  nHeader = i;
  for(nTest=0; i<nIn; i=iNext, nTest++){
    char cSaved;
    if( strncmp(&zIn[i], "/****<",6)==0 ){
      char *z = strstr(&zIn[i], ">****/");
      if( z ){
        z += 6;
        if( verboseFlag ) printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]);
        i += (int)(z-&zIn[i]);
................................................................................
    if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
    if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");
    zSql = &zIn[i];
    if( verboseFlag ){
      printf("INPUT (offset: %d, size: %d): [%s]\n",
              i, (int)strlen(&zIn[i]), &zIn[i]);
    }else if( multiTest && !quietFlag ){
      int pct = 10*iNext/nIn;
      if( pct!=lastPct ){
        if( lastPct<0 ) printf("fuzz test:");
        printf(" %d%%", pct*10);
        fflush(stdout);
        lastPct = pct;
      }
    }
    switch( iMode ){
      case FZMODE_Glob:
        zSql = zToFree = sqlite3_mprintf("SELECT glob(%s);", zSql);
................................................................................
    rc = sqlite3_close(db);
    if( rc ){
      abendError("sqlite3_close() failed with rc=%d", rc);
    }
    if( sqlite3_memory_used()>0 ){
      abendError("memory in use after close: %lld bytes", sqlite3_memory_used());
    }
    if( nTest==1 ){
      /* Simulate an error if the TEST_FAILURE environment variable is "5" */
      char *zFailCode = getenv("TEST_FAILURE");
      if( zFailCode && zFailCode[0]=='5' && zFailCode[1]==0 ){
        abendError("simulated failure");
      }
    }
  }
  if( !verboseFlag && multiTest && !quietFlag ) printf("\n");
  if( nTest>1 && !quietFlag ){
    printf("%d fuzz tests with no errors\nSQLite %s %s\n",
           nTest, sqlite3_libversion(), sqlite3_sourceid());
  }
  if( zDataOut ){

    int n = 0;
    FILE *out = fopen(zDataOut, "wb");
    if( out==0 ) abendError("cannot open %s for writing", zDataOut);
    if( nHeader>0 ) fwrite(zIn, nHeader, 1, out);
    sqlite3_finalize(pStmt);
    rc = sqlite3_prepare_v2(dataDb, "SELECT sql FROM testcase", -1, &pStmt, 0);
    if( rc ) abendError("%s", sqlite3_errmsg(dataDb));
    while( sqlite3_step(pStmt)==SQLITE_ROW ){
      fprintf(out,"/****<%d>****/", ++n);
      fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out);
    }

This page was generated in about 0.047s by Fossil 2.13 [7de85417de] 2020-09-21 14:21:35