Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | test-using-fuzzcheck |
| Files: | files | file ages | folders |
| SHA1: |
4a5f6f1f0128657fd8d4d99d0682edd5 |
| User & Date: | drh 2015-05-26 18:15:08 |
Context
|
2015-05-26
| ||
| 18:58 | Fix a one-byte buffer overread that may follow a syntax error while preparing an SQL statement. check-in: 07500393 user: dan tags: test-using-fuzzcheck | |
| 18:58 | Fix fuzzcheck so that it responds correctly to the TEST_FAILURE environment variable. check-in: 76770c9e user: drh tags: test-using-fuzzcheck | |
| 18:15 | The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. check-in: 4a5f6f1f user: drh tags: test-using-fuzzcheck | |
| 17:57 | Enhance fuzzcheck with the ability to store descriptions in each source database and to run multiple source databases in a single invocation. check-in: 193364c8 user: drh tags: trunk | |
Changes
Changes to Makefile.in.
518
519
520
521
522
523
524
525
526
527
528
529
530
531
...
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
|
# TESTPROGS = \ testfixture$(TEXE) \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ sqldiff$(TEXE) # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) Makefile: $(TOP)/Makefile.in ................................................................................ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) # A very detailed test running most or all test cases fulltest: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/all.test # Really really long testing soaktest: $(TESTPROGS) fuzzoomtest ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 # Do extra testing but not everything. fulltestonly: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing fuzztest: fuzzershell$(TEXE) ./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt fuzzoomtest: fuzzershell$(TEXE) ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom # This is the common case. Run many tests but not those that take # a really long time. # test: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) fuzzershell$(TEXE) valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # smoketest: $(TESTPROGS) fuzzershell$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl echo "#define TCLSH 2" > $@ echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ echo "static const char *tclsh_main_loop(void){" >> $@ |
>
>
>
>
>
>
|
|
|
<
<
<
|
|
|
|
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
...
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
|
# TESTPROGS = \ testfixture$(TEXE) \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ sqldiff$(TEXE) # Databases containing fuzzer test cases # FUZZDATA = \ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) Makefile: $(TOP)/Makefile.in ................................................................................ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) # A very detailed test running most or all test cases fulltest: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/all.test # Really really long testing soaktest: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 # Do extra testing but not everything. fulltestonly: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing fuzztest: fuzzcheck$(TEXE) ./fuzzcheck$(TEXE) $(FUZZDATA) # This is the common case. Run many tests but not those that take # a really long time. # test: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) fuzzcheck$(TEXE) valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA) OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # smoketest: $(TESTPROGS) fuzzcheck$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl echo "#define TCLSH 2" > $@ echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ echo "static const char *tclsh_main_loop(void){" >> $@ |
Changes to Makefile.msc.
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
....
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
|
# TESTPROGS = \ testfixture.exe \ sqlite3.exe \ sqlite3_analyzer.exe \ sqldiff.exe # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) ................................................................................ extensiontest: testfixture.exe testloadext.dll .\testfixture.exe $(TOP)\test\loadext.test fulltest: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\all.test soaktest: $(TESTPROGS) fuzzoomtest .\testfixture.exe $(TOP)\test\all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner fuzztest: fuzzershell.exe .\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt fuzzoomtest: fuzzershell.exe .\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom test: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\veryquick.test smoketest: $(TESTPROGS) fuzzershell.exe .\testfixture.exe $(TOP)\test\main.test sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ echo static const char *tclsh_main_loop(void){ >> $@ |
>
>
>
>
>
>
>
|
|
|
<
<
<
|
|
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
....
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
|
# TESTPROGS = \ testfixture.exe \ sqlite3.exe \ sqlite3_analyzer.exe \ sqldiff.exe # Databases containing fuzzer test cases # FUZZDATA = \ $(TOP)\test\fuzzdata1.db \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) ................................................................................ extensiontest: testfixture.exe testloadext.dll .\testfixture.exe $(TOP)\test\loadext.test fulltest: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\all.test soaktest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) test: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\veryquick.test smoketest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\main.test sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ echo static const char *tclsh_main_loop(void){ >> $@ |
Changes to main.mk.
398
399
400
401
402
403
404
405
406
407
408
409
410
411
...
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
|
# executables needed for testing # TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ sqldiff$(EXE) # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.a sqlite3$(EXE) libsqlite3.a: $(LIBOBJ) ................................................................................ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fulltest: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/all.test soaktest: $(TESTPROGS) fuzzoomtest ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner fuzztest: fuzzershell$(EXE) ./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt fuzzoomtest: fuzzershell$(EXE) ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom test: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) fuzzershell$(EXE) valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # smoketest: $(TESTPROGS) fuzzershell$(EXE) ./testfixture$(EXE) $(TOP)/test/main.test # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ |
>
>
>
>
>
>
>
|
|
|
<
<
<
|
|
|
|
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
...
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
|
# executables needed for testing # TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ sqldiff$(EXE) # Databases containing fuzzer test cases # FUZZDATA = \ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.a sqlite3$(EXE) libsqlite3.a: $(LIBOBJ) ................................................................................ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fulltest: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/all.test soaktest: $(TESTPROGS) ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner fuzztest: fuzzcheck$(EXE) $(FUZZDATA) ./fuzzcheck$(EXE) $(FUZZDATA) test: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA) valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA) OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # smoketest: $(TESTPROGS) fuzzcheck$(EXE) ./testfixture$(EXE) $(TOP)/test/main.test # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ |
Added test/fuzzdata1.db.
cannot compute difference between binary files
Deleted test/fuzzdata1.txt.
cannot compute difference between binary files
Added test/fuzzdata2.db.
cannot compute difference between binary files
Deleted test/fuzzdata2.txt.
cannot compute difference between binary files
Added test/fuzzdata3.db.
cannot compute difference between binary files
Deleted test/mkfuzzdata1.tcl.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
#!/usr/bin/tclsh # # Run this script in order to rebuild the fuzzdata1.txt file containing # fuzzer data for the fuzzershell utility that is create by afl-fuzz. # # This script gathers all of the test cases identified by afl-fuzz and # runs afl-cmin and afl-tmin over them all to try to generate a mimimum # set of tests that cover all observed behavior. # # Options: # # --afl-bin DIR1 DIR1 contains the AFL binaries # --fuzzershell PATH Full pathname of instrumented fuzzershell # --afl-data DIR3 DIR3 is the "-o" directory from afl-fuzz # -o FILE Write results into FILE # set AFLBIN {} set FUZZERSHELL {} set AFLDATA {} set OUTFILE {} proc usage {} { puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\ --afl-data DIR -o FILE" exit 1 } proc cmdlineerr {msg} { puts stderr $msg usage } for {set i 0} {$i<[llength $argv]} {incr i} { set x [lindex $argv $i] if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"} set x [string trimleft $x -] incr i if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"} set a [lindex $argv $i] switch -- $x { afl-bin {set AFLBIN $a} afl-data {set AFLDATA $a} fuzzershell {set FUZZERSHELL $a} o {set OUTFILE $a} default {cmdlineerr "unknown option: --$x"} } } proc checkarg {varname option} { set val [set ::$varname] if {$val==""} {cmdlineerr "required option missing: --$option"} } checkarg AFLBIN afl-bin checkarg AFLDATA afl-data checkarg FUZZERSHELL fuzzershell checkarg OUTFILE o proc checkexec {x} { if {![file exec $x]} {cmdlineerr "cannot find $x"} } checkexec $AFLBIN/afl-cmin checkexec $AFLBIN/afl-tmin checkexec $FUZZERSHELL proc checkdir {x} { if {![file isdir $x]} {cmdlineerr "no such directory: $x"} } checkdir $AFLDATA/queue proc progress {msg} { puts "******** $msg" flush stdout } progress "mkdir tmp1 tmp2" file mkdir tmp1 tmp2 progress "copying test cases from $AFLDATA into tmp1..." set n 0 foreach file [glob -nocomplain $AFLDATA/queue/id:*] { incr n file copy $file tmp1/$n } foreach file [glob -nocomplain $AFLDATA/crash*/id:*] { incr n file copy $file tmp1/$n } progress "total $n files copied." progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL" exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout progress "afl-cmin complete." # # Experiments show that running afl-tmin is too slow for this application. # And it doesn't really make the test cases that much smaller. So let's # just skip it. # # foreach file [glob tmp2/*] { # progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL" # exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \ # $FUZZERSHELL >&@ stdout # } progress "generating final output into $OUTFILE" set out [open $OUTFILE wb] puts $out "# Test data for use with fuzzershell. Automatically # generated using $argv0. This file contains binary data #" set n 0 foreach file [glob tmp2/*] { incr n puts -nonewline $out "/****<$n>****/" set in [open $file rb] puts -nonewline $out [read $in] close $in } close $out progress "done. $n test cases written to $OUTFILE" progress "clean-up..." file delete -force tmp1 progress "culled test cases left in the tmp2 directory" |
< < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < |