Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add AFL-generated test cases in the test/fuzzdata1.txt file. Automatically run fuzzershell against those cases on a "make test". |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
627ea83c26b420088f101801eb876512 |
User & Date: | drh 2015-04-24 16:09:12.862 |
Context
2015-04-24
| ||
16:16 | Add the fuzztest target to the MSVC makefile. (check-in: 98edc6152c user: drh tags: trunk) | |
16:09 | Add AFL-generated test cases in the test/fuzzdata1.txt file. Automatically run fuzzershell against those cases on a "make test". (check-in: 627ea83c26 user: drh tags: trunk) | |
14:47 | Add the --unique-cases option to fuzzershell. (check-in: 7cb718491b user: drh tags: trunk) | |
Changes
Changes to Makefile.in.
︙ | ︙ | |||
947 948 949 950 951 952 953 | TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(TEXE): $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) # A very detailed test running most or all test cases | | > > > > | | 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 | TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(TEXE): $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) # A very detailed test running most or all test cases fulltest: testfixture$(TEXE) sqlite3$(TEXE) fuzztest ./testfixture$(TEXE) $(TOP)/test/all.test # Really really long testing soaktest: testfixture$(TEXE) sqlite3$(TEXE) ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 # Do extra testing but not aeverything. fulltestonly: testfixture$(TEXE) sqlite3$(TEXE) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing fuzztest: fuzzershell$(TEXE) ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt # This is the common case. Run many tests but not those that take # a really long time. # test: testfixture$(TEXE) sqlite3$(TEXE) fuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: testfixture$(TEXE) sqlite3$(TEXE) OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind |
︙ | ︙ |
Changes to main.mk.
︙ | ︙ | |||
629 630 631 632 633 634 635 | fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) | | | > > > | | 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 | fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ -DSQLITE_ENABLE_FTS3=1 \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fulltest: testfixture$(EXE) sqlite3$(EXE) fuzztest ./testfixture$(EXE) $(TOP)/test/all.test soaktest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 fulltestonly: testfixture$(EXE) sqlite3$(EXE) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner fuzztest: fuzzershell$(EXE) ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt test: testfixture$(EXE) sqlite3$(EXE) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ |
︙ | ︙ |
Added test/fuzzdata1.txt.
cannot compute difference between binary files
Changes to test/releasetest.tcl.
︙ | ︙ | |||
195 196 197 198 199 200 201 202 203 204 205 206 207 208 | # different names for them all so that they results appear in separate # subdirectories. # Fail0 {-O0} Fail2 {-O0} Fail3 {-O0} Fail4 {-O0} }] array set ::Platforms [strip_comments { Linux-x86_64 { "Check-Symbols" checksymbols "Debug-One" "mptest test" "Have-Not" test | > | 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | # different names for them all so that they results appear in separate # subdirectories. # Fail0 {-O0} Fail2 {-O0} Fail3 {-O0} Fail4 {-O0} FuzzFail {-O0} }] array set ::Platforms [strip_comments { Linux-x86_64 { "Check-Symbols" checksymbols "Debug-One" "mptest test" "Have-Not" test |
︙ | ︙ | |||
251 252 253 254 255 256 257 258 259 260 261 262 263 264 | # Failure-Detection { Fail0 "TEST_FAILURE=0 test" Sanitize "TEST_FAILURE=1 test" Fail2 "TEST_FAILURE=2 valgrindtest" Fail3 "TEST_FAILURE=3 valgrindtest" Fail4 "TEST_FAILURE=4 test" } }] # End of configuration section. ######################################################################### ######################################################################### | > | 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 | # Failure-Detection { Fail0 "TEST_FAILURE=0 test" Sanitize "TEST_FAILURE=1 test" Fail2 "TEST_FAILURE=2 valgrindtest" Fail3 "TEST_FAILURE=3 valgrindtest" Fail4 "TEST_FAILURE=4 test" FuzzFail "TEST_FAILURE=5 test" } }] # End of configuration section. ######################################################################### ######################################################################### |
︙ | ︙ |
Changes to tool/fuzzershell.c.
︙ | ︙ | |||
369 370 371 372 373 374 375 376 377 378 379 380 381 382 | int quietFlag = 0; /* --quiet or -q flag */ int nTest = 0; /* Number of test cases run */ int multiTest = 0; /* True if there will be multiple test cases */ int lastPct = -1; /* Previous percentage done output */ sqlite3 *dataDb = 0; /* Database holding compacted input data */ sqlite3_stmt *pStmt = 0; /* Statement to insert testcase into dataDb */ const char *zDataOut = 0; /* Write compacted data to this output file */ g.zArgv0 = argv[0]; for(i=1; i<argc; i++){ const char *z = argv[i]; if( z[0]=='-' ){ z++; | > | 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 | int quietFlag = 0; /* --quiet or -q flag */ int nTest = 0; /* Number of test cases run */ int multiTest = 0; /* True if there will be multiple test cases */ int lastPct = -1; /* Previous percentage done output */ sqlite3 *dataDb = 0; /* Database holding compacted input data */ sqlite3_stmt *pStmt = 0; /* Statement to insert testcase into dataDb */ const char *zDataOut = 0; /* Write compacted data to this output file */ int nHeader = 0; /* Bytes of header comment text on input file */ g.zArgv0 = argv[0]; for(i=1; i<argc; i++){ const char *z = argv[i]; if( z[0]=='-' ){ z++; |
︙ | ︙ | |||
520 521 522 523 524 525 526 | } if( zInitDb ){ rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0); if( rc!=SQLITE_OK ){ abendError("unable to open initialization database \"%s\"", zInitDb); } } | > > > > > | | 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 | } if( zInitDb ){ rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0); if( rc!=SQLITE_OK ){ abendError("unable to open initialization database \"%s\"", zInitDb); } } for(i=0; i<nIn; i=iNext+1){ /* Skip initial lines beginning with '#' */ if( zIn[i]!='#' ) break; for(iNext=i+1; iNext<nIn && zIn[iNext]!='\n'; iNext++){} } nHeader = i; for(nTest=0; i<nIn; i=iNext, nTest++){ char cSaved; if( strncmp(&zIn[i], "/****<",6)==0 ){ char *z = strstr(&zIn[i], ">****/"); if( z ){ z += 6; if( verboseFlag ) printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]); i += (int)(z-&zIn[i]); |
︙ | ︙ | |||
578 579 580 581 582 583 584 | if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize); if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL"); zSql = &zIn[i]; if( verboseFlag ){ printf("INPUT (offset: %d, size: %d): [%s]\n", i, (int)strlen(&zIn[i]), &zIn[i]); }else if( multiTest && !quietFlag ){ | | > | | 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 | if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize); if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL"); zSql = &zIn[i]; if( verboseFlag ){ printf("INPUT (offset: %d, size: %d): [%s]\n", i, (int)strlen(&zIn[i]), &zIn[i]); }else if( multiTest && !quietFlag ){ int pct = 10*iNext/nIn; if( pct!=lastPct ){ if( lastPct<0 ) printf("fuzz test:"); printf(" %d%%", pct*10); fflush(stdout); lastPct = pct; } } switch( iMode ){ case FZMODE_Glob: zSql = zToFree = sqlite3_mprintf("SELECT glob(%s);", zSql); |
︙ | ︙ | |||
617 618 619 620 621 622 623 | rc = sqlite3_close(db); if( rc ){ abendError("sqlite3_close() failed with rc=%d", rc); } if( sqlite3_memory_used()>0 ){ abendError("memory in use after close: %lld bytes", sqlite3_memory_used()); } | > > > > > | > > > | < > > | 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 | rc = sqlite3_close(db); if( rc ){ abendError("sqlite3_close() failed with rc=%d", rc); } if( sqlite3_memory_used()>0 ){ abendError("memory in use after close: %lld bytes", sqlite3_memory_used()); } if( nTest==1 ){ /* Simulate an error if the TEST_FAILURE environment variable is "5" */ char *zFailCode = getenv("TEST_FAILURE"); if( zFailCode && zFailCode[0]=='5' && zFailCode[1]==0 ){ abendError("simulated failure"); } } } if( !verboseFlag && multiTest && !quietFlag ) printf("\n"); if( nTest>1 && !quietFlag ){ printf("%d fuzz tests with no errors\nSQLite %s %s\n", nTest, sqlite3_libversion(), sqlite3_sourceid()); } if( zDataOut ){ int n = 0; FILE *out = fopen(zDataOut, "wb"); if( out==0 ) abendError("cannot open %s for writing", zDataOut); if( nHeader>0 ) fwrite(zIn, nHeader, 1, out); sqlite3_finalize(pStmt); rc = sqlite3_prepare_v2(dataDb, "SELECT sql FROM testcase", -1, &pStmt, 0); if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); while( sqlite3_step(pStmt)==SQLITE_ROW ){ fprintf(out,"/****<%d>****/", ++n); fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out); } |
︙ | ︙ |