Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Enhance fuzzershell to accept multiple input files. Add the test/fuzzdata2.txt fuzz test content. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
ab5523aafe4817232388d28ea99be095 |
User & Date: | drh 2015-05-01 19:21:12.995 |
Context
2015-05-01
| ||
20:34 | Enhance the fuzzershell --uniquecases option to output results in order of increasing runtime and to include the runtime in the comment separator of the output. (check-in: 04630b989d user: drh tags: trunk) | |
19:21 | Enhance fuzzershell to accept multiple input files. Add the test/fuzzdata2.txt fuzz test content. (check-in: ab5523aafe user: drh tags: trunk) | |
18:00 | Fix an assert in RTREE that would fire if the rtree table is misdeclared. (check-in: 9a45409cc4 user: drh tags: trunk) | |
Changes
Changes to Makefile.in.
︙ | ︙ | |||
960 961 962 963 964 965 966 | # Do extra testing but not aeverything. fulltestonly: testfixture$(TEXE) sqlite3$(TEXE) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing fuzztest: fuzzershell$(TEXE) | | | 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 | # Do extra testing but not aeverything. fulltestonly: testfixture$(TEXE) sqlite3$(TEXE) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing fuzztest: fuzzershell$(TEXE) ./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt fuzzoomtest: fuzzershell$(TEXE) ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom # This is the common case. Run many tests but not those that take # a really long time. # |
︙ | ︙ |
Changes to Makefile.msc.
︙ | ︙ | |||
1640 1641 1642 1643 1644 1645 1646 | fulltestonly: testfixture.exe sqlite3.exe fuzztest .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner fuzztest: fuzzershell.exe | | | 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 | fulltestonly: testfixture.exe sqlite3.exe fuzztest .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner fuzztest: fuzzershell.exe .\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt fuzzoomtest: fuzzershell.exe .\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom test: testfixture.exe sqlite3.exe fuzztest .\testfixture.exe $(TOP)\test\veryquick.test |
︙ | ︙ |
Changes to main.mk.
︙ | ︙ | |||
642 643 644 645 646 647 648 | fulltestonly: testfixture$(EXE) sqlite3$(EXE) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner fuzztest: fuzzershell$(EXE) | | | 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 | fulltestonly: testfixture$(EXE) sqlite3$(EXE) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner fuzztest: fuzzershell$(EXE) ./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt fuzzoomtest: fuzzershell$(EXE) ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom test: testfixture$(EXE) sqlite3$(EXE) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test |
︙ | ︙ |
Added test/fuzzdata2.txt.
cannot compute difference between binary files
Changes to tool/fuzzershell.c.
︙ | ︙ | |||
311 312 313 314 315 316 317 | /* End of the eval() implementation ******************************************************************************/ /* ** Print sketchy documentation for this utility program */ static void showHelp(void){ | | | > < < | 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 | /* End of the eval() implementation ******************************************************************************/ /* ** Print sketchy documentation for this utility program */ static void showHelp(void){ printf("Usage: %s [options] ?FILE...?\n", g.zArgv0); printf( "Read SQL text from FILE... (or from standard input if FILE... is omitted)\n" "and then evaluate each block of SQL contained therein.\n" "Options:\n" " --autovacuum Enable AUTOVACUUM mode\n" " --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" " --help Show this help text\n" " --lookaside N SZ Configure lookaside for N slots of SZ bytes each\n" " --oom Run each test multiple times in a simulated OOM loop\n" " --pagesize N Set the page size to N\n" " --pcache N SZ Configure N pages of pagecache each of size SZ bytes\n" " -q Reduced output\n" " --quiet Reduced output\n" " --scratch N SZ Configure scratch memory for N slots of SZ bytes each\n" |
︙ | ︙ | |||
393 394 395 396 397 398 399 | break; } } if( v>0x7fffffff ) abendError("parameter too large - max 2147483648"); return (int)(isNeg? -v : v); } | < < < < < < < < | | | | < | | | | < < | < > > > > > < | < > < < < < < < < < < < < < < < < < < < < < < < < | 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 | break; } } if( v>0x7fffffff ) abendError("parameter too large - max 2147483648"); return (int)(isNeg? -v : v); } int main(int argc, char **argv){ char *zIn = 0; /* Input text */ int nAlloc = 0; /* Number of bytes allocated for zIn[] */ int nIn = 0; /* Number of bytes of zIn[] used */ size_t got; /* Bytes read from input */ int rc = SQLITE_OK; /* Result codes from API functions */ int i; /* Loop counter */ int iNext; /* Next block of SQL */ sqlite3 *db; /* Open database */ char *zErrMsg = 0; /* Error message returned from sqlite3_exec() */ const char *zEncoding = 0; /* --utf16be or --utf16le */ int nHeap = 0, mnHeap = 0; /* Heap size from --heap */ int nLook = 0, szLook = 0; /* --lookaside configuration */ int nPCache = 0, szPCache = 0;/* --pcache configuration */ int nScratch = 0, szScratch=0;/* --scratch configuration */ int pageSize = 0; /* Desired page size. 0 means default */ void *pHeap = 0; /* Allocated heap space */ void *pLook = 0; /* Allocated lookaside space */ void *pPCache = 0; /* Allocated storage for pcache */ void *pScratch = 0; /* Allocated storage for scratch */ int doAutovac = 0; /* True for --autovacuum */ char *zSql; /* SQL to run */ char *zToFree = 0; /* Call sqlite3_free() on this afte running zSql */ const char *zCkGlob = 0; /* Inputs must match this glob */ int verboseFlag = 0; /* --verbose or -v flag */ int quietFlag = 0; /* --quiet or -q flag */ int nTest = 0; /* Number of test cases run */ int multiTest = 0; /* True if there will be multiple test cases */ int lastPct = -1; /* Previous percentage done output */ sqlite3 *dataDb = 0; /* Database holding compacted input data */ sqlite3_stmt *pStmt = 0; /* Statement to insert testcase into dataDb */ const char *zDataOut = 0; /* Write compacted data to this output file */ int nHeader = 0; /* Bytes of header comment text on input file */ int oomFlag = 0; /* --oom */ int oomCnt = 0; /* Counter for the OOM loop */ char zErrBuf[200]; /* Space for the error message */ const char *zFailCode; /* Value of the TEST_FAILURE environment var */ const char *zPrompt; /* Initial prompt when large-file fuzzing */ int nInFile = 0; /* Number of input files to read */ char **azInFile = 0; /* Array of input file names */ int jj; /* Loop counter for azInFile[] */ zFailCode = getenv("TEST_FAILURE"); g.zArgv0 = argv[0]; zPrompt = "<stdin>"; for(i=1; i<argc; i++){ const char *z = argv[i]; if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; if( strcmp(z,"autovacuum")==0 ){ doAutovac = 1; }else if( strcmp(z, "f")==0 && i+1<argc ){ i++; goto addNewInFile; }else if( strcmp(z,"heap")==0 ){ if( i>=argc-2 ) abendError("missing arguments on %s\n", argv[i]); nHeap = integerValue(argv[i+1]); mnHeap = integerValue(argv[i+2]); i += 2; }else if( strcmp(z,"help")==0 ){ showHelp(); return 0; }else if( strcmp(z,"lookaside")==0 ){ if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]); nLook = integerValue(argv[i+1]); szLook = integerValue(argv[i+2]); i += 2; }else if( strcmp(z,"oom")==0 ){ oomFlag = 1; }else if( strcmp(z,"pagesize")==0 ){ if( i>=argc-1 ) abendError("missing argument on %s", argv[i]); pageSize = integerValue(argv[++i]); |
︙ | ︙ | |||
541 542 543 544 545 546 547 | quietFlag = 0; verboseFlag = 1; }else { abendError("unknown option: %s", argv[i]); } }else{ | > > > | > > > | 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 | quietFlag = 0; verboseFlag = 1; }else { abendError("unknown option: %s", argv[i]); } }else{ addNewInFile: nInFile++; azInFile = realloc(azInFile, sizeof(azInFile[0])*nInFile); if( azInFile==0 ) abendError("out of memory"); azInFile[nInFile-1] = argv[i]; } } /* Do global SQLite initialization */ sqlite3_config(SQLITE_CONFIG_LOG, verboseFlag ? shellLog : shellLogNoop, 0); if( nHeap>0 ){ pHeap = malloc( nHeap ); if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap); rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap); if( rc ) abendError("heap configuration failed: %d\n", rc); } |
︙ | ︙ | |||
579 580 581 582 583 584 585 | if( nPCache>0 && szPCache>0 ){ pPCache = malloc( nPCache*(sqlite3_int64)szPCache ); if( pPCache==0 ) fatalError("cannot allocate %lld-byte pcache", nPCache*(sqlite3_int64)szPCache); rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache); if( rc ) abendError("pcache configuration failed: %d", rc); } | < < < < < < < < | < > > > | > | | > > > > > | > > | > > > > > > | | > > > > > > | > > > > > > > > > > > > > | | | | | > > > | | | | | | | | | | | | | | | | | > > > > | | | | | | | | | | | | | > > > | | | | | | | | | | | | | | | | | < < < < < < < < < < | > > | | | | | | | | | | | | | | | | | | | | | | < < < < < < < < < < | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > > > | | | | | > > > | | | | | | | > | > > > | | | | | | | | | | | | > > > > > > > > > > > | 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 | if( nPCache>0 && szPCache>0 ){ pPCache = malloc( nPCache*(sqlite3_int64)szPCache ); if( pPCache==0 ) fatalError("cannot allocate %lld-byte pcache", nPCache*(sqlite3_int64)szPCache); rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache); if( rc ) abendError("pcache configuration failed: %d", rc); } /* If the --unique-cases option was supplied, open the database that will ** be used to gather unique test cases. */ if( zDataOut ){ rc = sqlite3_open(":memory:", &dataDb); if( rc ) abendError("cannot open :memory: database"); rc = sqlite3_exec(dataDb, "CREATE TABLE testcase(sql BLOB PRIMARY KEY) WITHOUT ROWID;",0,0,0); if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); rc = sqlite3_prepare_v2(dataDb, "INSERT OR IGNORE INTO testcase(sql)VALUES(?1)", -1, &pStmt, 0); if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); } /* Initialize the input buffer used to hold SQL text */ if( nInFile==0 ) nInFile = 1; nAlloc = 1000; zIn = malloc(nAlloc); if( zIn==0 ) fatalError("out of memory"); /* Loop over all input files */ for(jj=0; jj<nInFile; jj++){ /* Read the complete content of the next input file into zIn[] */ FILE *in; if( azInFile ){ int j, k; in = fopen(azInFile[jj],"rb"); if( in==0 ){ abendError("cannot open %s for reading", azInFile[jj]); } zPrompt = azInFile[jj]; for(j=k=0; zPrompt[j]; j++) if( zPrompt[j]=='/' ) k = j+1; zPrompt += k; }else{ in = stdin; zPrompt = "<stdin>"; } while( !feof(in) ){ zIn = realloc(zIn, nAlloc); if( zIn==0 ) fatalError("out of memory"); got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); nIn += (int)got; zIn[nIn] = 0; if( got==0 ) break; nAlloc += nAlloc+1000; } if( in!=stdin ) fclose(in); lastPct = -1; /* Skip initial lines of the input file that begin with "#" */ for(i=0; i<nIn; i=iNext+1){ if( zIn[i]!='#' ) break; for(iNext=i+1; iNext<nIn && zIn[iNext]!='\n'; iNext++){} } nHeader = i; /* Process all test cases contained within the input file. */ for(; i<nIn; i=iNext, nTest++, g.zTestName[0]=0){ char cSaved; if( strncmp(&zIn[i], "/****<",6)==0 ){ char *z = strstr(&zIn[i], ">****/"); if( z ){ z += 6; sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "%.*s", (int)(z-&zIn[i]) - 12, &zIn[i+6]); if( verboseFlag ){ printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]); fflush(stdout); } i += (int)(z-&zIn[i]); multiTest = 1; } } for(iNext=i; iNext<nIn && strncmp(&zIn[iNext],"/****<",6)!=0; iNext++){} /* Store unique test cases in the in the dataDb database if the ** --unique-cases flag is present */ if( zDataOut ){ sqlite3_bind_blob(pStmt, 1, &zIn[i], iNext-i, SQLITE_STATIC); rc = sqlite3_step(pStmt); if( rc!=SQLITE_DONE ) abendError("%s", sqlite3_errmsg(dataDb)); sqlite3_reset(pStmt); continue; } cSaved = zIn[iNext]; zIn[iNext] = 0; if( zCkGlob && sqlite3_strglob(zCkGlob,&zIn[i])!=0 ){ zIn[iNext] = cSaved; continue; } /* Print out the SQL of the next test case is --verbose is enabled */ zSql = &zIn[i]; if( verboseFlag ){ printf("INPUT (offset: %d, size: %d): [%s]\n", i, (int)strlen(&zIn[i]), &zIn[i]); }else if( multiTest && !quietFlag ){ if( oomFlag ){ printf("%s\n", g.zTestName); }else{ int pct = (10*iNext)/nIn; if( pct!=lastPct ){ if( lastPct<0 ) printf("%s:", zPrompt); printf(" %d%%", pct*10); lastPct = pct; } } } fflush(stdout); /* Run the next test case. Run it multiple times in --oom mode */ if( oomFlag ){ oomCnt = g.iOomCntdown = 1; g.nOomFault = 0; g.bOomOnce = 1; if( verboseFlag ){ printf("Once.%d\n", oomCnt); fflush(stdout); } }else{ oomCnt = 0; } do{ rc = sqlite3_open_v2( "main.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY, 0); if( rc!=SQLITE_OK ){ abendError("Unable to open the in-memory database"); } if( pLook ){ rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE,pLook,szLook,nLook); if( rc!=SQLITE_OK ) abendError("lookaside configuration filed: %d", rc); } #ifndef SQLITE_OMIT_TRACE sqlite3_trace(db, verboseFlag ? traceCallback : traceNoop, 0); #endif sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0); sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0); sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000); if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding); if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize); if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL"); g.bOomEnable = 1; if( verboseFlag ){ zErrMsg = 0; rc = sqlite3_exec(db, zSql, execCallback, 0, &zErrMsg); if( zErrMsg ){ sqlite3_snprintf(sizeof(zErrBuf),zErrBuf,"%z", zErrMsg); zErrMsg = 0; } }else { rc = sqlite3_exec(db, zSql, execNoop, 0, 0); } g.bOomEnable = 0; rc = sqlite3_close(db); if( rc ){ abendError("sqlite3_close() failed with rc=%d", rc); } if( sqlite3_memory_used()>0 ){ abendError("memory in use after close: %lld bytes",sqlite3_memory_used()); } if( oomFlag ){ /* Limit the number of iterations of the OOM loop to OOM_MAX. If the ** first pass (single failure) exceeds 2/3rds of OOM_MAX this skip the ** second pass (continuous failure after first) completely. */ if( g.nOomFault==0 || oomCnt>OOM_MAX ){ if( g.bOomOnce && oomCnt<=(OOM_MAX*2/3) ){ oomCnt = g.iOomCntdown = 1; g.bOomOnce = 0; }else{ oomCnt = 0; } }else{ g.iOomCntdown = ++oomCnt; g.nOomFault = 0; } if( oomCnt ){ if( verboseFlag ){ printf("%s.%d\n", g.bOomOnce ? "Once" : "Multi", oomCnt); fflush(stdout); } nTest++; } } }while( oomCnt>0 ); /* Free the SQL from the current test case */ if( zToFree ){ sqlite3_free(zToFree); zToFree = 0; } zIn[iNext] = cSaved; /* Show test-case results in --verbose mode */ if( verboseFlag ){ printf("RESULT-CODE: %d\n", rc); if( zErrMsg ){ printf("ERROR-MSG: [%s]\n", zErrBuf); } fflush(stdout); } /* Simulate an error if the TEST_FAILURE environment variable is "5". ** This is used to verify that automated test script really do spot ** errors that occur in this test program. */ if( zFailCode ){ if( zFailCode[0]=='5' && zFailCode[1]==0 ){ abendError("simulated failure"); }else if( zFailCode[0]!=0 ){ /* If TEST_FAILURE is something other than 5, just exit the test ** early */ printf("\nExit early due to TEST_FAILURE being set"); break; } } } if( !verboseFlag && multiTest && !quietFlag && !oomFlag ) printf("\n"); } /* Report total number of tests run */ if( nTest>1 && !quietFlag ){ printf("%s: 0 errors out of %d tests\nSQLite %s %s\n", g.zArgv0, nTest, sqlite3_libversion(), sqlite3_sourceid()); } /* Write the unique test cases if the --unique-cases flag was used */ if( zDataOut ){ int n = 0; FILE *out = fopen(zDataOut, "wb"); if( out==0 ) abendError("cannot open %s for writing", zDataOut); if( nHeader>0 ) fwrite(zIn, nHeader, 1, out); sqlite3_finalize(pStmt); rc = sqlite3_prepare_v2(dataDb, "SELECT sql FROM testcase", -1, &pStmt, 0); if( rc ) abendError("%s", sqlite3_errmsg(dataDb)); while( sqlite3_step(pStmt)==SQLITE_ROW ){ fprintf(out,"/****<%d>****/", ++n); fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out); } fclose(out); sqlite3_finalize(pStmt); sqlite3_close(dataDb); } /* Clean up and exit. */ free(azInFile); free(zIn); free(pHeap); free(pLook); free(pScratch); free(pPCache); return 0; } |