/ Check-in [e3de8291]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Update the releasetest.tcl script so that it can run multiple tests in parallel in separate processes.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e3de82919d6f7ef5c2c5ab9a932a480604856950
User & Date: drh 2015-11-03 14:49:35
Context
2015-11-03
15:16
In the releasetest.tcl script, show the --jobs object on the command-line summary. Report the time of each "starting:" event. Do not show errors on a --dryrun. check-in: b7510777 user: drh tags: trunk
14:49
Update the releasetest.tcl script so that it can run multiple tests in parallel in separate processes. check-in: e3de8291 user: drh tags: trunk
06:23
Further enhancement and a bit of cleanup. Closed-Leaf check-in: 0d8b5978 user: mistachkin tags: mp-releasetest
2015-11-02
15:08
On unix, if a file is opened via a symlink, create, read and write journal and wal files based on the name of the actual db file, not the symlink. check-in: 6d5ce3ed user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to test/bc_common.tcl.

     3      3   
     4      4   proc bc_find_binaries {zCaption} {
     5      5     # Search for binaries to test against. Any executable files that match
     6      6     # our naming convention are assumed to be testfixture binaries to test
     7      7     # against.
     8      8     #
     9      9     set binaries [list]
    10         -  set pattern "[file tail [info nameofexec]]?*"
           10  +  set self [file tail [info nameofexec]]
           11  +  set pattern "$self?*"
    11     12     if {$::tcl_platform(platform)=="windows"} {
    12     13       set pattern [string map {\.exe {}} $pattern]
    13     14     }
    14     15     foreach file [glob -nocomplain $pattern] {
           16  +    if {$file==$self} continue
    15     17       if {[file executable $file] && [file isfile $file]} {lappend binaries $file}
    16     18     }
    17     19   
    18     20     if {[llength $binaries]==0} {
    19     21       puts "WARNING: No historical binaries to test against."
    20     22       puts "WARNING: Omitting backwards-compatibility tests"
    21     23     }

Changes to test/releasetest.tcl.

    15     15       --quick                            (Run "veryquick.test" only)
    16     16       --veryquick                        (Run "make smoketest" only)
    17     17       --msvc                             (Use MSVC as the compiler)
    18     18       --buildonly                        (Just build testfixture - do not run)
    19     19       --dryrun                           (Print what would have happened)
    20     20       --info                             (Show diagnostic info)
    21     21       --with-tcl=DIR                     (Use TCL build at DIR)
           22  +    --jobs     N                       (Use N processes - default 1)
    22     23   
    23     24   The default value for --srcdir is the parent of the directory holding
    24     25   this script.
    25     26   
    26     27   The script determines the default value for --platform using the
    27     28   $tcl_platform(os) and $tcl_platform(machine) variables.  Supported
    28     29   platforms are "Linux-x86", "Linux-x86_64", "Darwin-i386",
................................................................................
   281    282   }]
   282    283   
   283    284   
   284    285   # End of configuration section.
   285    286   #########################################################################
   286    287   #########################################################################
   287    288   
          289  +# Configuration verification: Check that each entry in the list of configs
          290  +# specified for each platforms exists.
          291  +#
   288    292   foreach {key value} [array get ::Platforms] {
   289    293     foreach {v t} $value {
   290    294       if {0==[info exists ::Configs($v)]} {
   291    295         puts stderr "No such configuration: \"$v\""
   292    296         exit -1
   293    297       }
   294    298     }
................................................................................
   382    386       set errmsg "Test did not complete"
   383    387       if {[file readable core]} {
   384    388         append errmsg " - core file exists"
   385    389       }
   386    390     }
   387    391   }
   388    392   
   389         -proc run_test_suite {name testtarget config} {
          393  +#--------------------------------------------------------------------------
          394  +# This command is invoked as the [main] routine for scripts run with the
          395  +# "--slave" option.
          396  +#
          397  +# For each test (i.e. "configure && make test" execution), the master
          398  +# process spawns a process with the --slave option. It writes two lines
          399  +# to the slaves stdin. The first contains a single boolean value - the
          400  +# value of ::TRACE to use in the slave script. The second line contains a
          401  +# list in the same format as each element of the list passed to the
          402  +# [run_all_test_suites] command in the master process.
          403  +#
          404  +# The slave then runs the "configure && make test" commands specified. It
          405  +# exits successfully if the tests passes, or with a non-zero error code
          406  +# otherwise.
          407  +#
          408  +proc run_slave_test {} {
          409  +  # Read global vars configuration from stdin.
          410  +  set V [gets stdin]
          411  +  foreach {::TRACE ::MSVC ::DRYRUN} $V {}
          412  +
          413  +  # Read the test-suite configuration from stdin.
          414  +  set T [gets stdin]
          415  +  foreach {title dir configOpts testtarget makeOpts cflags opts} $T {}
          416  +
          417  +  # Create and switch to the test directory.
          418  +  trace_cmd file mkdir $dir
          419  +  trace_cmd cd $dir
          420  +  catch {file delete core}
          421  +  catch {file delete test.log}
          422  +
          423  +  # Run the "./configure && make" commands.
          424  +  set rc 0
          425  +  set rc [catch [configureCommand $configOpts]]
          426  +  if {!$rc} {
          427  +    if {[info exists ::env(TCLSH_CMD)]} {
          428  +      set savedEnv(TCLSH_CMD) $::env(TCLSH_CMD)
          429  +    } else {
          430  +      unset -nocomplain savedEnv(TCLSH_CMD)
          431  +    }
          432  +    set ::env(TCLSH_CMD) [file nativename [info nameofexecutable]]
          433  +    set rc [catch [makeCommand $testtarget $makeOpts $cflags $opts]]
          434  +    if {[info exists savedEnv(TCLSH_CMD)]} {
          435  +      set ::env(TCLSH_CMD) $savedEnv(TCLSH_CMD)
          436  +    } else {
          437  +      unset -nocomplain ::env(TCLSH_CMD)
          438  +    }
          439  +  }
          440  +
          441  +  # Exis successfully if the test passed, or with a non-zero error code
          442  +  # otherwise.
          443  +  exit $rc
          444  +}
          445  +
          446  +# This command is invoked in the master process each time a slave
          447  +# file-descriptor is readable.
          448  +#
          449  +proc slave_fileevent {fd T tm1} {
          450  +  global G
          451  +  foreach {title dir configOpts testtarget makeOpts cflags opts} $T {}
          452  +
          453  +  if {[eof $fd]} {
          454  +    fconfigure $fd -blocking 1
          455  +    set rc [catch { close $fd }]
          456  +
          457  +    set errmsg {}
          458  +    set logfile [file join $dir test.log]
          459  +    if {[file exists $logfile]} {
          460  +      count_tests_and_errors [file join $dir test.log] rc errmsg
          461  +    } elseif {$rc==0} {
          462  +      set rc 1
          463  +      set errmsg "no test.log file..."
          464  +    }
          465  +
          466  +    if {!$::TRACE} {
          467  +      set tm2 [clock seconds]
          468  +      set hours [expr {($tm2-$tm1)/3600}]
          469  +      set minutes [expr {(($tm2-$tm1)/60)%60}]
          470  +      set seconds [expr {($tm2-$tm1)%60}]
          471  +      set tm [format (%02d:%02d:%02d) $hours $minutes $seconds]
          472  +
          473  +      if {$rc} {
          474  +        set status FAIL
          475  +        incr ::NERR
          476  +      } else {
          477  +        set status Ok
          478  +      }
          479  +
          480  +      set n [string length $title]
          481  +      PUTS "finished: ${title}[string repeat . [expr {53-$n}]] $status $tm"
          482  +      if {$errmsg!=""} {PUTS "     $errmsg"}
          483  +      flush stdout
          484  +    }
          485  +
          486  +    incr G(nJob) -1
          487  +  } else {
          488  +    set line [gets $fd]
          489  +    if {[string trim $line] != ""} {
          490  +      puts "Trace   : $title - \"$line\""
          491  +    }
          492  +  }
          493  +}
          494  +
          495  +#--------------------------------------------------------------------------
          496  +# The only argument passed to this function is a list of test-suites to
          497  +# run. Each "test-suite" is itself a list consisting of the following
          498  +# elements:
          499  +#
          500  +#   * Test title (for display).
          501  +#   * The name of the directory to run the test in.
          502  +#   * The argument for [configureCommand]
          503  +#   * The first argument for [makeCommand]
          504  +#   * The second argument for [makeCommand]
          505  +#   * The third argument for [makeCommand]
          506  +#
          507  +proc run_all_test_suites {alltests} {
          508  +  global G
          509  +  set tests $alltests
          510  +
          511  +  set G(nJob) 0
          512  +
          513  +  while {[llength $tests]>0 || $G(nJob)>0} {
          514  +    if {$G(nJob)>=$::JOBS || [llength $tests]==0} {
          515  +      vwait G(nJob)
          516  +    }
          517  +
          518  +    if {[llength $tests]>0} {
          519  +      set T [lindex $tests 0]
          520  +      set tests [lrange $tests 1 end]
          521  +      foreach {title dir configOpts testtarget makeOpts cflags opts} $T {}
          522  +      if {!$::TRACE} {
          523  +        set n [string length $title]
          524  +        PUTS "starting: ${title}"
          525  +        flush stdout
          526  +      }
          527  +
          528  +      # Run the job.
          529  +      #
          530  +      set tm1 [clock seconds]
          531  +      incr G(nJob)
          532  +      set script [file normalize [info script]]
          533  +      set fd [open "|[info nameofexecutable] $script --slave" r+]
          534  +      fconfigure $fd -blocking 0
          535  +      fileevent $fd readable [list slave_fileevent $fd $T $tm1]
          536  +      puts $fd [list $::TRACE $::MSVC $::DRYRUN]
          537  +      puts $fd [list {*}$T]
          538  +      flush $fd
          539  +    }
          540  +  }
          541  +}
          542  +
          543  +proc add_test_suite {listvar name testtarget config} {
          544  +  upvar $listvar alltests
          545  +
   390    546     # Tcl variable $opts is used to build up the value used to set the
   391    547     # OPTS Makefile variable. Variable $cflags holds the value for
   392    548     # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but
   393    549     # CFLAGS is only passed to gcc.
   394    550     #
          551  +  set makeOpts ""
   395    552     set cflags [expr {$::MSVC ? "-Zi" : "-g"}]
   396    553     set opts ""
   397    554     set title ${name}($testtarget)
   398    555     set configOpts $::WITHTCL
   399    556   
   400    557     regsub -all {#[^\n]*\n} $config \n config
   401    558     foreach arg $config {
   402    559       if {[regexp {^-[UD]} $arg]} {
   403    560         lappend opts $arg
   404    561       } elseif {[regexp {^[A-Z]+=} $arg]} {
   405    562         lappend testtarget $arg
   406    563       } elseif {[regexp {^--(enable|disable)-} $arg]} {
          564  +      if {$::MSVC} {
          565  +        if {$arg eq "--disable-amalgamation"} {
          566  +          lappend makeOpts USE_AMALGAMATION=0
          567  +          continue
          568  +        }
          569  +        if {$arg eq "--disable-shared"} {
          570  +          lappend makeOpts USE_CRT_DLL=0 DYNAMIC_SHELL=0
          571  +          continue
          572  +        }
          573  +        if {$arg eq "--enable-fts5"} {
          574  +          lappend opts -DSQLITE_ENABLE_FTS5
          575  +          continue
          576  +        }
          577  +        if {$arg eq "--enable-json1"} {
          578  +          lappend opts -DSQLITE_ENABLE_JSON1
          579  +          continue
          580  +        }
          581  +        if {$arg eq "--enable-shared"} {
          582  +          lappend makeOpts USE_CRT_DLL=1 DYNAMIC_SHELL=1
          583  +          continue
          584  +        }
          585  +      }
   407    586         lappend configOpts $arg
   408    587       } else {
          588  +      if {$::MSVC} {
          589  +        if {$arg eq "-g"} {
          590  +          lappend cflags -Zi
          591  +          continue
          592  +        }
          593  +        if {[regexp -- {^-O(\d+)$} $arg all level]} then {
          594  +          lappend makeOpts OPTIMIZATIONS=$level
          595  +          continue
          596  +        }
          597  +      }
   409    598         lappend cflags $arg
   410    599       }
   411    600     }
   412    601   
   413         -  set cflags [join $cflags " "]
   414         -  set opts   [join $opts " "]
   415         -  append opts " -DSQLITE_NO_SYNC=1"
          602  +  # Disable sync to make testing faster.
          603  +  #
          604  +  lappend opts -DSQLITE_NO_SYNC=1
   416    605   
   417    606     # Some configurations already set HAVE_USLEEP; in that case, skip it.
   418    607     #
   419         -  if {![regexp { -DHAVE_USLEEP$} $opts]
   420         -         && ![regexp { -DHAVE_USLEEP[ =]+} $opts]} {
   421         -    append opts " -DHAVE_USLEEP=1"
          608  +  if {[lsearch -regexp $opts {^-DHAVE_USLEEP(?:=|$)}]==-1} {
          609  +    lappend opts -DHAVE_USLEEP=1
          610  +  }
          611  +
          612  +  # Add the define for this platform.
          613  +  #
          614  +  if {$::tcl_platform(platform)=="windows"} {
          615  +    lappend opts -DSQLITE_OS_WIN=1
          616  +  } else {
          617  +    lappend opts -DSQLITE_OS_UNIX=1
   422    618     }
   423    619   
   424    620     # Set the sub-directory to use.
   425    621     #
   426    622     set dir [string tolower [string map {- _ " " _} $name]]
   427    623   
   428         -  if {$::tcl_platform(platform)=="windows"} {
   429         -    append opts " -DSQLITE_OS_WIN=1"
   430         -  } else {
   431         -    append opts " -DSQLITE_OS_UNIX=1"
   432         -  }
          624  +  # Join option lists into strings, using space as delimiter.
          625  +  #
          626  +  set makeOpts [join $makeOpts " "]
          627  +  set cflags   [join $cflags " "]
          628  +  set opts     [join $opts " "]
   433    629   
   434         -  if {!$::TRACE} {
   435         -    set n [string length $title]
   436         -    PUTS -nonewline "${title}[string repeat . [expr {63-$n}]]"
   437         -    flush stdout
   438         -  }
   439         -
   440         -  set rc 0
   441         -  set tm1 [clock seconds]
   442         -  set origdir [pwd]
   443         -  trace_cmd file mkdir $dir
   444         -  trace_cmd cd $dir
   445         -  set errmsg {}
   446         -  catch {file delete core}
   447         -  set rc [catch [configureCommand $configOpts]]
   448         -  if {!$rc} {
   449         -    set rc [catch [makeCommand $testtarget $cflags $opts]]
   450         -    count_tests_and_errors test.log rc errmsg
   451         -  }
   452         -  trace_cmd cd $origdir
   453         -  set tm2 [clock seconds]
   454         -
   455         -  if {!$::TRACE} {
   456         -    set hours [expr {($tm2-$tm1)/3600}]
   457         -    set minutes [expr {(($tm2-$tm1)/60)%60}]
   458         -    set seconds [expr {($tm2-$tm1)%60}]
   459         -    set tm [format (%02d:%02d:%02d) $hours $minutes $seconds]
   460         -    if {$rc} {
   461         -      PUTS " FAIL $tm"
   462         -      incr ::NERR
   463         -    } else {
   464         -      PUTS " Ok   $tm"
   465         -    }
   466         -    if {$errmsg!=""} {PUTS "     $errmsg"}
   467         -  }
          630  +  lappend alltests [list \
          631  +      $title $dir $configOpts $testtarget $makeOpts $cflags $opts]
   468    632   }
   469    633   
   470    634   # The following procedure returns the "configure" command to be exectued for
   471    635   # the current platform, which may be Windows (via MinGW, etc).
   472    636   #
   473    637   proc configureCommand {opts} {
   474    638     if {$::MSVC} return [list]; # This is not needed for MSVC.
................................................................................
   480    644     foreach x $opts {lappend result $x}
   481    645     lappend result >& test.log
   482    646   }
   483    647   
   484    648   # The following procedure returns the "make" command to be executed for the
   485    649   # specified targets, compiler flags, and options.
   486    650   #
   487         -proc makeCommand { targets cflags opts } {
          651  +proc makeCommand { targets makeOpts cflags opts } {
   488    652     set result [list trace_cmd exec]
   489    653     if {$::MSVC} {
   490    654       set nmakeDir [file nativename $::SRCDIR]
   491         -    set nmakeFile [file join $nmakeDir Makefile.msc]
   492         -    lappend result nmake /f $nmakeFile TOP=$nmakeDir clean
          655  +    set nmakeFile [file nativename [file join $nmakeDir Makefile.msc]]
          656  +    lappend result nmake /f $nmakeFile TOP=$nmakeDir
   493    657     } else {
   494         -    lappend result make clean
          658  +    lappend result make
          659  +  }
          660  +  foreach makeOpt $makeOpts {
          661  +    lappend result $makeOpt
   495    662     }
          663  +  lappend result clean
   496    664     foreach target $targets {
   497    665       lappend result $target
   498    666     }
   499    667     lappend result CFLAGS=$cflags OPTS=$opts >>& test.log
   500    668   }
   501    669   
   502    670   # The following procedure prints its arguments if ::TRACE is true.
................................................................................
   503    671   # And it executes the command of its arguments in the calling context
   504    672   # if ::DRYRUN is false.
   505    673   #
   506    674   proc trace_cmd {args} {
   507    675     if {$::TRACE} {
   508    676       PUTS $args
   509    677     }
          678  +  set res ""
   510    679     if {!$::DRYRUN} {
   511         -    uplevel 1 $args
          680  +    set res [uplevel 1 $args]
   512    681     }
          682  +  return $res
   513    683   }
   514    684   
   515    685   
   516    686   # This proc processes the command line options passed to this script.
   517    687   # Currently the only option supported is "-makefile", default
   518    688   # "releasetest.mk". Set the ::MAKEFILE variable to the value of this
   519    689   # option.
................................................................................
   520    690   #
   521    691   proc process_options {argv} {
   522    692     set ::SRCDIR    [file normalize [file dirname [file dirname $::argv0]]]
   523    693     set ::QUICK     0
   524    694     set ::MSVC      0
   525    695     set ::BUILDONLY 0
   526    696     set ::DRYRUN    0
   527         -  set ::EXEC      exec
   528    697     set ::TRACE     0
          698  +  set ::JOBS      1
   529    699     set ::WITHTCL   {}
   530    700     set config {}
   531    701     set platform $::tcl_platform(os)-$::tcl_platform(machine)
   532    702   
   533    703     for {set i 0} {$i < [llength $argv]} {incr i} {
   534    704       set x [lindex $argv $i]
   535    705       if {[regexp {^--[a-z]} $x]} {set x [string range $x 1 end]}
   536    706       switch -glob -- $x {
          707  +      -slave {
          708  +        run_slave_test
          709  +        exit
          710  +      }
          711  +
   537    712         -srcdir {
   538    713           incr i
   539    714           set ::SRCDIR [file normalize [lindex $argv $i]]
   540    715         }
   541    716   
   542    717         -platform {
   543    718           incr i
   544    719           set platform [lindex $argv $i]
   545    720         }
          721  +
          722  +      -jobs {
          723  +        incr i
          724  +        set ::JOBS [lindex $argv $i]
          725  +      }
   546    726   
   547    727         -quick {
   548    728           set ::QUICK 1
   549    729         }
   550    730         -veryquick {
   551    731           set ::QUICK 2
   552    732         }
................................................................................
   593    773           foreach y [lsort [array names ::Configs]] {
   594    774             PUTS "   [list $y]"
   595    775           }
   596    776           exit
   597    777         }
   598    778   
   599    779         -g {
   600         -        if {$::MSVC} {
   601         -          lappend ::EXTRACONFIG -Zi
   602         -        } else {
   603         -          lappend ::EXTRACONFIG [lindex $argv $i]
   604         -        }
          780  +        lappend ::EXTRACONFIG [lindex $argv $i]
   605    781         }
   606    782   
   607    783         -with-tcl=* {
   608    784           set ::WITHTCL -$x
   609    785         }
   610    786   
   611    787         -D* -
................................................................................
   685    861           set target testfixture
   686    862           if {$::MSVC} {append target .exe}
   687    863         }
   688    864       }
   689    865       set config_options [concat $::Configs($zConfig) $::EXTRACONFIG]
   690    866   
   691    867       incr NTEST
   692         -    run_test_suite $zConfig $target $config_options
          868  +    add_test_suite all $zConfig $target $config_options
   693    869   
   694    870       # If the configuration included the SQLITE_DEBUG option, then remove
   695    871       # it and run veryquick.test. If it did not include the SQLITE_DEBUG option
   696    872       # add it and run veryquick.test.
   697    873       if {$target!="checksymbols" && $target!="valgrindtest"
   698    874              && $target!="fuzzoomtest" && !$::BUILDONLY && $::QUICK<2} {
   699    875         set debug_idx [lsearch -glob $config_options -DSQLITE_DEBUG*]
   700    876         set xtarget $target
   701    877         regsub -all {fulltest[a-z]*} $xtarget test xtarget
   702    878         regsub -all {fuzzoomtest} $xtarget fuzztest xtarget
   703    879         if {$debug_idx < 0} {
   704    880           incr NTEST
   705    881           append config_options " -DSQLITE_DEBUG=1"
   706         -        run_test_suite "${zConfig}_debug" $xtarget $config_options
          882  +        add_test_suite all "${zConfig}_debug" $xtarget $config_options
   707    883         } else {
   708    884           incr NTEST
   709    885           regsub { *-DSQLITE_MEMDEBUG[^ ]* *} $config_options { } config_options
   710    886           regsub { *-DSQLITE_DEBUG[^ ]* *} $config_options { } config_options
   711         -        run_test_suite "${zConfig}_ndebug" $xtarget $config_options
          887  +        add_test_suite all "${zConfig}_ndebug" $xtarget $config_options
   712    888         }
   713    889       }
   714    890     }
          891  +
          892  +  run_all_test_suites $all
   715    893   
   716    894     set elapsetime [expr {[clock seconds]-$STARTTIME}]
   717    895     set hr [expr {$elapsetime/3600}]
   718    896     set min [expr {($elapsetime/60)%60}]
   719    897     set sec [expr {$elapsetime%60}]
   720    898     set etime [format (%02d:%02d:%02d) $hr $min $sec]
   721    899     PUTS [string repeat * 79]