1ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#! @PERL@ 2ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown##--------------------------------------------------------------------## 3ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown##--- Valgrind regression testing script vg_regtest ---## 4ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown##--------------------------------------------------------------------## 5ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 6ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# This file is part of Valgrind, a dynamic binary instrumentation 7ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# framework. 8ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 9ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Copyright (C) 2003 Nicholas Nethercote 10ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# njn@valgrind.org 11ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 12ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# This program is free software; you can redistribute it and/or 13ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# modify it under the terms of the GNU General Public License as 14ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# published by the Free Software Foundation; either version 2 of the 15ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# License, or (at your option) any later version. 16ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 17ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# This program is distributed in the hope that it will be useful, but 18ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# WITHOUT ANY WARRANTY; without even the implied warranty of 19ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# General Public License for more details. 21ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 22ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# You should have received a copy of the GNU General Public License 23ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# along with this program; if not, write to the Free Software 24ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 25ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 02111-1307, USA. 26ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 27ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# The GNU General Public License is contained in the file COPYING. 28ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 29ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 30ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# usage: vg_regtest [options] <dirs | files> 31ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 32ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Options: 33ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# --all: run tests in all subdirs 34ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# --valgrind: valgrind launcher to use. Default is ./coregrind/valgrind. 35ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# (This option should probably only be used in conjunction with 36ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# --valgrind-lib.) 37ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# --valgrind-lib: valgrind libraries to use. Default is $tests_dir/.in_place. 38ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# (This option should probably only be used in conjunction with 39ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# --valgrind.) 40b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# --keep-unfiltered: keep a copy of the unfiltered output/error output 41b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# of each test by adding an extension .unfiltered.out 42ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 43436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov# --outer-valgrind: run this valgrind under the given outer valgrind. 44436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov# This valgrind must be configured with --enable-inner. 45436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov# --outer-tool: tool to use by the outer valgrind (default memcheck). 46436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov# --outer-args: use this as outer tool args. 47436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov# 48ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# The easiest way is to run all tests in valgrind/ with (assuming you installed 49ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# in $PREFIX): 50ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 51ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# $PREFIX/bin/vg_regtest --all 52ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 53ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# You can specify individual files to test, or whole directories, or both. 54ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Directories are traversed recursively, except for ones named, for example, 55ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# CVS/ or docs/. 56ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 57ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Each test is defined in a file <test>.vgtest, containing one or more of the 58ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# following lines, in any order: 59ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - prog: <prog to run> (compulsory) 60ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - args: <args for prog> (default: none) 61ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - vgopts: <Valgrind options> (default: none; 62ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# multiple are allowed) 63ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - stdout_filter: <filter to run stdout through> (default: none) 64ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - stderr_filter: <filter to run stderr through> (default: ./filter_stderr) 65b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stdout_filter_args: <args for stdout_filter> (default: basename of .vgtest file) 66b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stderr_filter_args: <args for stderr_filter> (default: basename of .vgtest file) 67b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# 68b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - progB: <prog to run in parallel with prog> (default: none) 69b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - argsB: <args for progB> (default: none) 70b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stdinB: <input file for progB> (default: none) 71b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stdoutB_filter: <filter progB stdout through> (default: none) 72b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stderrB_filter: <filter progB stderr through> (default: ./filter_stderr) 73b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stdoutB_filter_args: <args for stdout_filterB> (default: basename of .vgtest file) 74b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# - stderrB_filter_args: <args for stderr_filterB> (default: basename of .vgtest file) 75b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# 76ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - prereq: <prerequisite command> (default: none) 77ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - post: <post-test check command> (default: none) 78ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - cleanup: <post-test cleanup cmd> (default: none) 79ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 80b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# If prog or probB is a relative path, it will be prefix with the test directory. 81ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Note that filters are necessary for stderr results to filter out things that 82ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# always change, eg. process id numbers. 83b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# Note that if a progB is specified, it is started in background (before prog). 84ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 85ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Expected stdout (filtered) is kept in <test>.stdout.exp* (can be more 86ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# than one expected output). It can be missing if it would be empty. Expected 87ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# stderr (filtered) is kept in <test>.stderr.exp*. There must be at least 88ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# one stderr.exp* file. Any .exp* file that ends in '~' or '#' is ignored; 89ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# this is because Emacs creates temporary files of these names. 90ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 91b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# Expected output for progB is handled similarly, except that 92b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# expected stdout and stderr for progB are in <test>.stdoutB.exp* 93b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# and <test>.stderrB.exp*. 94b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# 95ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# If results don't match, the output can be found in <test>.std<strm>.out, 96ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# and the diff between expected and actual in <test>.std<strm>.diff*. 97b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# (for progB, in <test>.std<strm>2.out and <test>.std<strm>2.diff*). 98ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 99ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# The prerequisite command, if present, works like this: 100ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - if it returns 0 the test is run 101ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - if it returns 1 the test is skipped 102ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# - if it returns anything else the script aborts. 103ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# The idea here is results other than 0 or 1 are likely to be due to 104ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# problems with the commands, and you don't want to conflate them with the 1 105ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# case, which would happen if you just tested for zero or non-zero. 106ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 107ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# The post-test command, if present, must return 0 and its stdout must match 108ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# the expected stdout which is kept in <test>.post.exp*. 109ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 110ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Sometimes it is useful to run all the tests at a high sanity check 111ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# level or with arbitrary other flags. To make this simple, extra 112ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# options, applied to all tests run, are read from $EXTRA_REGTEST_OPTS, 113ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# and handed to valgrind prior to any other flags specified by the 114ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# .vgtest file. 115ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 116ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Some more notes on adding regression tests for a new tool are in 117ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# docs/xml/manual-writing-tools.xml. 118ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 119ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 120ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownuse warnings; 121ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownuse strict; 122ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 123ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 124ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Global vars 125ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 126ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $usage="\n" 127b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . "Usage:\n" 128436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " vg_regtest [--all, --valgrind, --valgrind-lib, --keep-unfiltered\n" 129436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " --outer-valgrind, --outer-tool, --outer-args]\n" 130b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . " Use EXTRA_REGTEST_OPTS to supply extra args for all tests\n" 131b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . "\n"; 132ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 133ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $tmp="vg_regtest.tmp.$$"; 134ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 135ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Test variables 136ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $vgopts; # valgrind options 137ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $prog; # test prog 138ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $args; # test prog args 139ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $stdout_filter; # filter program to run stdout results file through 140ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $stderr_filter; # filter program to run stderr results file through 141b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stdout_filter_args; # arguments passed to stdout_filter 142b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stderr_filter_args; # arguments passed to stderr_filter 143b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $progB; # Same but for progB 144b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $argsB; # 145b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stdoutB_filter; # 146b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stderrB_filter; # 147b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stdoutB_filter_args;# arguments passed to stdout_filterB 148b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stderrB_filter_args;# arguments passed to stderr_filterB 149b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $stdinB; # Input file for progB 150ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $prereq; # prerequisite test to satisfy before running test 151ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $post; # check command after running test 152ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $cleanup; # cleanup command to run 153ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 154ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy @failures; # List of failed tests 155ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 156ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $num_tests_done = 0; 157b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy %num_failures = (stderr => 0, stdout => 0, 158b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov stderrB => 0, stdoutB => 0, 159b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov post => 0); 160ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 161ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Default valgrind to use is this build tree's (uninstalled) one 162ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $valgrind = "./coregrind/valgrind"; 163ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 164ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownchomp(my $tests_dir = `pwd`); 165ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 166436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov# Outer valgrind to use, and args to use for it. 167436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovmy $outer_valgrind; 168436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovmy $outer_tool = "memcheck"; 169436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanovmy $outer_args; 170436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 171ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $valgrind_lib = "$tests_dir/.in_place"; 172b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovmy $keepunfiltered = 0; 173ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 174ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# default filter is the one named "filter_stderr" in the test's directory 175ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy $default_stderr_filter = "filter_stderr"; 176ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 177ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 178ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 179ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Process command line, setup 180ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 181ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 182ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# If $prog is a relative path, it prepends $dir to it. Useful for two reasons: 183ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 184ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 1. Can prepend "." onto programs to avoid trouble with users who don't have 185ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# "." in their path (by making $dir = ".") 186ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 2. Can prepend the current dir to make the command absolute to avoid 187ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# subsequent trouble when we change directories. 188ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# 189ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Also checks the program exists and is executable. 190ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub validate_program ($$$$) 191ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 192ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my ($dir, $prog, $must_exist, $must_be_executable) = @_; 193ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 194ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # If absolute path, leave it alone. If relative, make it 195ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # absolute -- by prepending current dir -- so we can change 196ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # dirs and still use it. 197ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $prog = "$dir/$prog" if ($prog !~ /^\//); 198ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($must_exist) { 199ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (-f $prog) or die "vg_regtest: `$prog' not found or not a file ($dir)\n"; 200ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 201ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($must_be_executable) { 202ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (-x $prog) or die "vg_regtest: `$prog' not executable ($dir)\n"; 203ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 204ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 205ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return $prog; 206ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 207ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 208ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub process_command_line() 209ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 210ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $alldirs = 0; 211ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my @fs; 212ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 213ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for my $arg (@ARGV) { 214ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($arg =~ /^-/) { 215ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($arg =~ /^--all$/) { 216ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $alldirs = 1; 217ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($arg =~ /^--valgrind=(.*)$/) { 218ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $valgrind = $1; 219436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } elsif ($arg =~ /^--outer-valgrind=(.*)$/) { 220436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov $outer_valgrind = $1; 221436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } elsif ($arg =~ /^--outer-tool=(.*)$/) { 222436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov $outer_tool = $1; 223436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } elsif ($arg =~ /^--outer-args=(.*)$/) { 224436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov $outer_args = $1; 225ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($arg =~ /^--valgrind-lib=(.*)$/) { 226ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $valgrind_lib = $1; 227b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($arg =~ /^--keep-unfiltered$/) { 228b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $keepunfiltered = 1; 229ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 230ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown die $usage; 231ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 232ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 233ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown push(@fs, $arg); 234ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 235ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 236ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $valgrind = validate_program($tests_dir, $valgrind, 1, 0); 237436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov 238436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (defined $outer_valgrind) { 239436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov $outer_valgrind = validate_program($tests_dir, $outer_valgrind, 1, 1); 240436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (not defined $outer_args) { 241436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov $outer_args = 242436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov " --command-line-only=yes" 243436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " --run-libc-freeres=no --sim-hints=enable-outer" 244436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " --smc-check=all-non-file" 245436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " --vgdb=no --trace-children=yes --read-var-info=no" 246436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " --suppressions=" 247436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . validate_program($tests_dir,"./tests/outer_inner.supp",1,0) 248436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " --memcheck:leak-check=full --memcheck:show-reachable=no" 249436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . " "; 250436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 251436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 252ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 253ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($alldirs) { 254ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown @fs = (); 255ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown foreach my $f (glob "*") { 256ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown push(@fs, $f) if (-d $f); 257ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 258ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 259ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 260ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (0 != @fs) or die "No test files or directories specified\n"; 261ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 262ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return @fs; 263ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 264ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 265ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 266ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Read a .vgtest file 267ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 268ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub read_vgtest_file($) 269ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 270ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my ($f) = @_; 271ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 272ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Defaults. 273b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($vgopts, $prog, $args) = ("", undef, ""); 274b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($stdout_filter, $stderr_filter) = (undef, undef); 275b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($progB, $argsB, $stdinB) = (undef, "", undef); 276b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($stdoutB_filter, $stderrB_filter) = (undef, undef); 277b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($prereq, $post, $cleanup) = (undef, undef, undef); 278b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($stdout_filter_args, $stderr_filter_args) = (undef, undef); 279b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov ($stdoutB_filter_args, $stderrB_filter_args) = (undef, undef); 280ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 281ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Every test directory must have a "filter_stderr" 282ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $stderr_filter = validate_program(".", $default_stderr_filter, 1, 1); 283b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stderrB_filter = validate_program(".", $default_stderr_filter, 1, 1); 284b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 285ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 286ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown open(INPUTFILE, "< $f") || die "File $f not openable\n"; 287ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 288ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown while (my $line = <INPUTFILE>) { 289ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($line =~ /^\s*#/ || $line =~ /^\s*$/) { 290ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown next; 291ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*vgopts:\s*(.*)$/) { 292ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $addvgopts = $1; 293ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $addvgopts =~ s/\${PWD}/$ENV{PWD}/g; 294ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $vgopts = $vgopts . " " . $addvgopts; # Nb: Make sure there's a space! 295ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*prog:\s*(.*)$/) { 296ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $prog = validate_program(".", $1, 0, 0); 297ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*args:\s*(.*)$/) { 298ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $args = $1; 299ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*stdout_filter:\s*(.*)$/) { 300ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $stdout_filter = validate_program(".", $1, 1, 1); 301ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*stderr_filter:\s*(.*)$/) { 302ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $stderr_filter = validate_program(".", $1, 1, 1); 303b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stdout_filter_args:\s*(.*)$/) { 304b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stdout_filter_args = $1; 305b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stderr_filter_args:\s*(.*)$/) { 306b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stderr_filter_args = $1; 307b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*progB:\s*(.*)$/) { 308b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $progB = validate_program(".", $1, 0, 0); 309b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*argsB:\s*(.*)$/) { 310b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $argsB = $1; 311b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stdinB:\s*(.*)$/) { 312b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stdinB = $1; 313b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stdoutB_filter:\s*(.*)$/) { 314b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stdoutB_filter = validate_program(".", $1, 1, 1); 315b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stderrB_filter:\s*(.*)$/) { 316b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stderrB_filter = validate_program(".", $1, 1, 1); 317b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stdoutB_filter_args:\s*(.*)$/) { 318b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stdoutB_filter_args = $1; 319b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } elsif ($line =~ /^\s*stderrB_filter_args:\s*(.*)$/) { 320b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stderrB_filter_args = $1; 321ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*prereq:\s*(.*)$/) { 322ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $prereq = $1; 323ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*post:\s*(.*)$/) { 324ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $post = $1; 325ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) { 326ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $cleanup = $1; 327ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 328ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown die "Bad line in $f: $line\n"; 329ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 330ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 331ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown close(INPUTFILE); 332ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 333ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (!defined $prog) { 334ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $prog = ""; # allow no prog for testing error and --help cases 335ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 336ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 337ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 338ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 339ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Do one test 340ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 341ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Since most of the program time is spent in system() calls, need this to 342ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# propagate a Ctrl-C enabling us to quit. 343ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub mysystem($) 344ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 345ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $exit_code = system($_[0]); 346ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ($exit_code == 2) and exit 1; # 2 is SIGINT 347ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return $exit_code; 348ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 349ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 350b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# if $keepunfiltered, copies $1 to $1.unfiltered.out 351b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov# renames $0 tp $1 352b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanovsub filtered_rename($$) 353b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov{ 354b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if ($keepunfiltered == 1) { 355b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("cp $_[1] $_[1].unfiltered.out"); 356b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 357b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov rename ($_[0], $_[1]); 358b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov} 359b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 360b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 361ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# from a directory name like "/foo/cachesim/tests/" determine the tool name 362ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub determine_tool() 363ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 364ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $dir = `pwd`; 365ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $dir =~ /.*\/([^\/]+)\/tests.*/; # foo/tool_name/tests/foo 366ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return $1; 367ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 368ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 369ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Compare output against expected output; it should match at least one of 370ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# them. 371ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub do_diffs($$$$) 372ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 373ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my ($fullname, $name, $mid, $f_exps) = @_; 374ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 375ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown for my $f_exp (@$f_exps) { 376ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (-r $f_exp) or die "Could not read `$f_exp'\n"; 377ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 378ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Emacs produces temporary files that end in '~' and '#'. We ignore 379ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # these. 380ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($f_exp !~ /[~#]$/) { 381ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # $n is the (optional) suffix after the ".exp"; we tack it onto 382ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # the ".diff" file. 383ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $n = ""; 384ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($f_exp =~ /.*\.exp(.*)$/) { 385ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $n = $1; 386ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 387ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $n = ""; 388ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown ($f_exp eq "/dev/null") or die "Unexpected .exp file: $f_exp\n"; 389ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 390ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 391ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown mysystem("@DIFF@ $f_exp $name.$mid.out > $name.$mid.diff$n"); 392ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 393ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (not -s "$name.$mid.diff$n") { 394ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # A match; remove .out and any previously created .diff files. 395ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unlink("$name.$mid.out"); 396ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown unlink(<$name.$mid.diff*>); 397ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 398ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 399ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 400ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 401ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # If we reach here, none of the .exp files matched. 402ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "*** $name failed ($mid) ***\n"; 403ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown push(@failures, sprintf("%-40s ($mid)", "$fullname")); 404ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_failures{$mid}++; 405ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 406ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 407ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub do_one_test($$) 408ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 409ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my ($dir, $vgtest) = @_; 410ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $vgtest =~ /^(.*)\.vgtest/; 411ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $name = $1; 412ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $fullname = "$dir/$name"; 413ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 414ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Pull any extra options (for example, --sanity-level=4) 415ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # from $EXTRA_REGTEST_OPTS. 416ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $maybe_extraopts = $ENV{"EXTRA_REGTEST_OPTS"}; 417ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $extraopts = $maybe_extraopts ? $maybe_extraopts : ""; 418ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 419ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown read_vgtest_file($vgtest); 420ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 421ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (defined $prereq) { 422ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $prereq_res = system("$prereq"); 423ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (0 == $prereq_res) { 424ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Do nothing (ie. continue with the test) 425ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif (256 == $prereq_res) { 426ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Nb: weird Perl-ism -- exit code of '1' is seen by Perl as 256... 427ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Prereq failed, skip. 428ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:"); 429ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return; 430ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 431ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Bad prereq; abort. 432ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $prereq_res /= 256; 433ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown die "prereq returned $prereq_res: $prereq\n"; 434ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 435ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 436ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 437ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 438b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (defined $progB) { 439b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # If there is a progB, let's start it in background: 440b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov printf("%-16s valgrind $extraopts $vgopts $prog $args (progB: $progB $argsB)\n", 441b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov "$name:"); 442b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # progB.done used to detect child has finished. See below. 443b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # Note: redirection of stdout and stderr is before $progB to allow argsB 444b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # to e.g. redirect stdoutB to stderrB 445b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (defined $stdinB) { 446b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("(rm -f progB.done;" 447b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . " < $stdinB > $name.stdoutB.out 2> $name.stderrB.out $progB $argsB;" 448b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . "touch progB.done) &"); 449b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 450b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("(rm -f progB.done;" 451b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . " > $name.stdoutB.out 2> $name.stderrB.out $progB $argsB;" 452b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . "touch progB.done) &"); 453b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 454b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } else { 455b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov printf("%-16s valgrind $extraopts $vgopts $prog $args\n", "$name:"); 456b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 457b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 458ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Pass the appropriate --tool option for the directory (can be overridden 459436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov # by an "args:" line, though). 460ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $tool=determine_tool(); 461436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov if (defined $outer_valgrind ) { 462436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov # in an outer-inner setup, only set VALGRIND_LIB_INNER 463436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov mysystem( "VALGRIND_LIB_INNER=$valgrind_lib " 464436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "$outer_valgrind " 465436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "--tool=" . $outer_tool . " " 466436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "$outer_args " 467436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "--log-file=" . "$name.outer.log " 468436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "$valgrind --command-line-only=yes --memcheck:leak-check=no " 469436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "--sim-hints=no-inner-prefix " 470436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "--tool=$tool $extraopts $vgopts " 471436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "$prog $args > $name.stdout.out 2> $name.stderr.out"); 472436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } else { 473436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov # Set both VALGRIND_LIB and VALGRIND_LIB_INNER in case this Valgrind 474436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov # was configured with --enable-inner. 475436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov mysystem( "VALGRIND_LIB=$valgrind_lib VALGRIND_LIB_INNER=$valgrind_lib " 476436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "$valgrind --command-line-only=yes --memcheck:leak-check=no " 477436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "--tool=$tool $extraopts $vgopts " 478436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov . "$prog $args > $name.stdout.out 2> $name.stderr.out"); 479436e89c602e787e7a27dd6624b09beed41a0da8aDmitriy Ivanov } 480ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 481ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Filter stdout 482ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (defined $stdout_filter) { 483b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stdout_filter_args = $name if (! defined $stdout_filter_args); 484b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("$stdout_filter $stdout_filter_args < $name.stdout.out > $tmp"); 485b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov filtered_rename($tmp, "$name.stdout.out"); 486ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 487ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Find all the .stdout.exp files. If none, use /dev/null. 488ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my @stdout_exps = <$name.stdout.exp*>; 489ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown @stdout_exps = ( "/dev/null" ) if (0 == scalar @stdout_exps); 490ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do_diffs($fullname, $name, "stdout", \@stdout_exps); 491ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 492ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Filter stderr 493b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stderr_filter_args = $name if (! defined $stderr_filter_args); 494b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("$stderr_filter $stderr_filter_args < $name.stderr.out > $tmp"); 495b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov filtered_rename($tmp, "$name.stderr.out"); 496ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Find all the .stderr.exp files. At least one must exist. 497ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my @stderr_exps = <$name.stderr.exp*>; 498ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (0 != scalar @stderr_exps) or die "Could not find `$name.stderr.exp*'\n"; 499ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do_diffs($fullname, $name, "stderr", \@stderr_exps); 500ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 501b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (defined $progB) { 502b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # wait for the child to be finished 503b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # tried things such as: 504b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # wait; 505b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # $SIG{CHLD} = sub { wait }; 506b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # but nothing worked: 507b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # e.g. running mssnapshot.vgtest in a loop failed from time to time 508b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # due to some missing output (not yet written?). 509b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # So, we search progB.done during max 100 times 100 millisecond. 510b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov my $count; 511b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov for ($count = 1; $count <= 100; $count++) { 512b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (-f "progB.done") or select(undef, undef, undef, 0.100); 513b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 514b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # Filter stdout 515b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov if (defined $stdoutB_filter) { 516b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stdoutB_filter_args = $name if (! defined $stdoutB_filter_args); 517b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("$stdoutB_filter $stdoutB_filter_args < $name.stdoutB.out > $tmp"); 518b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov filtered_rename($tmp, "$name.stdoutB.out"); 519b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 520b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # Find all the .stdoutB.exp files. If none, use /dev/null. 521b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov my @stdoutB_exps = <$name.stdoutB.exp*>; 522b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov @stdoutB_exps = ( "/dev/null" ) if (0 == scalar @stdoutB_exps); 523b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do_diffs($fullname, $name, "stdoutB", \@stdoutB_exps); 524b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 525b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # Filter stderr 526b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $stderrB_filter_args = $name if (! defined $stderrB_filter_args); 527b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov mysystem("$stderrB_filter $stderrB_filter_args < $name.stderrB.out > $tmp"); 528b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov filtered_rename($tmp, "$name.stderrB.out"); 529b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov # Find all the .stderrB.exp files. At least one must exist. 530b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov my @stderrB_exps = <$name.stderrB.exp*>; 531b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov (0 != scalar @stderrB_exps) or die "Could not find `$name.stderrB.exp*'\n"; 532b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov do_diffs($fullname, $name, "stderrB", \@stderrB_exps); 533b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov } 534b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 535ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Maybe do post-test check 536ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (defined $post) { 537ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (mysystem("$post > $name.post.out") != 0) { 538ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print("post check failed: $post\n"); 539ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_failures{"post"}++; 540ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 541ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Find all the .post.exp files. If none, use /dev/null. 542ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my @post_exps = <$name.post.exp*>; 543ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown @post_exps = ( "/dev/null" ) if (0 == scalar @post_exps); 544ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do_diffs($fullname, $name, "post", \@post_exps); 545ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 546ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 547ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 548ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (defined $cleanup) { 549ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (system("$cleanup") == 0) or 550ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print("(cleanup operation failed: $cleanup)\n"); 551ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 552ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 553ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_tests_done++; 554ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 555ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 556ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 557ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Test one directory (and any subdirs) 558ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 559ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub test_one_dir($$); # forward declaration 560ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 561ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub test_one_dir($$) 562ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 563ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my ($dir, $prev_dirs) = @_; 564ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $dir =~ s/\/$//; # trim a trailing '/' 565ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 566ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Ignore dirs into which we should not recurse. 567ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; } 568ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 569ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown (-x "$tests_dir/tests/arch_test") or die 570ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown "vg_regtest: 'arch_test' is missing. Did you forget to 'make check'?\n"; 571ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 572ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Ignore any dir whose name matches that of an architecture which is not 573ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # the architecture we are running on. Eg. when running on x86, ignore 574ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # ppc/ directories ('arch_test' returns 1 for this case). Likewise for 575ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # the OS and platform. 576ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Nb: weird Perl-ism -- exit code of '1' is seen by Perl as 256... 577ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (256 == system("$tests_dir/tests/arch_test $dir")) { return; } 578ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (256 == system("$tests_dir/tests/os_test $dir")) { return; } 579ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($dir =~ /(\w+)-(\w+)/ && 580ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 256 == system("sh $tests_dir/tests/platform_test $1 $2")) { return; } 581ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 582ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown chdir($dir) or die "Could not change into $dir\n"; 583ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 584ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Nb: Don't prepend a '/' to the base directory 585ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir; 586ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $dashes = "-" x (50 - length $full_dir); 587ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 588ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my @fs = glob "*"; 589ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $found_tests = (0 != (grep { $_ =~ /\.vgtest$/ } @fs)); 590ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 591ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($found_tests) { 592ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "-- Running tests in $full_dir $dashes\n"; 593ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 594ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown foreach my $f (@fs) { 595ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (-d $f) { 596ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown test_one_dir($f, $full_dir); 597ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif ($f =~ /\.vgtest$/) { 598ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do_one_test($full_dir, $f); 599ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 600ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 601ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($found_tests) { 602ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "-- Finished tests in $full_dir $dashes\n"; 603ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 604ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 605ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown chdir(".."); 606ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 607ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 608ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 609ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# Summarise results 610ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 611ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub plural($) 612ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 613ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown return ( $_[0] == 1 ? "" : "s" ); 614ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 615ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 616ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub summarise_results 617ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 618ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $x = ( $num_tests_done == 1 ? "test" : "tests" ); 619ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 620ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown printf("\n== %d test%s, %d stderr failure%s, %d stdout failure%s, " 621b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov . "%d stderrB failure%s, %d stdoutB failure%s, " 622ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown . "%d post failure%s ==\n", 623ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_tests_done, plural($num_tests_done), 624ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_failures{"stderr"}, plural($num_failures{"stderr"}), 625ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_failures{"stdout"}, plural($num_failures{"stdout"}), 626b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $num_failures{"stderrB"}, plural($num_failures{"stderrB"}), 627b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov $num_failures{"stdoutB"}, plural($num_failures{"stdoutB"}), 628ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $num_failures{"post"}, plural($num_failures{"post"})); 629ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 630ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown foreach my $failure (@failures) { 631ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "$failure\n"; 632ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 633ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "\n"; 634ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 635ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 636ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 637ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# main(), sort of 638ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown#---------------------------------------------------------------------------- 639ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsub warn_about_EXTRA_REGTEST_OPTS() 640ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown{ 641ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "WARNING: \$EXTRA_REGTEST_OPTS is set. You probably don't want\n"; 642ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "to run the regression tests with it set, unless you are doing some\n"; 643ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "strange experiment, and/or you really know what you are doing.\n"; 644ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "\n"; 645ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 646ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 647ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown# nuke VALGRIND_OPTS 648ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown$ENV{"VALGRIND_OPTS"} = ""; 649ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 650ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownif ($ENV{"EXTRA_REGTEST_OPTS"}) { 651ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown print "\n"; 652ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown warn_about_EXTRA_REGTEST_OPTS(); 653ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 654ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 655ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownmy @fs = process_command_line(); 656ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownforeach my $f (@fs) { 657ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if (-d $f) { 658ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown test_one_dir($f, ""); 659ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 660ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # Allow the .vgtest suffix to be given or omitted 661ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown if ($f =~ /.vgtest$/ && -r $f) { 662ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown # do nothing 663ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } elsif (-r "$f.vgtest") { 664ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown $f = "$f.vgtest"; 665ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } else { 666ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown die "`$f' neither a directory nor a readable test file/name\n" 667ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 668ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $dir = `dirname $f`; chomp $dir; 669ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown my $file = `basename $f`; chomp $file; 670ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown chdir($dir) or die "Could not change into $dir\n"; 671ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown do_one_test($dir, $file); 672ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown chdir($tests_dir); 673ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown } 674ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 675ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownsummarise_results(); 676ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 677ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownif ($ENV{"EXTRA_REGTEST_OPTS"}) { 678ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown warn_about_EXTRA_REGTEST_OPTS(); 679ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 680ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 681ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brownif (0 == $num_failures{"stdout"} && 682ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 0 == $num_failures{"stderr"} && 683b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 0 == $num_failures{"stdoutB"} && 684b32f58018498ea2225959b0ba11c18f0c433deefEvgeniy Stepanov 0 == $num_failures{"stderrB"} && 685ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 0 == $num_failures{"post"}) { 686ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown exit 0; 687ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} else { 688ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown exit 1; 689ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown} 690ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown 691ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown##--------------------------------------------------------------------## 692ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown##--- end vg_regtest ---## 693ed07e00d438c74b7a23c01bfffde77e3968305e4Jeff Brown##--------------------------------------------------------------------## 694