test.py revision ae093bfb93c7d6b6cc143546a4211995a9db4ebf
1ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl#!/usr/bin/env python2.7 2ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 3ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# Copyright 2013, ARM Limited 4ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# All rights reserved. 5ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# 6ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# Redistribution and use in source and binary forms, with or without 7ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# modification, are permitted provided that the following conditions are met: 8ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# 9ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# * Redistributions of source code must retain the above copyright notice, 10ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# this list of conditions and the following disclaimer. 11ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# * Redistributions in binary form must reproduce the above copyright notice, 12ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# this list of conditions and the following disclaimer in the documentation 13ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# and/or other materials provided with the distribution. 14ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# * Neither the name of ARM Limited nor the names of its contributors may be 15ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# used to endorse or promote products derived from this software without 16ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# specific prior written permission. 17ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# 18ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 19ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 29ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport os 30ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport sys 31ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport argparse 32ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport re 33ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport subprocess 34ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport time 35ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport util 36ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 37ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 38ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixlfrom printer import EnsureNewLine, Print, UpdateProgress 39ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 40ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 41ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixldef BuildOptions(): 42ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result = argparse.ArgumentParser(description = 43ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl '''This tool runs each test reported by $CCTEST --list (and filtered as 44ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl specified). A summary will be printed, and detailed test output will be 45ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl stored in log/$CCTEST.''') 46ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result.add_argument('filters', metavar='filter', nargs='*', 47ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl help='Run tests matching all of the (regexp) filters.') 48ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result.add_argument('--cctest', action='store', required=True, 49ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl help='The cctest executable to run.') 50ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result.add_argument('--coloured_trace', action='store_true', 51ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl help='''Pass --coloured_trace to cctest. This will put 52ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl colour codes in the log files. The coloured output 53ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl can be viewed by "less -R", for example.''') 54ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result.add_argument('--coverage', action='store_true', 55ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl help='Run coverage tests.') 56ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result.add_argument('--debugger', action='store_true', 57ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl help='''Pass --debugger to cctest, so that the debugger is 58ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl used instead of the simulator. This has no effect 59ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl when running natively.''') 60ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl result.add_argument('--verbose', action='store_true', 61ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl help='Print verbose output.') 62ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl return result.parse_args() 63ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 64ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 65ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixldef VerbosePrint(string): 66ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if args.verbose: 67ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print(string) 68ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 69ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 70ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl# A class representing an individual test. 71ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixlclass Test: 72ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl def __init__(self, name): 73ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl self.name = name 74ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl self.logpath = os.path.join('log', os.path.basename(args.cctest)) 75ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if args.debugger: 76ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl basename = name + '_debugger' 77ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl else: 78ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl basename = name 79ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl self.logout = os.path.join(self.logpath, basename + '.stdout') 80ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl self.logerr = os.path.join(self.logpath, basename + '.stderr') 81ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 82ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl # Run the test. 83ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl # Use a thread to be able to control the test. 84ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl def Run(self): 85ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl command = [args.cctest, '--trace_sim', '--trace_reg', self.name] 86ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if args.coloured_trace: 87ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl command.append('--coloured_trace') 88ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 89ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl VerbosePrint('==== Running ' + self.name + '... ====') 90ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl sys.stdout.flush() 91ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 92ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl process = subprocess.Popen(command, 93ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl stdout=subprocess.PIPE, 94ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl stderr=subprocess.PIPE) 95ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Get the output and return status of the test. 96ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl stdout, stderr = process.communicate() 97ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl retcode = process.poll() 98ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 99ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Write stdout and stderr to the log. 100ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if not os.path.exists(self.logpath): os.makedirs(self.logpath) 101ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl with open(self.logout, 'w') as f: f.write(stdout) 102ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl with open(self.logerr, 'w') as f: f.write(stderr) 103ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 104ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if retcode == 0: 105ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Success. 106ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # We normally only print the command on failure, but with --verbose we 107ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # should also print it on success. 108ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl VerbosePrint('COMMAND: ' + ' '.join(command)) 109ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl VerbosePrint('LOG (stdout): ' + self.logout) 110ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl VerbosePrint('LOG (stderr): ' + self.logerr + '\n') 111ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl else: 112ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Failure. 113ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print('--- FAILED ' + self.name + ' ---') 114ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print('COMMAND: ' + ' '.join(command)) 115ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print('LOG (stdout): ' + self.logout) 116ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print('LOG (stderr): ' + self.logerr + '\n') 117ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 118ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl return retcode 119ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 120ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 121ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl# Scan matching tests and return a test manifest. 122ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixldef ReadManifest(filters): 123ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl status, output = util.getstatusoutput(args.cctest + ' --list') 124ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if status != 0: util.abort('Failed to list all tests') 125ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 126ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl names = output.split() 127ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl for f in filters: 128ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl names = filter(re.compile(f).search, names) 129ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 130ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl return map(Test, names) 131ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 132ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 133ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl# Run all tests in the manifest. 134ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixldef RunTests(manifest): 135ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl count = len(manifest) 136ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl passed = 0 137ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl failed = 0 138ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 139ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if count == 0: 140ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print('No tests to run.') 141ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl return 0 142ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 143ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl Print('Running %d tests...' % (count)) 144ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl start_time = time.time() 145ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 146ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl for test in manifest: 147ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Update the progress counter with the name of the test we're about to run. 148ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl UpdateProgress(start_time, passed, failed, count, args.verbose, test.name) 149ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl retcode = test.Run() 150ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Update the counters and progress indicator. 151ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if retcode == 0: 152ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl passed += 1 153ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl else: 154ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl failed += 1 155ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl UpdateProgress(start_time, passed, failed, count, args.verbose, '== Done ==') 156ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 157ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl return failed # 0 indicates success. 158ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 159ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 160ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlif __name__ == '__main__': 161ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl # $ROOT/tools/test.py 162ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl root_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) 163ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 164ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Parse the arguments. 165ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl args = BuildOptions() 166ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 167ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # Find a valid path to args.cctest (in case it doesn't begin with './'). 168ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl args.cctest = os.path.join('.', args.cctest) 169ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl 170ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if not os.access(args.cctest, os.X_OK): 171ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl print "'" + args.cctest + "' is not executable or does not exist." 172ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl sys.exit(1) 173ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 174ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl # List all matching tests. 175ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl manifest = ReadManifest(args.filters) 176ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 177ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl # Delete coverage data files. 178ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if args.coverage: 179ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl status, output = util.getstatusoutput('find obj/coverage -name "*.gcda" -exec rm {} \;') 180ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 181ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl # Run the tests. 182ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl status = RunTests(manifest) 183ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl EnsureNewLine() 184ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 185ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl # Print coverage information. 186ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl if args.coverage: 187ae093bfb93c7d6b6cc143546a4211995a9db4ebfarmvixl cmd = 'tggcov -R summary_all,untested_functions_per_file obj/coverage/src/a64' 188ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, 189ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl stderr=subprocess.PIPE) 190ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl stdout, stderr = p.communicate() 191ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl print(stdout) 192ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 193ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl sys.exit(status) 194ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 195