1ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl#!/usr/bin/env python2.7 2ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 3b78f13911bfe6eda303e91ef215c87a165aae8aeAlexandre Rames# Copyright 2015, VIXL authors 4ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# All rights reserved. 5ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# 6ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# Redistribution and use in source and binary forms, with or without 7ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# modification, are permitted provided that the following conditions are met: 8ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# 9ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# * Redistributions of source code must retain the above copyright notice, 10ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# this list of conditions and the following disclaimer. 11ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# * Redistributions in binary form must reproduce the above copyright notice, 12ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# this list of conditions and the following disclaimer in the documentation 13ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# and/or other materials provided with the distribution. 14ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# * Neither the name of ARM Limited nor the names of its contributors may be 15ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# used to endorse or promote products derived from this software without 16ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# specific prior written permission. 17ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# 18ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 19ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 29ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport argparse 30db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport fcntl 31db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport git 32db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport itertools 33db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport multiprocessing 34db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport os 35db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlfrom os.path import join 36c68cb64496485710cdb5b8480f8fee287058c93farmvixlimport platform 37db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport re 38ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport subprocess 39db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport sys 40ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlimport time 41ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 42db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport config 430f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixlimport clang_format 44db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport lint 45db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport printer 46db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport test 47db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport threaded_tests 48db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlimport util 49ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 50ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 51db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldir_root = config.dir_root 52ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 53db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldef Optionify(name): 54db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return '--' + name 55ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 56ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 57db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# The options that can be tested are abstracted to provide an easy way to add 58db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# new ones. 59db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# Environment options influence the environment. They can be used for example to 60db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# set the compiler used. 61db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# Build options are options passed to scons, with a syntax like `scons opt=val` 62db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# Runtime options are options passed to the test program. 63db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# See the definition of `test_options` below. 64ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 65db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# 'all' is a special value for the options. If specified, all other values of 66db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl# the option are tested. 67db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlclass TestOption(object): 68db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl type_environment = 'type_environment' 69db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl type_build = 'type_build' 70db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl type_run = 'type_run' 71ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 72db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def __init__(self, option_type, name, help, 73db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, val_test_default = None, 74db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # If unset, the user can pass any value. 751c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois strict_choices = True, test_independently = False): 76ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl self.name = name 77db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.option_type = option_type 78db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.help = help 79db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.val_test_choices = val_test_choices 80db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.strict_choices = strict_choices 811c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois self.test_independently = test_independently 82db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if val_test_default is not None: 83db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.val_test_default = val_test_default 845799d6c5d10729eaade85ad608109c83ed1ae63barmvixl else: 85db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.val_test_default = val_test_choices[0] 86db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 87db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def ArgList(self, to_test): 88db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl res = [] 89db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if to_test == 'all': 90db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for value in self.val_test_choices: 91db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if value != 'all': 92db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl res.append(self.GetOptionString(value)) 93ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl else: 94db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for value in to_test: 95db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl res.append(self.GetOptionString(value)) 96db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return res 97db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 98db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlclass EnvironmentOption(TestOption): 99db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl option_type = TestOption.type_environment 100db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def __init__(self, name, environment_variable_name, help, 101db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, val_test_default = None, 102db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl strict_choices = True): 103db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl super(EnvironmentOption, self).__init__(EnvironmentOption.option_type, 104db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl name, 105db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help, 106db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, 107db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_default, 108db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl strict_choices = strict_choices) 109db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl self.environment_variable_name = environment_variable_name 110db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 111db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def GetOptionString(self, value): 112db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return self.environment_variable_name + '=' + value 113db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 114db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 115db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlclass BuildOption(TestOption): 116db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl option_type = TestOption.type_build 117db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def __init__(self, name, help, 118db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, val_test_default = None, 1191c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois strict_choices = True, test_independently = False): 120db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl super(BuildOption, self).__init__(BuildOption.option_type, 121db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl name, 122db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help, 123db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, 124db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_default, 1251c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois strict_choices = strict_choices, 1261c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois test_independently = test_independently) 127db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def GetOptionString(self, value): 128db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return self.name + '=' + value 129db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 130db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 131db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlclass RuntimeOption(TestOption): 132db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl option_type = TestOption.type_run 133db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def __init__(self, name, help, 134db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, val_test_default = None): 135db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl super(RuntimeOption, self).__init__(RuntimeOption.option_type, 136db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl name, 137db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help, 138db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices, 139db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_default) 140db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def GetOptionString(self, value): 141db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if value == 'on': 142db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return Optionify(self.name) 143db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl else: 144db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return None 145db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 146db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 147db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 148db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlenvironment_option_compiler = \ 149db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.', 150db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices=['all'] + config.tested_compilers, 151db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl strict_choices = False) 152db6443499376478f5281607a3923e6ffc4c8d8ecarmvixltest_environment_options = [ 153db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl environment_option_compiler 154db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl] 155db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 156db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlbuild_option_mode = \ 157db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl BuildOption('mode', 'Test with the specified build modes.', 158db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices=['all'] + config.build_options_modes) 159db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlbuild_option_standard = \ 160db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl BuildOption('std', 'Test with the specified C++ standard.', 161db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices=['all'] + config.tested_cpp_standards, 162db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl strict_choices = False) 1639a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfettabuild_option_target = \ 1649a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta BuildOption('target', 'Test with the specified isa enabled.', 1659a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta val_test_choices=['all'] + config.build_options_target, 1661c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois strict_choices = False, test_independently = True) 1671c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langloisbuild_option_negative_testing = \ 1681c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois BuildOption('negative_testing', 'Test with negative testing enabled.', 1691c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois val_test_choices=['all'] + config.build_options_negative_testing, 1701c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois strict_choices = False, test_independently = True) 171db6443499376478f5281607a3923e6ffc4c8d8ecarmvixltest_build_options = [ 172db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl build_option_mode, 1731c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois build_option_standard, 1749a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta build_option_target, 1751c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois build_option_negative_testing 176db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl] 177db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 178db6443499376478f5281607a3923e6ffc4c8d8ecarmvixlruntime_option_debugger = \ 179db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl RuntimeOption('debugger', 180db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl '''Test with the specified configurations for the debugger. 181db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl Note that this is only tested if we are using the simulator.''', 182db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl val_test_choices=['all', 'on', 'off']) 183db6443499376478f5281607a3923e6ffc4c8d8ecarmvixltest_runtime_options = [ 184db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl runtime_option_debugger 185db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl] 186db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 187db6443499376478f5281607a3923e6ffc4c8d8ecarmvixltest_options = \ 188db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_environment_options + test_build_options + test_runtime_options 189db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 1905799d6c5d10729eaade85ad608109c83ed1ae63barmvixl 191db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldef BuildOptions(): 192db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl args = argparse.ArgumentParser( 193db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl description = 1940f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl '''This tool runs all tests matching the specified filters for multiple 195db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl environment, build options, and runtime options configurations.''', 196db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Print default values. 197db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl formatter_class=argparse.ArgumentDefaultsHelpFormatter) 198db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 199db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl args.add_argument('filters', metavar='filter', nargs='*', 200db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help='Run tests matching all of the (regexp) filters.') 201db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 202db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # We automatically build the script options from the options to be tested. 203db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_arguments = args.add_argument_group( 204db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 'Test options', 205db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 'These options indicate what should be tested') 206db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for option in test_options: 207db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl choices = option.val_test_choices if option.strict_choices else None 208db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help = option.help 209db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if not option.strict_choices: 210db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help += ' Supported values: {' + ','.join(option.val_test_choices) + '}' 211db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_arguments.add_argument(Optionify(option.name), 212db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl nargs='+', 213db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl choices=choices, 214db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl default=option.val_test_default, 215db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help=help, 216db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl action='store') 217db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 218db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments = args.add_argument_group('General options') 219db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments.add_argument('--fast', action='store_true', 2200f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl help='''Skip the lint and clang-format tests, 2210f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl and run only with one compiler, in one mode, 2220f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl with one C++ standard, and with an appropriate 223dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames default for runtime options.''') 22459d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley general_arguments.add_argument('--dry-run', action='store_true', 22559d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley help='''Don't actually build or run anything, 22659d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley but print the configurations that would be 22759d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley tested.''') 228db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments.add_argument( 229db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl '--jobs', '-j', metavar='N', type=int, nargs='?', 230db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl default=multiprocessing.cpu_count(), 231db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl const=multiprocessing.cpu_count(), 232db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help='''Runs the tests using N jobs. If the option is set but no value is 233db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl provided, the script will use as many jobs as it thinks useful.''') 234db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments.add_argument('--nobench', action='store_true', 235db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help='Do not run benchmarks.') 236db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments.add_argument('--nolint', action='store_true', 237db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help='Do not run the linter.') 2380f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl general_arguments.add_argument('--noclang-format', action='store_true', 2390f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl help='Do not run clang-format.') 240db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments.add_argument('--notest', action='store_true', 241db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help='Do not run tests.') 24273064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames general_arguments.add_argument('--fail-early', action='store_true', 24373064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames help='Exit as soon as a test fails.') 244a3b21463768ff7521130a1cffa228e859157afeePierre Langlois sim_default = 'none' if platform.machine() == 'aarch64' else 'aarch64' 245db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl general_arguments.add_argument( 246a3b21463768ff7521130a1cffa228e859157afeePierre Langlois '--simulator', action='store', choices=['aarch64', 'none'], 247db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl default=sim_default, 248db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl help='Explicitly enable or disable the simulator.') 249684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl general_arguments.add_argument( 250684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl '--under_valgrind', action='store_true', 251684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl help='''Run the test-runner commands under Valgrind. 252684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl Note that a few tests are known to fail because of 253684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl issues in Valgrind''') 254db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return args.parse_args() 255db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 256db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 257db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldef RunCommand(command, environment_options = None): 258db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Create a copy of the environment. We do not want to pollute the environment 259db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # of future commands run. 260db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl environment = os.environ 261db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Configure the environment. 262db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # TODO: We currently pass the options as strings, so we need to parse them. We 263db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # should instead pass them as a data structure and build the string option 264db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`. 265db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if environment_options: 266db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for option in environment_options: 267db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl opt, val = option.split('=') 268db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl environment[opt] = val 269db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 270db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printable_command = '' 271db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if environment_options: 272db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printable_command += ' '.join(environment_options) + ' ' 273db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printable_command += ' '.join(command) 274db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 275db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printable_command_orange = \ 276db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR 277db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.PrintOverwritableLine(printable_command_orange) 278db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl sys.stdout.flush() 279db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 280db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Start a process for the command. 281db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Interleave `stderr` and `stdout`. 282db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl p = subprocess.Popen(command, 283db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl stdout=subprocess.PIPE, 284db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl stderr=subprocess.STDOUT, 285db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl env=environment) 286db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 287db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # We want to be able to display a continuously updated 'work indicator' while 288db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # the process is running. Since the process can hang if the `stdout` pipe is 289db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # full, we need to pull from it regularly. We cannot do so via the 290db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # `readline()` function because it is blocking, and would thus cause the 291db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # indicator to not be updated properly. So use file control mechanisms 292db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # instead. 293db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl indicator = ' (still working: %d seconds elapsed)' 294db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 295db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Mark the process output as non-blocking. 296db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL) 297db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK) 298db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 299db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl t_start = time.time() 300db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl t_last_indication = t_start 301db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl process_output = '' 302db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 303db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Keep looping as long as the process is running. 304db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl while p.poll() is None: 305db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Avoid polling too often. 306db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl time.sleep(0.1) 307db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Update the progress indicator. 308db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl t_current = time.time() 309db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1): 310db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.PrintOverwritableLine( 311db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printable_command_orange + indicator % int(t_current - t_start)) 312db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl sys.stdout.flush() 313db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl t_last_indication = t_current 314db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Pull from the process output. 315db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl while True: 316db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl try: 317db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl line = os.read(p.stdout.fileno(), 1024) 318db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl except OSError: 319db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl line = '' 320db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl break 321db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if line == '': break 322db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl process_output += line 323db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 324db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # The process has exited. Don't forget to retrieve the rest of its output. 325db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl out, err = p.communicate() 326db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl rc = p.poll() 327db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl process_output += out 328db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 329db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if rc == 0: 330db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR) 331db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl else: 332db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR) 333db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print(process_output) 334db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return rc 3355799d6c5d10729eaade85ad608109c83ed1ae63barmvixl 3364a102baf640077d6794c0b33bb976f94b86c532barmvixl 337db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldef RunLinter(): 338b2746629e867237cf6e54647872f0bd46d0130f2Alexandre Rames rc, default_tracked_files = lint.GetDefaultFilesToLint() 339db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if rc: 340db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return rc 341b2746629e867237cf6e54647872f0bd46d0130f2Alexandre Rames return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files), 342db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl jobs = args.jobs, progress_prefix = 'cpp lint: ') 3434a102baf640077d6794c0b33bb976f94b86c532barmvixl 3444a102baf640077d6794c0b33bb976f94b86c532barmvixl 3450f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixldef RunClangFormat(): 3460f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl return clang_format.ClangFormatFiles(clang_format.GetCppSourceFilesToFormat(), 3470f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl jobs = args.jobs, 3480f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl progress_prefix = 'clang-format: ') 3490f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl 3500f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl 3514a102baf640077d6794c0b33bb976f94b86c532barmvixl 352db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldef BuildAll(build_options, jobs): 353db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)] 354db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl scons_command += list(build_options) 355db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return RunCommand(scons_command, list(environment_options)) 356ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 3574a102baf640077d6794c0b33bb976f94b86c532barmvixl 3589a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta# Work out if the given options or args allow to run on the specified arch. 3599a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta# * arches is a list of ISA/architecture (a64, aarch32, etc) 3609a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta# * options are test.py's command line options if any. 3619a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta# * args are the arguments given to the build script. 3629a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfettadef CanRunOn(arches, options, args): 3639a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # First we check in the build specific options. 3649a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta for option in options: 3659a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta if 'target' in option: 3669a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # The option format is 'target=x,y,z'. 3679a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta for target in (option.split('='))[1].split(','): 3689a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta if target in arches: 3699a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta return True 3709a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3719a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # There was a target build option but it didn't include the target arch. 3729a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta return False 3739a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3749a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # No specific build option, check the script arguments. 3759a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # The meaning of 'all' will depend on the platform, e.g. 32-bit compilers 3769a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # cannot handle Aarch64 while 64-bit compiler can handle Aarch32. To avoid 3779a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta # any issues no benchmarks are run for target='all'. 3789a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta if args.target == 'all': return False 3799a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3809a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta for target in args.target[0].split(','): 3819a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta if target in arches: 3829a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta return True 3839a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3849a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta return False 3859a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3869a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3879a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfettadef CanRunAarch64(options, args): 3889a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta return CanRunOn(['aarch64', 'a64'], options, args) 3899a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3909a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3919a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfettadef CanRunAarch32(options, args): 3929a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta return CanRunOn(['aarch32', 'a32', 't32'], options, args) 3939a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3949a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta 3959a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfettadef RunBenchmarks(options, args): 396db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl rc = 0 3979a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta if CanRunAarch32(options, args): 3981c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks) 3991c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois for bench in benchmark_names: 4001c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois rc |= RunCommand( 4011c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois [os.path.realpath( 4029cd420f65453048b29e4d259f9cfdc440f1d9b92Martyn Capewell join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10']) 4039a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta if CanRunAarch64(options, args): 4041c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks) 4051c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois for bench in benchmark_names: 4061c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois rc |= RunCommand( 4071c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois [util.relrealpath( 4089cd420f65453048b29e4d259f9cfdc440f1d9b92Martyn Capewell join(config.dir_build_latest, 4099cd420f65453048b29e4d259f9cfdc440f1d9b92Martyn Capewell 'benchmarks/aarch64', bench)), '10']) 410db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return rc 4114a102baf640077d6794c0b33bb976f94b86c532barmvixl 4124a102baf640077d6794c0b33bb976f94b86c532barmvixl 413db6443499376478f5281607a3923e6ffc4c8d8ecarmvixldef PrintStatus(success): 414db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print('\n$ ' + ' '.join(sys.argv)) 415db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if success: 416db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print('SUCCESS') 417db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl else: 418db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print('FAILURE') 4194a102baf640077d6794c0b33bb976f94b86c532barmvixl 420ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 421ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 422ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixlif __name__ == '__main__': 423db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl util.require_program('scons') 424db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl rc = 0 425ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 426ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl args = BuildOptions() 427ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dcarmvixl 42873064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames def MaybeExitEarly(rc): 42973064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames if args.fail_early and rc != 0: 43073064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames PrintStatus(rc == 0) 43173064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames sys.exit(rc) 43273064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames 433684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl if args.under_valgrind: 434684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl util.require_program('valgrind') 435684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl 436db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if args.fast: 437db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def SetFast(option, specified, default): 438db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl option.val_test_choices = \ 439dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames [default if specified == 'all' else specified[0]] 440dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames # `g++` is very slow to compile a few aarch32 test files. 441dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames SetFast(environment_option_compiler, args.compiler, 'clang++') 442dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames SetFast(build_option_standard, args.std, 'c++98') 443dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames SetFast(build_option_mode, args.mode, 'debug') 444dd7de867971f6309d8380262e5f7abeca2a8c713Alexandre Rames SetFast(runtime_option_debugger, args.debugger, 'on') 445db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 44659d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley if not args.nolint and not (args.fast or args.dry_run): 447db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl rc |= RunLinter() 44873064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames MaybeExitEarly(rc) 44973064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames 45059d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley if not args.noclang_format and not (args.fast or args.dry_run): 4510f35e36b7f5d1d2f4d95989b418447e1a4bcc8cdarmvixl rc |= RunClangFormat() 45273064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames MaybeExitEarly(rc) 453db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 454db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Don't try to test the debugger if we are not running with the simulator. 455db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if not args.simulator: 456db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_runtime_options = \ 457db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl filter(lambda x: x.name != 'debugger', test_runtime_options) 458db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 459db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # List all combinations of options that will be tested. 460db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl def ListCombinations(args, options): 4611c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois opts_list = [ 4621c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois opt.ArgList(args.__dict__[opt.name]) 4631c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois for opt in options 4641c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois if not opt.test_independently 4651c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois ] 466db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl return list(itertools.product(*opts_list)) 4671c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois # List combinations of options that should only be tested independently. 468d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley def ListIndependentCombinations(args, options, base): 4691c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois n = [] 4701c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois for opt in options: 4711c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois if opt.test_independently: 4721c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois for o in opt.ArgList(args.__dict__[opt.name]): 473d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley n.append(base + (o,)) 4741c1488c11349b4dada8ca3cd0f56824b9f93478aPierre Langlois return n 47538d5d1bb12664c89d51f5c1bd75014ab2aabe4e9Georgia Kouveli # TODO: We should refine the configurations we test by default, instead of 47638d5d1bb12664c89d51f5c1bd75014ab2aabe4e9Georgia Kouveli # always testing all possible combinations. 477db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_env_combinations = ListCombinations(args, test_environment_options) 478db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_build_combinations = ListCombinations(args, test_build_options) 479d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley if not args.fast: 480d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley test_build_combinations.extend( 481d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley ListIndependentCombinations(args, 482d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley test_build_options, 483d503ed20afc22b8c8e6f299961169971df8e9602Jacob Bramley test_build_combinations[0])) 484db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl test_runtime_combinations = ListCombinations(args, test_runtime_options) 485db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 486db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for environment_options in test_env_combinations: 487db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for build_options in test_build_combinations: 48859d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley if (args.dry_run): 48959d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley for runtime_options in test_runtime_combinations: 49059d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley print(' '.join(filter(None, environment_options)) + ', ' + 49159d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley ' '.join(filter(None, build_options)) + ', ' + 49259d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley ' '.join(filter(None, runtime_options))) 49359d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley continue 49459d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley 495db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Avoid going through the build stage if we are not using the build 496db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # result. 497db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if not (args.notest and args.nobench): 498db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl build_rc = BuildAll(build_options, args.jobs) 499db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Don't run the tests for this configuration if the build failed. 500db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if build_rc != 0: 501db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl rc |= build_rc 50273064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames MaybeExitEarly(rc) 503db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl continue 504db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 505db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # Use the realpath of the test executable so that the commands printed 506db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl # can be copy-pasted and run. 50781c76e61334b6a3f4ae63b1dc52ca2ed51b7350fAlexandre Rames test_executable = util.relrealpath( 508db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl join(config.dir_build_latest, 'test', 'test-runner')) 509db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 510db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if not args.notest: 511db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl printer.Print(test_executable) 512db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 513db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl for runtime_options in test_runtime_combinations: 514db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if not args.notest: 515db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl runtime_options = [x for x in runtime_options if x is not None] 516db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl prefix = ' ' + ' '.join(runtime_options) + ' ' 517db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl rc |= threaded_tests.RunTests(test_executable, 518db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl args.filters, 519db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl list(runtime_options), 520684cd2a7f5845539b58d0da7e012e39df49ceff0armvixl args.under_valgrind, 521db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl jobs = args.jobs, prefix = prefix) 52273064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames MaybeExitEarly(rc) 523db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 524db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl if not args.nobench: 5259a9331faeba996d6c85e6e2a6355ccfc22c6cab6Rodolph Perfetta rc |= RunBenchmarks(build_options, args) 52673064a2fd9da446561b1d5fe18e972f9ee95bd36Alexandre Rames MaybeExitEarly(rc) 527db6443499376478f5281607a3923e6ffc4c8d8ecarmvixl 52859d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley if not args.dry_run: 52959d74ae24e5b2eea26c524ae96a54293d127cfb3Jacob Bramley PrintStatus(rc == 0) 5307c0ea8b64c6907a69ea79b3dd6d9f4dcec918b66Alexandre Rames 5317c0ea8b64c6907a69ea79b3dd6d9f4dcec918b66Alexandre Rames sys.exit(rc) 532