1#!/usr/bin/env python2.7
2
3# Copyright 2015, ARM Limited
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import argparse
30import fcntl
31import git
32import itertools
33import multiprocessing
34import os
35from os.path import join
36import platform
37import re
38import subprocess
39import sys
40import time
41
42import config
43import lint
44import printer
45import test
46import threaded_tests
47import util
48
49
50dir_root = config.dir_root
51
52def Optionify(name):
53  return '--' + name
54
55
56# The options that can be tested are abstracted to provide an easy way to add
57# new ones.
58# Environment options influence the environment. They can be used for example to
59# set the compiler used.
60# Build options are options passed to scons, with a syntax like `scons opt=val`
61# Runtime options are options passed to the test program.
62# See the definition of `test_options` below.
63
64# 'all' is a special value for the options. If specified, all other values of
65# the option are tested.
66class TestOption(object):
67  type_environment = 'type_environment'
68  type_build = 'type_build'
69  type_run = 'type_run'
70
71  def __init__(self, option_type, name, help,
72               val_test_choices, val_test_default = None,
73               # If unset, the user can pass any value.
74               strict_choices = True):
75    self.name = name
76    self.option_type = option_type
77    self.help = help
78    self.val_test_choices = val_test_choices
79    self.strict_choices = strict_choices
80    if val_test_default is not None:
81      self.val_test_default = val_test_default
82    else:
83      self.val_test_default = val_test_choices[0]
84
85  def ArgList(self, to_test):
86    res = []
87    if to_test == 'all':
88      for value in self.val_test_choices:
89        if value != 'all':
90          res.append(self.GetOptionString(value))
91    else:
92      for value in to_test:
93        res.append(self.GetOptionString(value))
94    return res
95
96class EnvironmentOption(TestOption):
97  option_type = TestOption.type_environment
98  def __init__(self, name, environment_variable_name, help,
99               val_test_choices, val_test_default = None,
100               strict_choices = True):
101    super(EnvironmentOption, self).__init__(EnvironmentOption.option_type,
102                                      name,
103                                      help,
104                                      val_test_choices,
105                                      val_test_default,
106                                      strict_choices = strict_choices)
107    self.environment_variable_name = environment_variable_name
108
109  def GetOptionString(self, value):
110    return self.environment_variable_name + '=' + value
111
112
113class BuildOption(TestOption):
114  option_type = TestOption.type_build
115  def __init__(self, name, help,
116               val_test_choices, val_test_default = None,
117               strict_choices = True):
118    super(BuildOption, self).__init__(BuildOption.option_type,
119                                      name,
120                                      help,
121                                      val_test_choices,
122                                      val_test_default,
123                                      strict_choices = strict_choices)
124  def GetOptionString(self, value):
125    return self.name + '=' + value
126
127
128class RuntimeOption(TestOption):
129  option_type = TestOption.type_run
130  def __init__(self, name, help,
131               val_test_choices, val_test_default = None):
132    super(RuntimeOption, self).__init__(RuntimeOption.option_type,
133                                        name,
134                                        help,
135                                        val_test_choices,
136                                        val_test_default)
137  def GetOptionString(self, value):
138    if value == 'on':
139      return Optionify(self.name)
140    else:
141      return None
142
143
144
145environment_option_compiler = \
146  EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.',
147                    val_test_choices=['all'] + config.tested_compilers,
148                    strict_choices = False)
149test_environment_options = [
150  environment_option_compiler
151]
152
153build_option_mode = \
154  BuildOption('mode', 'Test with the specified build modes.',
155              val_test_choices=['all'] + config.build_options_modes)
156build_option_standard = \
157  BuildOption('std', 'Test with the specified C++ standard.',
158              val_test_choices=['all'] + config.tested_cpp_standards,
159              strict_choices = False)
160test_build_options = [
161  build_option_mode,
162  build_option_standard
163]
164
165runtime_option_debugger = \
166  RuntimeOption('debugger',
167                '''Test with the specified configurations for the debugger.
168                Note that this is only tested if we are using the simulator.''',
169                val_test_choices=['all', 'on', 'off'])
170test_runtime_options = [
171  runtime_option_debugger
172]
173
174test_options = \
175  test_environment_options + test_build_options + test_runtime_options
176
177
178def BuildOptions():
179  args = argparse.ArgumentParser(
180    description =
181    '''This tool runs all tests matching the speficied filters for multiple
182    environment, build options, and runtime options configurations.''',
183    # Print default values.
184    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
185
186  args.add_argument('filters', metavar='filter', nargs='*',
187                    help='Run tests matching all of the (regexp) filters.')
188
189  # We automatically build the script options from the options to be tested.
190  test_arguments = args.add_argument_group(
191    'Test options',
192    'These options indicate what should be tested')
193  for option in test_options:
194    choices = option.val_test_choices if option.strict_choices else None
195    help = option.help
196    if not option.strict_choices:
197      help += ' Supported values: {' + ','.join(option.val_test_choices) + '}'
198    test_arguments.add_argument(Optionify(option.name),
199                                nargs='+',
200                                choices=choices,
201                                default=option.val_test_default,
202                                help=help,
203                                action='store')
204
205  general_arguments = args.add_argument_group('General options')
206  general_arguments.add_argument('--fast', action='store_true',
207                                 help='''Skip the lint tests, and run only with
208                                 one compiler, in one mode, with one C++
209                                 standard, and with an appropriate default for
210                                 runtime options. The compiler, mode, and C++
211                                 standard used are the first ones provided to
212                                 the script or in the default arguments.''')
213  general_arguments.add_argument(
214    '--jobs', '-j', metavar='N', type=int, nargs='?',
215    default=multiprocessing.cpu_count(),
216    const=multiprocessing.cpu_count(),
217    help='''Runs the tests using N jobs. If the option is set but no value is
218    provided, the script will use as many jobs as it thinks useful.''')
219  general_arguments.add_argument('--nobench', action='store_true',
220                                 help='Do not run benchmarks.')
221  general_arguments.add_argument('--nolint', action='store_true',
222                                 help='Do not run the linter.')
223  general_arguments.add_argument('--notest', action='store_true',
224                                 help='Do not run tests.')
225  sim_default = 'off' if platform.machine() == 'aarch64' else 'on'
226  general_arguments.add_argument(
227    '--simulator', action='store', choices=['on', 'off'],
228    default=sim_default,
229    help='Explicitly enable or disable the simulator.')
230  general_arguments.add_argument(
231    '--under_valgrind', action='store_true',
232    help='''Run the test-runner commands under Valgrind.
233            Note that a few tests are known to fail because of
234            issues in Valgrind''')
235  return args.parse_args()
236
237
238def RunCommand(command, environment_options = None):
239  # Create a copy of the environment. We do not want to pollute the environment
240  # of future commands run.
241  environment = os.environ
242  # Configure the environment.
243  # TODO: We currently pass the options as strings, so we need to parse them. We
244  # should instead pass them as a data structure and build the string option
245  # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`.
246  if environment_options:
247    for option in environment_options:
248      opt, val = option.split('=')
249      environment[opt] = val
250
251  printable_command = ''
252  if environment_options:
253    printable_command += ' '.join(environment_options) + ' '
254  printable_command += ' '.join(command)
255
256  printable_command_orange = \
257    printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
258  printer.PrintOverwritableLine(printable_command_orange)
259  sys.stdout.flush()
260
261  # Start a process for the command.
262  # Interleave `stderr` and `stdout`.
263  p = subprocess.Popen(command,
264                       stdout=subprocess.PIPE,
265                       stderr=subprocess.STDOUT,
266                       env=environment)
267
268  # We want to be able to display a continuously updated 'work indicator' while
269  # the process is running. Since the process can hang if the `stdout` pipe is
270  # full, we need to pull from it regularly. We cannot do so via the
271  # `readline()` function because it is blocking, and would thus cause the
272  # indicator to not be updated properly. So use file control mechanisms
273  # instead.
274  indicator = ' (still working: %d seconds elapsed)'
275
276  # Mark the process output as non-blocking.
277  flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
278  fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
279
280  t_start = time.time()
281  t_last_indication = t_start
282  process_output = ''
283
284  # Keep looping as long as the process is running.
285  while p.poll() is None:
286    # Avoid polling too often.
287    time.sleep(0.1)
288    # Update the progress indicator.
289    t_current = time.time()
290    if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
291      printer.PrintOverwritableLine(
292        printable_command_orange + indicator % int(t_current - t_start))
293      sys.stdout.flush()
294      t_last_indication = t_current
295    # Pull from the process output.
296    while True:
297      try:
298        line = os.read(p.stdout.fileno(), 1024)
299      except OSError:
300        line = ''
301        break
302      if line == '': break
303      process_output += line
304
305  # The process has exited. Don't forget to retrieve the rest of its output.
306  out, err = p.communicate()
307  rc = p.poll()
308  process_output += out
309
310  if rc == 0:
311    printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
312  else:
313    printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
314    printer.Print(process_output)
315  return rc
316
317
318def RunLinter():
319  rc, default_tracked_files = lint.GetDefaultTrackedFiles()
320  if rc:
321    return rc
322  return lint.LintFiles(map(lambda x: join(dir_root, x), default_tracked_files),
323                        jobs = args.jobs, progress_prefix = 'cpp lint: ')
324
325
326
327def BuildAll(build_options, jobs):
328  scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)]
329  scons_command += list(build_options)
330  return RunCommand(scons_command, list(environment_options))
331
332
333def RunBenchmarks():
334  rc = 0
335  benchmark_names = util.ListCCFilesWithoutExt(config.dir_benchmarks)
336  for bench in benchmark_names:
337    rc |= RunCommand(
338      [os.path.realpath(join(config.dir_build_latest, 'benchmarks', bench))])
339  return rc
340
341
342def PrintStatus(success):
343  printer.Print('\n$ ' + ' '.join(sys.argv))
344  if success:
345    printer.Print('SUCCESS')
346  else:
347    printer.Print('FAILURE')
348
349
350
351if __name__ == '__main__':
352  util.require_program('scons')
353  rc = 0
354
355  args = BuildOptions()
356
357  if args.under_valgrind:
358    util.require_program('valgrind')
359
360  if args.fast:
361    def SetFast(option, specified, default):
362      option.val_test_choices = \
363        [default[0] if specified == 'all' else specified[0]]
364    SetFast(environment_option_compiler, args.compiler, config.tested_compilers)
365    SetFast(build_option_mode, args.mode, config.build_options_modes)
366    SetFast(build_option_standard, args.std, config.tested_cpp_standards)
367    SetFast(runtime_option_debugger, args.debugger, ['on', 'off'])
368
369  if not args.nolint and not args.fast:
370    rc |= RunLinter()
371
372  # Don't try to test the debugger if we are not running with the simulator.
373  if not args.simulator:
374    test_runtime_options = \
375      filter(lambda x: x.name != 'debugger', test_runtime_options)
376
377  # List all combinations of options that will be tested.
378  def ListCombinations(args, options):
379    opts_list = map(lambda opt : opt.ArgList(args.__dict__[opt.name]), options)
380    return list(itertools.product(*opts_list))
381  test_env_combinations = ListCombinations(args, test_environment_options)
382  test_build_combinations = ListCombinations(args, test_build_options)
383  test_runtime_combinations = ListCombinations(args, test_runtime_options)
384
385  for environment_options in test_env_combinations:
386    for build_options in test_build_combinations:
387      # Avoid going through the build stage if we are not using the build
388      # result.
389      if not (args.notest and args.nobench):
390        build_rc = BuildAll(build_options, args.jobs)
391        # Don't run the tests for this configuration if the build failed.
392        if build_rc != 0:
393          rc |= build_rc
394          continue
395
396      # Use the realpath of the test executable so that the commands printed
397      # can be copy-pasted and run.
398      test_executable = os.path.realpath(
399        join(config.dir_build_latest, 'test', 'test-runner'))
400
401      if not args.notest:
402        printer.Print(test_executable)
403
404      for runtime_options in test_runtime_combinations:
405        if not args.notest:
406          runtime_options = [x for x in runtime_options if x is not None]
407          prefix = '  ' + ' '.join(runtime_options) + '  '
408          rc |= threaded_tests.RunTests(test_executable,
409                                        args.filters,
410                                        list(runtime_options),
411                                        args.under_valgrind,
412                                        jobs = args.jobs, prefix = prefix)
413
414      if not args.nobench:
415        rc |= RunBenchmarks()
416
417  PrintStatus(rc == 0)
418