test.py revision ad96eda8944ab1c1ba55715c50d9d6f0a3ed1dc 
1#!/usr/bin/env python2.7
2
3# Copyright 2013, ARM Limited
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import os
30import sys
31import argparse
32import re
33import subprocess
34import threading
35import time
36import util
37
38
39def BuildOptions():
40  result = argparse.ArgumentParser(description = 'Unit test tool')
41  result.add_argument('name_filters', metavar='name filters', nargs='*',
42                      help='Tests matching any of the regexp filters will be run.')
43  result.add_argument('--mode', action='store', choices=['release', 'debug', 'coverage'],
44                      default='release', help='Build mode')
45  result.add_argument('--simulator', action='store', choices=['on', 'off'],
46                      default='on', help='Use the builtin a64 simulator')
47  result.add_argument('--timeout', action='store', type=int, default=5,
48                      help='Timeout (in seconds) for each cctest (5sec default).')
49  result.add_argument('--nobuild', action='store_true',
50                      help='Do not (re)build the tests')
51  result.add_argument('--jobs', '-j', metavar='N', type=int, default=1,
52                      help='Allow N jobs at once.')
53  return result.parse_args()
54
55
56def BuildRequiredObjects(arguments):
57  status, output = util.getstatusoutput('scons ' +
58                                        'mode=' + arguments.mode + ' ' +
59                                        'simulator=' + arguments.simulator + ' ' +
60                                        'target=cctest ' +
61                                        '--jobs=' + str(arguments.jobs))
62
63  if status != 0:
64    print(output)
65    util.abort('Failed bulding cctest')
66
67
68# Display the run progress:
69# [time| progress|+ passed|- failed]
70def UpdateProgress(start_time, passed, failed, card):
71  minutes, seconds = divmod(time.time() - start_time, 60)
72  progress = float(passed + failed) / card * 100
73  passed_colour = '\x1b[32m' if passed != 0 else ''
74  failed_colour = '\x1b[31m' if failed != 0 else ''
75  indicator = '\r[%02d:%02d| %3d%%|' + passed_colour + '+ %d\x1b[0m|' + failed_colour + '- %d\x1b[0m]'
76  sys.stdout.write(indicator % (minutes, seconds, progress, passed, failed))
77
78
79def PrintError(s):
80  # Print the error on a new line.
81  sys.stdout.write('\n')
82  print(s)
83  sys.stdout.flush()
84
85
86# List all tests matching any of the provided filters.
87def ListTests(cctest, filters):
88  status, output = util.getstatusoutput(cctest +  ' --list')
89  if status != 0: util.abort('Failed to list all tests')
90
91  available_tests = output.split()
92  if filters:
93    filters = map(re.compile, filters)
94    def is_selected(test_name):
95      for e in filters:
96        if e.search(test_name):
97          return True
98      return False
99
100    return filter(is_selected, available_tests)
101  else:
102    return available_tests
103
104
105# A class representing a cctest.
106class CCtest:
107  cctest = None
108
109  def __init__(self, name, options = None):
110    self.name = name
111    self.options = options
112    self.process = None
113    self.stdout = None
114    self.stderr = None
115
116  def Command(self):
117    command = '%s %s' % (CCtest.cctest, self.name)
118    if self.options is not None:
119      command = '%s %s' % (commnad, ' '.join(options))
120
121    return command
122
123  # Run the test.
124  # Use a thread to be able to control the test.
125  def Run(self, arguments):
126    command = [CCtest.cctest, self.name]
127    if self.options is not None:
128      command += self.options
129
130    def execute():
131      self.process = subprocess.Popen(command,
132                                      stdout=subprocess.PIPE,
133                                      stderr=subprocess.PIPE)
134      self.stdout, self.stderr = self.process.communicate()
135
136    thread = threading.Thread(target=execute)
137    retcode = -1
138    # Append spaces to hide the previous test name if longer.
139    sys.stdout.write('  ' + self.name + ' ' * 20)
140    sys.stdout.flush()
141    # Start the test with a timeout.
142    thread.start()
143    thread.join(arguments.timeout)
144    if thread.is_alive():
145      # Too slow! Terminate.
146      PrintError('### TIMEOUT %s\nCOMMAND:\n%s' % (self.name, self.Command()))
147      # If timeout was too small the thread may not have run and self.process
148      # is still None. Therefore check.
149      if (self.process):
150        self.process.terminate()
151      # Allow 1 second to terminate. Else, exterminate!
152      thread.join(1)
153      if thread.is_alive():
154        thread.kill()
155        thread.join()
156      # retcode is already set for failure.
157    else:
158      # Check the return status of the test.
159      retcode = self.process.poll()
160      if retcode != 0:
161        PrintError('### FAILED %s\nSTDERR:\n%s\nSTDOUT:\n%s\nCOMMAND:\n%s'
162                   % (self.name, self.stderr.decode(), self.stdout.decode(),
163                      self.Command()))
164
165    return retcode
166
167
168# Run all tests in the list 'tests'.
169def RunTests(cctest, tests, arguments):
170  CCtest.cctest = cctest
171  card = len(tests)
172  passed = 0
173  failed = 0
174
175  if card == 0:
176    print('No test to run')
177    return 0
178
179  # When the simulator is on the tests are ran twice: with and without the
180  # debugger.
181  if arguments.simulator:
182    card *= 2
183
184  print('Running %d tests... (timeout = %ds)' % (card, arguments.timeout))
185  start_time = time.time()
186
187  # Initialize the progress indicator.
188  UpdateProgress(start_time, 0, 0, card)
189  for e in tests:
190    variants = [CCtest(e)]
191    if arguments.simulator: variants.append(CCtest(e, ['--debugger']))
192    for v in variants:
193      retcode = v.Run(arguments)
194      # Update the counters and progress indicator.
195      if retcode == 0:
196        passed += 1
197      else:
198        failed += 1
199    UpdateProgress(start_time, passed, failed, card)
200
201  return failed
202
203
204if __name__ == '__main__':
205  original_dir = os.path.abspath('.')
206  # $ROOT/tools/test.py
207  root_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
208  os.chdir(root_dir)
209
210  # Parse the arguments and build the executable.
211  args = BuildOptions()
212  if not args.nobuild:
213    BuildRequiredObjects(args)
214
215  # The test binary.
216  cctest = os.path.join(root_dir, 'cctest')
217  if args.simulator == 'on':
218    cctest += '_sim'
219  if args.mode == 'debug':
220    cctest += '_g'
221  elif args.mode == 'coverage':
222    cctest += '_gcov'
223
224  # List available tests.
225  tests = ListTests(cctest, args.name_filters)
226
227  # Delete coverage data files.
228  if args.mode == 'coverage':
229    status, output = util.getstatusoutput('find obj/coverage -name "*.gcda" -exec rm {} \;')
230
231  # Run the tests.
232  status = RunTests(cctest, tests, args)
233  sys.stdout.write('\n')
234
235  # Print coverage information.
236  if args.mode == 'coverage':
237    cmd = 'tggcov -R summary_all,untested_functions_per_file obj/coverage/src/aarch64'
238    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
239                         stderr=subprocess.PIPE)
240    stdout, stderr = p.communicate()
241    print(stdout)
242
243  # Restore original directory.
244  os.chdir(original_dir)
245
246  sys.exit(status)
247
248