1#!/usr/bin/env python 2 3""" 4lit - LLVM Integrated Tester. 5 6See lit.pod for more information. 7""" 8 9from __future__ import absolute_import 10import math, os, platform, random, re, sys, time, threading, traceback 11 12import lit.ProgressBar 13import lit.LitConfig 14import lit.Test 15import lit.Util 16 17import lit.discovery 18 19class TestingProgressDisplay: 20 def __init__(self, opts, numTests, progressBar=None): 21 self.opts = opts 22 self.numTests = numTests 23 self.current = None 24 self.lock = threading.Lock() 25 self.progressBar = progressBar 26 self.completed = 0 27 28 def update(self, test): 29 # Avoid locking overhead in quiet mode 30 if self.opts.quiet and not test.result.isFailure: 31 self.completed += 1 32 return 33 34 # Output lock. 35 self.lock.acquire() 36 try: 37 self.handleUpdate(test) 38 finally: 39 self.lock.release() 40 41 def finish(self): 42 if self.progressBar: 43 self.progressBar.clear() 44 elif self.opts.quiet: 45 pass 46 elif self.opts.succinct: 47 sys.stdout.write('\n') 48 49 def handleUpdate(self, test): 50 self.completed += 1 51 if self.progressBar: 52 self.progressBar.update(float(self.completed)/self.numTests, 53 test.getFullName()) 54 55 if self.opts.succinct and not test.result.isFailure: 56 return 57 58 if self.progressBar: 59 self.progressBar.clear() 60 61 print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(), 62 self.completed, self.numTests)) 63 64 if test.result.isFailure and self.opts.showOutput: 65 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(), 66 '*'*20)) 67 print(test.output) 68 print("*" * 20) 69 70 sys.stdout.flush() 71 72class TestProvider: 73 def __init__(self, tests, maxTime): 74 self.maxTime = maxTime 75 self.iter = iter(tests) 76 self.lock = threading.Lock() 77 self.startTime = time.time() 78 self.canceled = False 79 80 def cancel(self): 81 self.lock.acquire() 82 self.canceled = True 83 self.lock.release() 84 85 def get(self): 86 # Check if we have run out of time. 87 if self.maxTime is not None: 88 if time.time() - self.startTime > self.maxTime: 89 return None 90 91 # Otherwise take the next test. 92 self.lock.acquire() 93 if self.canceled: 94 self.lock.release() 95 return None 96 for item in self.iter: 97 break 98 else: 99 item = None 100 self.lock.release() 101 return item 102 103class Tester(threading.Thread): 104 def __init__(self, litConfig, provider, display): 105 threading.Thread.__init__(self) 106 self.litConfig = litConfig 107 self.provider = provider 108 self.display = display 109 110 def run(self): 111 while 1: 112 item = self.provider.get() 113 if item is None: 114 break 115 self.runTest(item) 116 117 def runTest(self, test): 118 result = None 119 startTime = time.time() 120 try: 121 result, output = test.config.test_format.execute(test, 122 self.litConfig) 123 except KeyboardInterrupt: 124 # This is a sad hack. Unfortunately subprocess goes 125 # bonkers with ctrl-c and we start forking merrily. 126 print('\nCtrl-C detected, goodbye.') 127 os.kill(0,9) 128 except: 129 if self.litConfig.debug: 130 raise 131 result = lit.Test.UNRESOLVED 132 output = 'Exception during script execution:\n' 133 output += traceback.format_exc() 134 output += '\n' 135 elapsed = time.time() - startTime 136 137 test.setResult(result, output, elapsed) 138 self.display.update(test) 139 140def runTests(numThreads, litConfig, provider, display): 141 # If only using one testing thread, don't use threads at all; this lets us 142 # profile, among other things. 143 if numThreads == 1: 144 t = Tester(litConfig, provider, display) 145 t.run() 146 return 147 148 # Otherwise spin up the testing threads and wait for them to finish. 149 testers = [Tester(litConfig, provider, display) 150 for i in range(numThreads)] 151 for t in testers: 152 t.start() 153 try: 154 for t in testers: 155 t.join() 156 except KeyboardInterrupt: 157 sys.exit(2) 158 159def main(builtinParameters = {}): 160 # Bump the GIL check interval, its more important to get any one thread to a 161 # blocking operation (hopefully exec) than to try and unblock other threads. 162 # 163 # FIXME: This is a hack. 164 import sys 165 sys.setcheckinterval(1000) 166 167 global options 168 from optparse import OptionParser, OptionGroup 169 parser = OptionParser("usage: %prog [options] {file-or-path}") 170 171 parser.add_option("-j", "--threads", dest="numThreads", metavar="N", 172 help="Number of testing threads", 173 type=int, action="store", default=None) 174 parser.add_option("", "--config-prefix", dest="configPrefix", 175 metavar="NAME", help="Prefix for 'lit' config files", 176 action="store", default=None) 177 parser.add_option("", "--param", dest="userParameters", 178 metavar="NAME=VAL", 179 help="Add 'NAME' = 'VAL' to the user defined parameters", 180 type=str, action="append", default=[]) 181 182 group = OptionGroup(parser, "Output Format") 183 # FIXME: I find these names very confusing, although I like the 184 # functionality. 185 group.add_option("-q", "--quiet", dest="quiet", 186 help="Suppress no error output", 187 action="store_true", default=False) 188 group.add_option("-s", "--succinct", dest="succinct", 189 help="Reduce amount of output", 190 action="store_true", default=False) 191 group.add_option("-v", "--verbose", dest="showOutput", 192 help="Show all test output", 193 action="store_true", default=False) 194 group.add_option("", "--no-progress-bar", dest="useProgressBar", 195 help="Do not use curses based progress bar", 196 action="store_false", default=True) 197 parser.add_option_group(group) 198 199 group = OptionGroup(parser, "Test Execution") 200 group.add_option("", "--path", dest="path", 201 help="Additional paths to add to testing environment", 202 action="append", type=str, default=[]) 203 group.add_option("", "--vg", dest="useValgrind", 204 help="Run tests under valgrind", 205 action="store_true", default=False) 206 group.add_option("", "--vg-leak", dest="valgrindLeakCheck", 207 help="Check for memory leaks under valgrind", 208 action="store_true", default=False) 209 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG", 210 help="Specify an extra argument for valgrind", 211 type=str, action="append", default=[]) 212 group.add_option("", "--time-tests", dest="timeTests", 213 help="Track elapsed wall time for each test", 214 action="store_true", default=False) 215 group.add_option("", "--no-execute", dest="noExecute", 216 help="Don't execute any tests (assume PASS)", 217 action="store_true", default=False) 218 parser.add_option_group(group) 219 220 group = OptionGroup(parser, "Test Selection") 221 group.add_option("", "--max-tests", dest="maxTests", metavar="N", 222 help="Maximum number of tests to run", 223 action="store", type=int, default=None) 224 group.add_option("", "--max-time", dest="maxTime", metavar="N", 225 help="Maximum time to spend testing (in seconds)", 226 action="store", type=float, default=None) 227 group.add_option("", "--shuffle", dest="shuffle", 228 help="Run tests in random order", 229 action="store_true", default=False) 230 group.add_option("", "--filter", dest="filter", metavar="REGEX", 231 help=("Only run tests with paths matching the given " 232 "regular expression"), 233 action="store", default=None) 234 parser.add_option_group(group) 235 236 group = OptionGroup(parser, "Debug and Experimental Options") 237 group.add_option("", "--debug", dest="debug", 238 help="Enable debugging (for 'lit' development)", 239 action="store_true", default=False) 240 group.add_option("", "--show-suites", dest="showSuites", 241 help="Show discovered test suites", 242 action="store_true", default=False) 243 group.add_option("", "--show-tests", dest="showTests", 244 help="Show all discovered tests", 245 action="store_true", default=False) 246 group.add_option("", "--repeat", dest="repeatTests", metavar="N", 247 help="Repeat tests N times (for timing)", 248 action="store", default=None, type=int) 249 parser.add_option_group(group) 250 251 (opts, args) = parser.parse_args() 252 253 if not args: 254 parser.error('No inputs specified') 255 256 if opts.numThreads is None: 257# Python <2.5 has a race condition causing lit to always fail with numThreads>1 258# http://bugs.python.org/issue1731717 259# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple 260# threads by default there. 261 if sys.hexversion >= 0x2050200: 262 opts.numThreads = lit.Util.detectCPUs() 263 else: 264 opts.numThreads = 1 265 266 inputs = args 267 268 # Create the user defined parameters. 269 userParams = dict(builtinParameters) 270 for entry in opts.userParameters: 271 if '=' not in entry: 272 name,val = entry,'' 273 else: 274 name,val = entry.split('=', 1) 275 userParams[name] = val 276 277 # Create the global config object. 278 litConfig = lit.LitConfig.LitConfig( 279 progname = os.path.basename(sys.argv[0]), 280 path = opts.path, 281 quiet = opts.quiet, 282 useValgrind = opts.useValgrind, 283 valgrindLeakCheck = opts.valgrindLeakCheck, 284 valgrindArgs = opts.valgrindArgs, 285 noExecute = opts.noExecute, 286 debug = opts.debug, 287 isWindows = (platform.system()=='Windows'), 288 params = userParams, 289 config_prefix = opts.configPrefix) 290 291 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs) 292 293 if opts.showSuites or opts.showTests: 294 # Aggregate the tests by suite. 295 suitesAndTests = {} 296 for t in tests: 297 if t.suite not in suitesAndTests: 298 suitesAndTests[t.suite] = [] 299 suitesAndTests[t.suite].append(t) 300 suitesAndTests = suitesAndTests.items() 301 suitesAndTests.sort(key = lambda item: item[0].name) 302 303 # Show the suites, if requested. 304 if opts.showSuites: 305 print('-- Test Suites --') 306 for ts,ts_tests in suitesAndTests: 307 print(' %s - %d tests' %(ts.name, len(ts_tests))) 308 print(' Source Root: %s' % ts.source_root) 309 print(' Exec Root : %s' % ts.exec_root) 310 311 # Show the tests, if requested. 312 if opts.showTests: 313 print('-- Available Tests --') 314 for ts,ts_tests in suitesAndTests: 315 ts_tests.sort(key = lambda test: test.path_in_suite) 316 for test in ts_tests: 317 print(' %s' % (test.getFullName(),)) 318 319 # Select and order the tests. 320 numTotalTests = len(tests) 321 322 # First, select based on the filter expression if given. 323 if opts.filter: 324 try: 325 rex = re.compile(opts.filter) 326 except: 327 parser.error("invalid regular expression for --filter: %r" % ( 328 opts.filter)) 329 tests = [t for t in tests 330 if rex.search(t.getFullName())] 331 332 # Then select the order. 333 if opts.shuffle: 334 random.shuffle(tests) 335 else: 336 tests.sort(key = lambda t: t.getFullName()) 337 338 # Finally limit the number of tests, if desired. 339 if opts.maxTests is not None: 340 tests = tests[:opts.maxTests] 341 342 # Don't create more threads than tests. 343 opts.numThreads = min(len(tests), opts.numThreads) 344 345 extra = '' 346 if len(tests) != numTotalTests: 347 extra = ' of %d' % numTotalTests 348 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra, 349 opts.numThreads) 350 351 if opts.repeatTests: 352 tests = [t.copyWithIndex(i) 353 for t in tests 354 for i in range(opts.repeatTests)] 355 356 progressBar = None 357 if not opts.quiet: 358 if opts.succinct and opts.useProgressBar: 359 try: 360 tc = lit.ProgressBar.TerminalController() 361 progressBar = lit.ProgressBar.ProgressBar(tc, header) 362 except ValueError: 363 print(header) 364 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ') 365 else: 366 print(header) 367 368 startTime = time.time() 369 display = TestingProgressDisplay(opts, len(tests), progressBar) 370 provider = TestProvider(tests, opts.maxTime) 371 372 try: 373 import win32api 374 except ImportError: 375 pass 376 else: 377 def console_ctrl_handler(type): 378 provider.cancel() 379 return True 380 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True) 381 382 runTests(opts.numThreads, litConfig, provider, display) 383 display.finish() 384 385 if not opts.quiet: 386 print('Testing Time: %.2fs'%(time.time() - startTime)) 387 388 # Update results for any tests which weren't run. 389 for t in tests: 390 if t.result is None: 391 t.setResult(lit.Test.UNRESOLVED, '', 0.0) 392 393 # List test results organized by kind. 394 hasFailures = False 395 byCode = {} 396 for t in tests: 397 if t.result not in byCode: 398 byCode[t.result] = [] 399 byCode[t.result].append(t) 400 if t.result.isFailure: 401 hasFailures = True 402 403 # FIXME: Show unresolved and (optionally) unsupported tests. 404 for title,code in (('Unexpected Passing Tests', lit.Test.XPASS), 405 ('Failing Tests', lit.Test.FAIL)): 406 elts = byCode.get(code) 407 if not elts: 408 continue 409 print('*'*20) 410 print('%s (%d):' % (title, len(elts))) 411 for t in elts: 412 print(' %s' % t.getFullName()) 413 sys.stdout.write('\n') 414 415 if opts.timeTests: 416 # Collate, in case we repeated tests. 417 times = {} 418 for t in tests: 419 key = t.getFullName() 420 times[key] = times.get(key, 0.) + t.elapsed 421 422 byTime = list(times.items()) 423 byTime.sort(key = lambda item: item[1]) 424 if byTime: 425 lit.Util.printHistogram(byTime, title='Tests') 426 427 for name,code in (('Expected Passes ', lit.Test.PASS), 428 ('Expected Failures ', lit.Test.XFAIL), 429 ('Unsupported Tests ', lit.Test.UNSUPPORTED), 430 ('Unresolved Tests ', lit.Test.UNRESOLVED), 431 ('Unexpected Passes ', lit.Test.XPASS), 432 ('Unexpected Failures', lit.Test.FAIL),): 433 if opts.quiet and not code.isFailure: 434 continue 435 N = len(byCode.get(code,[])) 436 if N: 437 print(' %s: %d' % (name,N)) 438 439 # If we encountered any additional errors, exit abnormally. 440 if litConfig.numErrors: 441 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors) 442 sys.exit(2) 443 444 # Warn about warnings. 445 if litConfig.numWarnings: 446 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings) 447 448 if hasFailures: 449 sys.exit(1) 450 sys.exit(0) 451 452if __name__=='__main__': 453 main() 454