1d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar#!/usr/bin/env python 2d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 3d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar""" 4d24f1f342a4637756297117ead63e2e966999126Daniel Dunbarlit - LLVM Integrated Tester. 5d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 6d24f1f342a4637756297117ead63e2e966999126Daniel DunbarSee lit.pod for more information. 7d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar""" 8d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 9b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbarfrom __future__ import absolute_import 1007f0f16bfd58c3e44babae65ccd88b9621d15f48Daniel Dunbarimport math, os, platform, random, re, sys, time 11d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 12b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbarimport lit.ProgressBar 13b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbarimport lit.LitConfig 14b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbarimport lit.Test 155b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbarimport lit.run 16128ce319ec47c46dc7da16aa3a75185899878745Daniel Dunbarimport lit.util 173d4a973ee4a2a134dd3df34a64a0a16256a5306cDaniel Dunbarimport lit.discovery 18d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 1907f0f16bfd58c3e44babae65ccd88b9621d15f48Daniel Dunbarclass TestingProgressDisplay(object): 20d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar def __init__(self, opts, numTests, progressBar=None): 21d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.opts = opts 22d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.numTests = numTests 23d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.current = None 24d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.progressBar = progressBar 25d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.completed = 0 26d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 27d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar def finish(self): 28d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if self.progressBar: 29d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.progressBar.clear() 30d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar elif self.opts.quiet: 31d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar pass 32d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar elif self.opts.succinct: 33d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar sys.stdout.write('\n') 34d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 35b11b690d3f36d552091423293489f845090efb4fDaniel Dunbar def update(self, test): 36d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.completed += 1 3736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if self.opts.incremental: 3936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines update_incremental_cache(test) 4036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 41d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if self.progressBar: 42d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.progressBar.update(float(self.completed)/self.numTests, 43d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar test.getFullName()) 44d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 45b11b690d3f36d552091423293489f845090efb4fDaniel Dunbar if not test.result.code.isFailure and \ 46b11b690d3f36d552091423293489f845090efb4fDaniel Dunbar (self.opts.quiet or self.opts.succinct): 47d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar return 48d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 49d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if self.progressBar: 50d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar self.progressBar.clear() 51d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 52ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar # Show the test result line. 53ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar test_name = test.getFullName() 54ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar print('%s: %s (%d of %d)' % (test.result.code.name, test_name, 554f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar self.completed, self.numTests)) 56d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 57ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar # Show the test failure output, if requested. 58ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar if test.result.code.isFailure and self.opts.showOutput: 594f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(), 604f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar '*'*20)) 61ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar print(test.result.output) 624f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print("*" * 20) 63d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 64ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar # Report test metrics, if present. 65ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar if test.result.metrics: 66ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar print("%s TEST '%s' RESULTS %s" % ('*'*10, test.getFullName(), 67ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar '*'*10)) 68ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar items = sorted(test.result.metrics.items()) 69ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar for metric_name, value in items: 70ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar print('%s: %s ' % (metric_name, value.format())) 71ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar print("*" * 10) 72ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 73ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar # Ensure the output is flushed. 74d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar sys.stdout.flush() 75d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 762849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbardef write_test_results(run, lit_config, testing_time, output_path): 772849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar try: 782849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar import json 792849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar except ImportError: 802849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar lit_config.fatal('test output unsupported with Python 2.5') 812849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 822849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # Construct the data we will write. 832849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar data = {} 842849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # Encode the current lit version as a schema version. 852849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar data['__version__'] = lit.__versioninfo__ 862849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar data['elapsed'] = testing_time 872849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # FIXME: Record some information on the lit configuration used? 882849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # FIXME: Record information from the individual test suites? 892849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 902849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # Encode the tests. 912849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar data['tests'] = tests_data = [] 922849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar for test in run.tests: 932849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar test_data = { 942849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 'name' : test.getFullName(), 952849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 'code' : test.result.code.name, 962849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 'output' : test.result.output, 972849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 'elapsed' : test.result.elapsed } 982849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 992849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # Add test metrics, if present. 1002849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar if test.result.metrics: 1012849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar test_data['metrics'] = metrics_data = {} 1022849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar for key, value in test.result.metrics.items(): 1032849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar metrics_data[key] = value.todata() 1042849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 1052849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar tests_data.append(test_data) 1062849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 1072849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # Write the output. 1082849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar f = open(output_path, 'w') 1092849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar try: 1102849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar json.dump(data, f, indent=2, sort_keys=True) 1112849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar f.write('\n') 1122849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar finally: 1132849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar f.close() 1142849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 11536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesdef update_incremental_cache(test): 11636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if not test.result.code.isFailure: 11736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 11836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines fname = test.getFilePath() 11936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines os.utime(fname, None) 12036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 12136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesdef sort_by_incremental_cache(run): 12236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines def sortIndex(test): 12336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines fname = test.getFilePath() 12436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines try: 12536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return -os.path.getmtime(fname) 12636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines except: 12736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 0 12836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines run.tests.sort(key = lambda t: sortIndex(t)) 12936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 1303d4a973ee4a2a134dd3df34a64a0a16256a5306cDaniel Dunbardef main(builtinParameters = {}): 131a0d3bcfd8cf26f2849e0ea48419f31659dacb0bfDaniel Dunbar # Use processes by default on Unix platforms. 132a0d3bcfd8cf26f2849e0ea48419f31659dacb0bfDaniel Dunbar isWindows = platform.system() == 'Windows' 133a4e71dea487750cd13a5a5821df3acee32988bf4Alp Toker useProcessesIsDefault = not isWindows 134a0d3bcfd8cf26f2849e0ea48419f31659dacb0bfDaniel Dunbar 135d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar global options 136d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar from optparse import OptionParser, OptionGroup 137d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser = OptionParser("usage: %prog [options] {file-or-path}") 138d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 13936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines parser.add_option("", "--version", dest="show_version", 14036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines help="Show version and exit", 14136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines action="store_true", default=False) 142d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option("-j", "--threads", dest="numThreads", metavar="N", 143d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Number of testing threads", 144d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar type=int, action="store", default=None) 145d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option("", "--config-prefix", dest="configPrefix", 146d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar metavar="NAME", help="Prefix for 'lit' config files", 147d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store", default=None) 148d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option("", "--param", dest="userParameters", 149d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar metavar="NAME=VAL", 150d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Add 'NAME' = 'VAL' to the user defined parameters", 151d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar type=str, action="append", default=[]) 152d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 153d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group = OptionGroup(parser, "Output Format") 154d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # FIXME: I find these names very confusing, although I like the 155d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # functionality. 156d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("-q", "--quiet", dest="quiet", 157d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Suppress no error output", 158d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 159d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("-s", "--succinct", dest="succinct", 160d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Reduce amount of output", 161d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 162d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("-v", "--verbose", dest="showOutput", 163d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Show all test output", 164d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 1652849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar group.add_option("-o", "--output", dest="output_path", 1662849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar help="Write test results to the provided path", 1672849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar action="store", type=str, metavar="PATH") 168d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--no-progress-bar", dest="useProgressBar", 169d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Do not use curses based progress bar", 170d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_false", default=True) 171d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option_group(group) 172d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 173d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group = OptionGroup(parser, "Test Execution") 174d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--path", dest="path", 175d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Additional paths to add to testing environment", 176d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="append", type=str, default=[]) 177d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--vg", dest="useValgrind", 178d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Run tests under valgrind", 179d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 18032989deb9641cf3878686b5634311a7a125f8f02Jeffrey Yasskin group.add_option("", "--vg-leak", dest="valgrindLeakCheck", 18132989deb9641cf3878686b5634311a7a125f8f02Jeffrey Yasskin help="Check for memory leaks under valgrind", 18232989deb9641cf3878686b5634311a7a125f8f02Jeffrey Yasskin action="store_true", default=False) 183d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG", 184d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Specify an extra argument for valgrind", 185d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar type=str, action="append", default=[]) 186d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--time-tests", dest="timeTests", 187d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Track elapsed wall time for each test", 188d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 1898f4aab8c2fe095ce1286fc1bd0bdd9157ed00398NAKAMURA Takumi group.add_option("", "--no-execute", dest="noExecute", 1908f4aab8c2fe095ce1286fc1bd0bdd9157ed00398NAKAMURA Takumi help="Don't execute any tests (assume PASS)", 1918f4aab8c2fe095ce1286fc1bd0bdd9157ed00398NAKAMURA Takumi action="store_true", default=False) 192d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option_group(group) 193d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 194d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group = OptionGroup(parser, "Test Selection") 195d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--max-tests", dest="maxTests", metavar="N", 196d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Maximum number of tests to run", 197d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store", type=int, default=None) 198d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--max-time", dest="maxTime", metavar="N", 199d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Maximum time to spend testing (in seconds)", 200d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store", type=float, default=None) 201d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--shuffle", dest="shuffle", 202d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Run tests in random order", 203d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 20436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines group.add_option("-i", "--incremental", dest="incremental", 20536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines help="Run modified and failing tests first (updates " 20636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "mtimes)", 20736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines action="store_true", default=False) 208e94bd36dc715bc05aff35e04528dbc409a87d97cDaniel Dunbar group.add_option("", "--filter", dest="filter", metavar="REGEX", 2097434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar help=("Only run tests with paths matching the given " 2107434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar "regular expression"), 2117434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar action="store", default=None) 212d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option_group(group) 213d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 214d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group = OptionGroup(parser, "Debug and Experimental Options") 215d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--debug", dest="debug", 216d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Enable debugging (for 'lit' development)", 217d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 218d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar group.add_option("", "--show-suites", dest="showSuites", 219d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar help="Show discovered test suites", 220d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar action="store_true", default=False) 2213915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar group.add_option("", "--show-tests", dest="showTests", 2223915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar help="Show all discovered tests", 2233915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar action="store_true", default=False) 2244ac723b53f2eb69e604891853ca87d1e2b3ee788Daniel Dunbar group.add_option("", "--use-processes", dest="useProcesses", 2254ac723b53f2eb69e604891853ca87d1e2b3ee788Daniel Dunbar help="Run tests in parallel with processes (not threads)", 226a0d3bcfd8cf26f2849e0ea48419f31659dacb0bfDaniel Dunbar action="store_true", default=useProcessesIsDefault) 2274ac723b53f2eb69e604891853ca87d1e2b3ee788Daniel Dunbar group.add_option("", "--use-threads", dest="useProcesses", 2284ac723b53f2eb69e604891853ca87d1e2b3ee788Daniel Dunbar help="Run tests in parallel with threads (not processes)", 229d80961c91f5318cdbbd5150f54ddad04102b769aRafael Espindola action="store_false", default=useProcessesIsDefault) 230d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.add_option_group(group) 231d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 232d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar (opts, args) = parser.parse_args() 233d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 23436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if opts.show_version: 23536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print("lit %s" % (lit.__version__,)) 23636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return 23736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 238d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if not args: 239d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar parser.error('No inputs specified') 240d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 241d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if opts.numThreads is None: 2422f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin# Python <2.5 has a race condition causing lit to always fail with numThreads>1 2432f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin# http://bugs.python.org/issue1731717 2442f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple 2452f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin# threads by default there. 2462f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin if sys.hexversion >= 0x2050200: 247128ce319ec47c46dc7da16aa3a75185899878745Daniel Dunbar opts.numThreads = lit.util.detectCPUs() 2482f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin else: 2492f36fe438062801ca69785ffc8e476aa1bfa49e6Torok Edwin opts.numThreads = 1 250d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 251d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar inputs = args 252d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 253d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # Create the user defined parameters. 2546647033565456120ba24d83450bada5bb3096187Daniel Dunbar userParams = dict(builtinParameters) 255d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar for entry in opts.userParameters: 256d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if '=' not in entry: 257d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar name,val = entry,'' 258d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar else: 259d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar name,val = entry.split('=', 1) 260d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar userParams[name] = val 261d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 262d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # Create the global config object. 263b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar litConfig = lit.LitConfig.LitConfig( 264b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar progname = os.path.basename(sys.argv[0]), 265b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar path = opts.path, 266b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar quiet = opts.quiet, 267b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar useValgrind = opts.useValgrind, 268b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar valgrindLeakCheck = opts.valgrindLeakCheck, 269b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar valgrindArgs = opts.valgrindArgs, 2708f4aab8c2fe095ce1286fc1bd0bdd9157ed00398NAKAMURA Takumi noExecute = opts.noExecute, 271b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar debug = opts.debug, 272a0d3bcfd8cf26f2849e0ea48419f31659dacb0bfDaniel Dunbar isWindows = isWindows, 273b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar params = userParams, 274b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar config_prefix = opts.configPrefix) 275d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 2765b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar # Perform test discovery. 2775b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar run = lit.run.Run(litConfig, 2785b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar lit.discovery.find_tests_for_inputs(litConfig, inputs)) 279d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 2803915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar if opts.showSuites or opts.showTests: 2813915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar # Aggregate the tests by suite. 2823d4a973ee4a2a134dd3df34a64a0a16256a5306cDaniel Dunbar suitesAndTests = {} 2835b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar for t in run.tests: 2843d4a973ee4a2a134dd3df34a64a0a16256a5306cDaniel Dunbar if t.suite not in suitesAndTests: 2853d4a973ee4a2a134dd3df34a64a0a16256a5306cDaniel Dunbar suitesAndTests[t.suite] = [] 286d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar suitesAndTests[t.suite].append(t) 28732b7d4dfc1eefd1081c5bf9533c25616aee129bdDaniel Dunbar suitesAndTests = list(suitesAndTests.items()) 2880b714f768605c9f5d3c8caf2b5708e617b82f86fDaniel Dunbar suitesAndTests.sort(key = lambda item: item[0].name) 289d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 2903915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar # Show the suites, if requested. 2913915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar if opts.showSuites: 2924f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print('-- Test Suites --') 2933915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar for ts,ts_tests in suitesAndTests: 2944f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(' %s - %d tests' %(ts.name, len(ts_tests))) 2954f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(' Source Root: %s' % ts.source_root) 2964f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(' Exec Root : %s' % ts.exec_root) 2973915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar 2983915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar # Show the tests, if requested. 2993915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar if opts.showTests: 3004f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print('-- Available Tests --') 3013915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar for ts,ts_tests in suitesAndTests: 3023915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar ts_tests.sort(key = lambda test: test.path_in_suite) 3033915c080ccdd872893412c0a8eaa29f2cec05761Daniel Dunbar for test in ts_tests: 3044f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(' %s' % (test.getFullName(),)) 305abb9de5257375dbf10c87bdbf40ecafa777a0881Daniel Dunbar 306abb9de5257375dbf10c87bdbf40ecafa777a0881Daniel Dunbar # Exit. 307abb9de5257375dbf10c87bdbf40ecafa777a0881Daniel Dunbar sys.exit(0) 308abb9de5257375dbf10c87bdbf40ecafa777a0881Daniel Dunbar 309d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # Select and order the tests. 3105b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar numTotalTests = len(run.tests) 3117434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar 3127434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar # First, select based on the filter expression if given. 3137434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar if opts.filter: 3147434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar try: 3157434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar rex = re.compile(opts.filter) 3167434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar except: 3177434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar parser.error("invalid regular expression for --filter: %r" % ( 3187434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar opts.filter)) 3195b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar run.tests = [t for t in run.tests 3205b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar if rex.search(t.getFullName())] 3217434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar 3227434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar # Then select the order. 323d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if opts.shuffle: 3245b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar random.shuffle(run.tests) 32536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines elif opts.incremental: 32636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines sort_by_incremental_cache(run) 327d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar else: 3285b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar run.tests.sort(key = lambda t: t.getFullName()) 3297434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar 3307434c9a053789c04d73bb58df41ad6fdf6a84e6aDaniel Dunbar # Finally limit the number of tests, if desired. 331d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if opts.maxTests is not None: 3325b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar run.tests = run.tests[:opts.maxTests] 333d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 334c797f2e9c7121c7a8c859ad2a8ede8f8639e4ff5NAKAMURA Takumi # Don't create more threads than tests. 3355b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar opts.numThreads = min(len(run.tests), opts.numThreads) 336c797f2e9c7121c7a8c859ad2a8ede8f8639e4ff5NAKAMURA Takumi 337d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar extra = '' 3385b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar if len(run.tests) != numTotalTests: 339d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar extra = ' of %d' % numTotalTests 3405b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar header = '-- Testing: %d%s tests, %d threads --'%(len(run.tests), extra, 341d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar opts.numThreads) 342d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 343d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar progressBar = None 344d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if not opts.quiet: 345d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if opts.succinct and opts.useProgressBar: 346d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar try: 347b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar tc = lit.ProgressBar.TerminalController() 348b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar progressBar = lit.ProgressBar.ProgressBar(tc, header) 349d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar except ValueError: 3504f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(header) 351b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ') 352d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar else: 3534f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(header) 354d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 355d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar startTime = time.time() 3565b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar display = TestingProgressDisplay(opts, len(run.tests), progressBar) 357ba460864440b4dd192bd2809f913babe0cf07031Nico Rieck try: 3584ac723b53f2eb69e604891853ca87d1e2b3ee788Daniel Dunbar run.execute_tests(display, opts.numThreads, opts.maxTime, 3594ac723b53f2eb69e604891853ca87d1e2b3ee788Daniel Dunbar opts.useProcesses) 360ec8e0592544de7b99b87535352a26b0420c51820Daniel Dunbar except KeyboardInterrupt: 361ec8e0592544de7b99b87535352a26b0420c51820Daniel Dunbar sys.exit(2) 362d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar display.finish() 363d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 3642849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar testing_time = time.time() - startTime 365d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if not opts.quiet: 3662849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar print('Testing Time: %.2fs' % (testing_time,)) 3672849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 3682849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar # Write out the test data, if requested. 3692849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar if opts.output_path is not None: 3702849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar write_test_results(run, litConfig, testing_time, opts.output_path) 371d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 372d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # List test results organized by kind. 373d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar hasFailures = False 374d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar byCode = {} 3755b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar for test in run.tests: 376ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar if test.result.code not in byCode: 377ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar byCode[test.result.code] = [] 378ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar byCode[test.result.code].append(test) 379ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar if test.result.code.isFailure: 380d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar hasFailures = True 381d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 38291a62c34d8e40f3f5b51b07b2edb156f7bcbbffcDaniel Dunbar # Print each test in any of the failing groups. 383b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar for title,code in (('Unexpected Passing Tests', lit.Test.XPASS), 38491a62c34d8e40f3f5b51b07b2edb156f7bcbbffcDaniel Dunbar ('Failing Tests', lit.Test.FAIL), 38591a62c34d8e40f3f5b51b07b2edb156f7bcbbffcDaniel Dunbar ('Unresolved Tests', lit.Test.UNRESOLVED)): 386d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar elts = byCode.get(code) 387d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if not elts: 388d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar continue 3894f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print('*'*20) 3904f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print('%s (%d):' % (title, len(elts))) 391ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar for test in elts: 392ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar print(' %s' % test.getFullName()) 3930d038e3e8852bf4fde949136ca9c2815f64febd0Daniel Dunbar sys.stdout.write('\n') 394d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 3955b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar if opts.timeTests and run.tests: 3960dd41a99e31d098165571911fbb7d9d9a453595fDaniel Dunbar # Order by time. 397ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar test_times = [(test.getFullName(), test.result.elapsed) 3985b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar for test in run.tests] 3990dd41a99e31d098165571911fbb7d9d9a453595fDaniel Dunbar lit.util.printHistogram(test_times, title='Tests') 4005b2efc28fd193701d2d05780c7b8d8ee5bf75c31Daniel Dunbar 401b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar for name,code in (('Expected Passes ', lit.Test.PASS), 402b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar ('Expected Failures ', lit.Test.XFAIL), 403b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar ('Unsupported Tests ', lit.Test.UNSUPPORTED), 404b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar ('Unresolved Tests ', lit.Test.UNRESOLVED), 405b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar ('Unexpected Passes ', lit.Test.XPASS), 406b35a1733607b07f36c3617932b67debb375a15c5Daniel Dunbar ('Unexpected Failures', lit.Test.FAIL),): 407d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if opts.quiet and not code.isFailure: 408d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar continue 409d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar N = len(byCode.get(code,[])) 410d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if N: 4114f2469c40c9020af2cf39ec0af518b6caba0a588Daniel Dunbar print(' %s: %d' % (name,N)) 412d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 413d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # If we encountered any additional errors, exit abnormally. 414d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if litConfig.numErrors: 4150d038e3e8852bf4fde949136ca9c2815f64febd0Daniel Dunbar sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors) 416d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar sys.exit(2) 417d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 418d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar # Warn about warnings. 419d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if litConfig.numWarnings: 4200d038e3e8852bf4fde949136ca9c2815f64febd0Daniel Dunbar sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings) 421d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 422d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar if hasFailures: 423d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar sys.exit(1) 424d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar sys.exit(0) 425d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar 426d24f1f342a4637756297117ead63e2e966999126Daniel Dunbarif __name__=='__main__': 427d24f1f342a4637756297117ead63e2e966999126Daniel Dunbar main() 428