main.py revision abb9de5257375dbf10c87bdbf40ecafa777a0881 
1#!/usr/bin/env python
2
3"""
4lit - LLVM Integrated Tester.
5
6See lit.pod for more information.
7"""
8
9from __future__ import absolute_import
10import math, os, platform, random, re, sys, time, threading, traceback
11
12import lit.ProgressBar
13import lit.LitConfig
14import lit.Test
15import lit.Util
16
17import lit.discovery
18
19class TestingProgressDisplay:
20    def __init__(self, opts, numTests, progressBar=None):
21        self.opts = opts
22        self.numTests = numTests
23        self.current = None
24        self.lock = threading.Lock()
25        self.progressBar = progressBar
26        self.completed = 0
27
28    def update(self, test):
29        # Avoid locking overhead in quiet mode
30        if self.opts.quiet and not test.result.isFailure:
31            self.completed += 1
32            return
33
34        # Output lock.
35        self.lock.acquire()
36        try:
37            self.handleUpdate(test)
38        finally:
39            self.lock.release()
40
41    def finish(self):
42        if self.progressBar:
43            self.progressBar.clear()
44        elif self.opts.quiet:
45            pass
46        elif self.opts.succinct:
47            sys.stdout.write('\n')
48
49    def handleUpdate(self, test):
50        self.completed += 1
51        if self.progressBar:
52            self.progressBar.update(float(self.completed)/self.numTests,
53                                    test.getFullName())
54
55        if self.opts.succinct and not test.result.isFailure:
56            return
57
58        if self.progressBar:
59            self.progressBar.clear()
60
61        print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
62                                     self.completed, self.numTests))
63
64        if test.result.isFailure and self.opts.showOutput:
65            print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
66                                              '*'*20))
67            print(test.output)
68            print("*" * 20)
69
70        sys.stdout.flush()
71
72class TestProvider:
73    def __init__(self, tests, maxTime):
74        self.maxTime = maxTime
75        self.iter = iter(tests)
76        self.lock = threading.Lock()
77        self.startTime = time.time()
78        self.canceled = False
79
80    def cancel(self):
81        self.lock.acquire()
82        self.canceled = True
83        self.lock.release()
84
85    def get(self):
86        # Check if we have run out of time.
87        if self.maxTime is not None:
88            if time.time() - self.startTime > self.maxTime:
89                return None
90
91        # Otherwise take the next test.
92        self.lock.acquire()
93        if self.canceled:
94          self.lock.release()
95          return None
96        for item in self.iter:
97            break
98        else:
99            item = None
100        self.lock.release()
101        return item
102
103class Tester(threading.Thread):
104    def __init__(self, litConfig, provider, display):
105        threading.Thread.__init__(self)
106        self.litConfig = litConfig
107        self.provider = provider
108        self.display = display
109
110    def run(self):
111        while 1:
112            item = self.provider.get()
113            if item is None:
114                break
115            self.runTest(item)
116
117    def runTest(self, test):
118        result = None
119        startTime = time.time()
120        try:
121            result, output = test.config.test_format.execute(test,
122                                                             self.litConfig)
123        except KeyboardInterrupt:
124            # This is a sad hack. Unfortunately subprocess goes
125            # bonkers with ctrl-c and we start forking merrily.
126            print('\nCtrl-C detected, goodbye.')
127            os.kill(0,9)
128        except:
129            if self.litConfig.debug:
130                raise
131            result = lit.Test.UNRESOLVED
132            output = 'Exception during script execution:\n'
133            output += traceback.format_exc()
134            output += '\n'
135        elapsed = time.time() - startTime
136
137        test.setResult(result, output, elapsed)
138        self.display.update(test)
139
140def runTests(numThreads, litConfig, provider, display):
141    # If only using one testing thread, don't use threads at all; this lets us
142    # profile, among other things.
143    if numThreads == 1:
144        t = Tester(litConfig, provider, display)
145        t.run()
146        return
147
148    # Otherwise spin up the testing threads and wait for them to finish.
149    testers = [Tester(litConfig, provider, display)
150               for i in range(numThreads)]
151    for t in testers:
152        t.start()
153    try:
154        for t in testers:
155            t.join()
156    except KeyboardInterrupt:
157        sys.exit(2)
158
159def main(builtinParameters = {}):
160    # Bump the GIL check interval, its more important to get any one thread to a
161    # blocking operation (hopefully exec) than to try and unblock other threads.
162    #
163    # FIXME: This is a hack.
164    sys.setcheckinterval(1000)
165
166    global options
167    from optparse import OptionParser, OptionGroup
168    parser = OptionParser("usage: %prog [options] {file-or-path}")
169
170    parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
171                      help="Number of testing threads",
172                      type=int, action="store", default=None)
173    parser.add_option("", "--config-prefix", dest="configPrefix",
174                      metavar="NAME", help="Prefix for 'lit' config files",
175                      action="store", default=None)
176    parser.add_option("", "--param", dest="userParameters",
177                      metavar="NAME=VAL",
178                      help="Add 'NAME' = 'VAL' to the user defined parameters",
179                      type=str, action="append", default=[])
180
181    group = OptionGroup(parser, "Output Format")
182    # FIXME: I find these names very confusing, although I like the
183    # functionality.
184    group.add_option("-q", "--quiet", dest="quiet",
185                     help="Suppress no error output",
186                     action="store_true", default=False)
187    group.add_option("-s", "--succinct", dest="succinct",
188                     help="Reduce amount of output",
189                     action="store_true", default=False)
190    group.add_option("-v", "--verbose", dest="showOutput",
191                     help="Show all test output",
192                     action="store_true", default=False)
193    group.add_option("", "--no-progress-bar", dest="useProgressBar",
194                     help="Do not use curses based progress bar",
195                     action="store_false", default=True)
196    parser.add_option_group(group)
197
198    group = OptionGroup(parser, "Test Execution")
199    group.add_option("", "--path", dest="path",
200                     help="Additional paths to add to testing environment",
201                     action="append", type=str, default=[])
202    group.add_option("", "--vg", dest="useValgrind",
203                     help="Run tests under valgrind",
204                     action="store_true", default=False)
205    group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
206                     help="Check for memory leaks under valgrind",
207                     action="store_true", default=False)
208    group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
209                     help="Specify an extra argument for valgrind",
210                     type=str, action="append", default=[])
211    group.add_option("", "--time-tests", dest="timeTests",
212                     help="Track elapsed wall time for each test",
213                     action="store_true", default=False)
214    group.add_option("", "--no-execute", dest="noExecute",
215                     help="Don't execute any tests (assume PASS)",
216                     action="store_true", default=False)
217    parser.add_option_group(group)
218
219    group = OptionGroup(parser, "Test Selection")
220    group.add_option("", "--max-tests", dest="maxTests", metavar="N",
221                     help="Maximum number of tests to run",
222                     action="store", type=int, default=None)
223    group.add_option("", "--max-time", dest="maxTime", metavar="N",
224                     help="Maximum time to spend testing (in seconds)",
225                     action="store", type=float, default=None)
226    group.add_option("", "--shuffle", dest="shuffle",
227                     help="Run tests in random order",
228                     action="store_true", default=False)
229    group.add_option("", "--filter", dest="filter", metavar="REGEX",
230                     help=("Only run tests with paths matching the given "
231                           "regular expression"),
232                     action="store", default=None)
233    parser.add_option_group(group)
234
235    group = OptionGroup(parser, "Debug and Experimental Options")
236    group.add_option("", "--debug", dest="debug",
237                      help="Enable debugging (for 'lit' development)",
238                      action="store_true", default=False)
239    group.add_option("", "--show-suites", dest="showSuites",
240                      help="Show discovered test suites",
241                      action="store_true", default=False)
242    group.add_option("", "--show-tests", dest="showTests",
243                      help="Show all discovered tests",
244                      action="store_true", default=False)
245    parser.add_option_group(group)
246
247    (opts, args) = parser.parse_args()
248
249    if not args:
250        parser.error('No inputs specified')
251
252    if opts.numThreads is None:
253# Python <2.5 has a race condition causing lit to always fail with numThreads>1
254# http://bugs.python.org/issue1731717
255# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
256# threads by default there.
257       if sys.hexversion >= 0x2050200:
258               opts.numThreads = lit.Util.detectCPUs()
259       else:
260               opts.numThreads = 1
261
262    inputs = args
263
264    # Create the user defined parameters.
265    userParams = dict(builtinParameters)
266    for entry in opts.userParameters:
267        if '=' not in entry:
268            name,val = entry,''
269        else:
270            name,val = entry.split('=', 1)
271        userParams[name] = val
272
273    # Create the global config object.
274    litConfig = lit.LitConfig.LitConfig(
275        progname = os.path.basename(sys.argv[0]),
276        path = opts.path,
277        quiet = opts.quiet,
278        useValgrind = opts.useValgrind,
279        valgrindLeakCheck = opts.valgrindLeakCheck,
280        valgrindArgs = opts.valgrindArgs,
281        noExecute = opts.noExecute,
282        debug = opts.debug,
283        isWindows = (platform.system()=='Windows'),
284        params = userParams,
285        config_prefix = opts.configPrefix)
286
287    tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
288
289    if opts.showSuites or opts.showTests:
290        # Aggregate the tests by suite.
291        suitesAndTests = {}
292        for t in tests:
293            if t.suite not in suitesAndTests:
294                suitesAndTests[t.suite] = []
295            suitesAndTests[t.suite].append(t)
296        suitesAndTests = list(suitesAndTests.items())
297        suitesAndTests.sort(key = lambda item: item[0].name)
298
299        # Show the suites, if requested.
300        if opts.showSuites:
301            print('-- Test Suites --')
302            for ts,ts_tests in suitesAndTests:
303                print('  %s - %d tests' %(ts.name, len(ts_tests)))
304                print('    Source Root: %s' % ts.source_root)
305                print('    Exec Root  : %s' % ts.exec_root)
306
307        # Show the tests, if requested.
308        if opts.showTests:
309            print('-- Available Tests --')
310            for ts,ts_tests in suitesAndTests:
311                ts_tests.sort(key = lambda test: test.path_in_suite)
312                for test in ts_tests:
313                    print('  %s' % (test.getFullName(),))
314
315        # Exit.
316        sys.exit(0)
317
318    # Select and order the tests.
319    numTotalTests = len(tests)
320
321    # First, select based on the filter expression if given.
322    if opts.filter:
323        try:
324            rex = re.compile(opts.filter)
325        except:
326            parser.error("invalid regular expression for --filter: %r" % (
327                    opts.filter))
328        tests = [t for t in tests
329                 if rex.search(t.getFullName())]
330
331    # Then select the order.
332    if opts.shuffle:
333        random.shuffle(tests)
334    else:
335        tests.sort(key = lambda t: t.getFullName())
336
337    # Finally limit the number of tests, if desired.
338    if opts.maxTests is not None:
339        tests = tests[:opts.maxTests]
340
341    # Don't create more threads than tests.
342    opts.numThreads = min(len(tests), opts.numThreads)
343
344    extra = ''
345    if len(tests) != numTotalTests:
346        extra = ' of %d' % numTotalTests
347    header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
348                                                      opts.numThreads)
349
350    progressBar = None
351    if not opts.quiet:
352        if opts.succinct and opts.useProgressBar:
353            try:
354                tc = lit.ProgressBar.TerminalController()
355                progressBar = lit.ProgressBar.ProgressBar(tc, header)
356            except ValueError:
357                print(header)
358                progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
359        else:
360            print(header)
361
362    startTime = time.time()
363    display = TestingProgressDisplay(opts, len(tests), progressBar)
364    provider = TestProvider(tests, opts.maxTime)
365
366    try:
367      import win32api
368    except ImportError:
369      pass
370    else:
371      def console_ctrl_handler(type):
372        provider.cancel()
373        return True
374      win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
375
376    runTests(opts.numThreads, litConfig, provider, display)
377    display.finish()
378
379    if not opts.quiet:
380        print('Testing Time: %.2fs'%(time.time() - startTime))
381
382    # Update results for any tests which weren't run.
383    for t in tests:
384        if t.result is None:
385            t.setResult(lit.Test.UNRESOLVED, '', 0.0)
386
387    # List test results organized by kind.
388    hasFailures = False
389    byCode = {}
390    for t in tests:
391        if t.result not in byCode:
392            byCode[t.result] = []
393        byCode[t.result].append(t)
394        if t.result.isFailure:
395            hasFailures = True
396
397    # Print each test in any of the failing groups.
398    for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
399                       ('Failing Tests', lit.Test.FAIL),
400                       ('Unresolved Tests', lit.Test.UNRESOLVED)):
401        elts = byCode.get(code)
402        if not elts:
403            continue
404        print('*'*20)
405        print('%s (%d):' % (title, len(elts)))
406        for t in elts:
407            print('    %s' % t.getFullName())
408        sys.stdout.write('\n')
409
410    if opts.timeTests:
411        # Collate, in case we repeated tests.
412        times = {}
413        for t in tests:
414            key = t.getFullName()
415            times[key] = times.get(key, 0.) + t.elapsed
416
417        byTime = list(times.items())
418        byTime.sort(key = lambda item: item[1])
419        if byTime:
420            lit.Util.printHistogram(byTime, title='Tests')
421
422    for name,code in (('Expected Passes    ', lit.Test.PASS),
423                      ('Expected Failures  ', lit.Test.XFAIL),
424                      ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
425                      ('Unresolved Tests   ', lit.Test.UNRESOLVED),
426                      ('Unexpected Passes  ', lit.Test.XPASS),
427                      ('Unexpected Failures', lit.Test.FAIL),):
428        if opts.quiet and not code.isFailure:
429            continue
430        N = len(byCode.get(code,[]))
431        if N:
432            print('  %s: %d' % (name,N))
433
434    # If we encountered any additional errors, exit abnormally.
435    if litConfig.numErrors:
436        sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
437        sys.exit(2)
438
439    # Warn about warnings.
440    if litConfig.numWarnings:
441        sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
442
443    if hasFailures:
444        sys.exit(1)
445    sys.exit(0)
446
447if __name__=='__main__':
448    main()
449