main.py revision 59c6b1073c48befe021de024a693bed94147120c 
1#!/usr/bin/env python
2
3"""
4lit - LLVM Integrated Tester.
5
6See lit.pod for more information.
7"""
8
9from __future__ import absolute_import
10import math, os, platform, random, re, sys, time, threading, traceback
11
12import lit.ProgressBar
13import lit.LitConfig
14import lit.Test
15import lit.Util
16
17import lit.discovery
18
19class TestingProgressDisplay:
20    def __init__(self, opts, numTests, progressBar=None):
21        self.opts = opts
22        self.numTests = numTests
23        self.current = None
24        self.lock = threading.Lock()
25        self.progressBar = progressBar
26        self.completed = 0
27
28    def update(self, test):
29        # Avoid locking overhead in quiet mode
30        if self.opts.quiet and not test.result.isFailure:
31            self.completed += 1
32            return
33
34        # Output lock.
35        self.lock.acquire()
36        try:
37            self.handleUpdate(test)
38        finally:
39            self.lock.release()
40
41    def finish(self):
42        if self.progressBar:
43            self.progressBar.clear()
44        elif self.opts.quiet:
45            pass
46        elif self.opts.succinct:
47            sys.stdout.write('\n')
48
49    def handleUpdate(self, test):
50        self.completed += 1
51        if self.progressBar:
52            self.progressBar.update(float(self.completed)/self.numTests,
53                                    test.getFullName())
54
55        if self.opts.succinct and not test.result.isFailure:
56            return
57
58        if self.progressBar:
59            self.progressBar.clear()
60
61        print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
62                                     self.completed, self.numTests))
63
64        if test.result.isFailure and self.opts.showOutput:
65            print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
66                                              '*'*20))
67            print(test.output)
68            print("*" * 20)
69
70        sys.stdout.flush()
71
72class TestProvider:
73    def __init__(self, tests, maxTime):
74        self.maxTime = maxTime
75        self.iter = iter(tests)
76        self.lock = threading.Lock()
77        self.startTime = time.time()
78        self.canceled = False
79
80    def cancel(self):
81        self.lock.acquire()
82        self.canceled = True
83        self.lock.release()
84
85    def get(self):
86        # Check if we have run out of time.
87        if self.maxTime is not None:
88            if time.time() - self.startTime > self.maxTime:
89                return None
90
91        # Otherwise take the next test.
92        self.lock.acquire()
93        if self.canceled:
94          self.lock.release()
95          return None
96        for item in self.iter:
97            break
98        else:
99            item = None
100        self.lock.release()
101        return item
102
103class Tester(threading.Thread):
104    def __init__(self, litConfig, provider, display):
105        threading.Thread.__init__(self)
106        self.litConfig = litConfig
107        self.provider = provider
108        self.display = display
109
110    def run(self):
111        while 1:
112            item = self.provider.get()
113            if item is None:
114                break
115            self.runTest(item)
116
117    def runTest(self, test):
118        result = None
119        startTime = time.time()
120        try:
121            result, output = test.config.test_format.execute(test,
122                                                             self.litConfig)
123        except KeyboardInterrupt:
124            # This is a sad hack. Unfortunately subprocess goes
125            # bonkers with ctrl-c and we start forking merrily.
126            print('\nCtrl-C detected, goodbye.')
127            os.kill(0,9)
128        except:
129            if self.litConfig.debug:
130                raise
131            result = lit.Test.UNRESOLVED
132            output = 'Exception during script execution:\n'
133            output += traceback.format_exc()
134            output += '\n'
135        elapsed = time.time() - startTime
136
137        test.setResult(result, output, elapsed)
138        self.display.update(test)
139
140def runTests(numThreads, litConfig, provider, display):
141    # If only using one testing thread, don't use threads at all; this lets us
142    # profile, among other things.
143    if numThreads == 1:
144        t = Tester(litConfig, provider, display)
145        t.run()
146        return
147
148    # Otherwise spin up the testing threads and wait for them to finish.
149    testers = [Tester(litConfig, provider, display)
150               for i in range(numThreads)]
151    for t in testers:
152        t.start()
153    try:
154        for t in testers:
155            t.join()
156    except KeyboardInterrupt:
157        sys.exit(2)
158
159def main(builtinParameters = {}):
160    # Bump the GIL check interval, its more important to get any one thread to a
161    # blocking operation (hopefully exec) than to try and unblock other threads.
162    #
163    # FIXME: This is a hack.
164    sys.setcheckinterval(1000)
165
166    global options
167    from optparse import OptionParser, OptionGroup
168    parser = OptionParser("usage: %prog [options] {file-or-path}")
169
170    parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
171                      help="Number of testing threads",
172                      type=int, action="store", default=None)
173    parser.add_option("", "--config-prefix", dest="configPrefix",
174                      metavar="NAME", help="Prefix for 'lit' config files",
175                      action="store", default=None)
176    parser.add_option("", "--param", dest="userParameters",
177                      metavar="NAME=VAL",
178                      help="Add 'NAME' = 'VAL' to the user defined parameters",
179                      type=str, action="append", default=[])
180
181    group = OptionGroup(parser, "Output Format")
182    # FIXME: I find these names very confusing, although I like the
183    # functionality.
184    group.add_option("-q", "--quiet", dest="quiet",
185                     help="Suppress no error output",
186                     action="store_true", default=False)
187    group.add_option("-s", "--succinct", dest="succinct",
188                     help="Reduce amount of output",
189                     action="store_true", default=False)
190    group.add_option("-v", "--verbose", dest="showOutput",
191                     help="Show all test output",
192                     action="store_true", default=False)
193    group.add_option("", "--no-progress-bar", dest="useProgressBar",
194                     help="Do not use curses based progress bar",
195                     action="store_false", default=True)
196    parser.add_option_group(group)
197
198    group = OptionGroup(parser, "Test Execution")
199    group.add_option("", "--path", dest="path",
200                     help="Additional paths to add to testing environment",
201                     action="append", type=str, default=[])
202    group.add_option("", "--vg", dest="useValgrind",
203                     help="Run tests under valgrind",
204                     action="store_true", default=False)
205    group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
206                     help="Check for memory leaks under valgrind",
207                     action="store_true", default=False)
208    group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
209                     help="Specify an extra argument for valgrind",
210                     type=str, action="append", default=[])
211    group.add_option("", "--time-tests", dest="timeTests",
212                     help="Track elapsed wall time for each test",
213                     action="store_true", default=False)
214    group.add_option("", "--no-execute", dest="noExecute",
215                     help="Don't execute any tests (assume PASS)",
216                     action="store_true", default=False)
217    parser.add_option_group(group)
218
219    group = OptionGroup(parser, "Test Selection")
220    group.add_option("", "--max-tests", dest="maxTests", metavar="N",
221                     help="Maximum number of tests to run",
222                     action="store", type=int, default=None)
223    group.add_option("", "--max-time", dest="maxTime", metavar="N",
224                     help="Maximum time to spend testing (in seconds)",
225                     action="store", type=float, default=None)
226    group.add_option("", "--shuffle", dest="shuffle",
227                     help="Run tests in random order",
228                     action="store_true", default=False)
229    group.add_option("", "--filter", dest="filter", metavar="REGEX",
230                     help=("Only run tests with paths matching the given "
231                           "regular expression"),
232                     action="store", default=None)
233    parser.add_option_group(group)
234
235    group = OptionGroup(parser, "Debug and Experimental Options")
236    group.add_option("", "--debug", dest="debug",
237                      help="Enable debugging (for 'lit' development)",
238                      action="store_true", default=False)
239    group.add_option("", "--show-suites", dest="showSuites",
240                      help="Show discovered test suites",
241                      action="store_true", default=False)
242    group.add_option("", "--show-tests", dest="showTests",
243                      help="Show all discovered tests",
244                      action="store_true", default=False)
245    parser.add_option_group(group)
246
247    (opts, args) = parser.parse_args()
248
249    if not args:
250        parser.error('No inputs specified')
251
252    if opts.numThreads is None:
253# Python <2.5 has a race condition causing lit to always fail with numThreads>1
254# http://bugs.python.org/issue1731717
255# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
256# threads by default there.
257       if sys.hexversion >= 0x2050200:
258               opts.numThreads = lit.Util.detectCPUs()
259       else:
260               opts.numThreads = 1
261
262    inputs = args
263
264    # Create the user defined parameters.
265    userParams = dict(builtinParameters)
266    for entry in opts.userParameters:
267        if '=' not in entry:
268            name,val = entry,''
269        else:
270            name,val = entry.split('=', 1)
271        userParams[name] = val
272
273    # Create the global config object.
274    litConfig = lit.LitConfig.LitConfig(
275        progname = os.path.basename(sys.argv[0]),
276        path = opts.path,
277        quiet = opts.quiet,
278        useValgrind = opts.useValgrind,
279        valgrindLeakCheck = opts.valgrindLeakCheck,
280        valgrindArgs = opts.valgrindArgs,
281        noExecute = opts.noExecute,
282        debug = opts.debug,
283        isWindows = (platform.system()=='Windows'),
284        params = userParams,
285        config_prefix = opts.configPrefix)
286
287    tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
288
289    if opts.showSuites or opts.showTests:
290        # Aggregate the tests by suite.
291        suitesAndTests = {}
292        for t in tests:
293            if t.suite not in suitesAndTests:
294                suitesAndTests[t.suite] = []
295            suitesAndTests[t.suite].append(t)
296        suitesAndTests = list(suitesAndTests.items())
297        suitesAndTests.sort(key = lambda item: item[0].name)
298
299        # Show the suites, if requested.
300        if opts.showSuites:
301            print('-- Test Suites --')
302            for ts,ts_tests in suitesAndTests:
303                print('  %s - %d tests' %(ts.name, len(ts_tests)))
304                print('    Source Root: %s' % ts.source_root)
305                print('    Exec Root  : %s' % ts.exec_root)
306
307        # Show the tests, if requested.
308        if opts.showTests:
309            print('-- Available Tests --')
310            for ts,ts_tests in suitesAndTests:
311                ts_tests.sort(key = lambda test: test.path_in_suite)
312                for test in ts_tests:
313                    print('  %s' % (test.getFullName(),))
314
315    # Select and order the tests.
316    numTotalTests = len(tests)
317
318    # First, select based on the filter expression if given.
319    if opts.filter:
320        try:
321            rex = re.compile(opts.filter)
322        except:
323            parser.error("invalid regular expression for --filter: %r" % (
324                    opts.filter))
325        tests = [t for t in tests
326                 if rex.search(t.getFullName())]
327
328    # Then select the order.
329    if opts.shuffle:
330        random.shuffle(tests)
331    else:
332        tests.sort(key = lambda t: t.getFullName())
333
334    # Finally limit the number of tests, if desired.
335    if opts.maxTests is not None:
336        tests = tests[:opts.maxTests]
337
338    # Don't create more threads than tests.
339    opts.numThreads = min(len(tests), opts.numThreads)
340
341    extra = ''
342    if len(tests) != numTotalTests:
343        extra = ' of %d' % numTotalTests
344    header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
345                                                      opts.numThreads)
346
347    progressBar = None
348    if not opts.quiet:
349        if opts.succinct and opts.useProgressBar:
350            try:
351                tc = lit.ProgressBar.TerminalController()
352                progressBar = lit.ProgressBar.ProgressBar(tc, header)
353            except ValueError:
354                print(header)
355                progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
356        else:
357            print(header)
358
359    startTime = time.time()
360    display = TestingProgressDisplay(opts, len(tests), progressBar)
361    provider = TestProvider(tests, opts.maxTime)
362
363    try:
364      import win32api
365    except ImportError:
366      pass
367    else:
368      def console_ctrl_handler(type):
369        provider.cancel()
370        return True
371      win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
372
373    runTests(opts.numThreads, litConfig, provider, display)
374    display.finish()
375
376    if not opts.quiet:
377        print('Testing Time: %.2fs'%(time.time() - startTime))
378
379    # Update results for any tests which weren't run.
380    for t in tests:
381        if t.result is None:
382            t.setResult(lit.Test.UNRESOLVED, '', 0.0)
383
384    # List test results organized by kind.
385    hasFailures = False
386    byCode = {}
387    for t in tests:
388        if t.result not in byCode:
389            byCode[t.result] = []
390        byCode[t.result].append(t)
391        if t.result.isFailure:
392            hasFailures = True
393
394    # Print each test in any of the failing groups.
395    for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
396                       ('Failing Tests', lit.Test.FAIL),
397                       ('Unresolved Tests', lit.Test.UNRESOLVED)):
398        elts = byCode.get(code)
399        if not elts:
400            continue
401        print('*'*20)
402        print('%s (%d):' % (title, len(elts)))
403        for t in elts:
404            print('    %s' % t.getFullName())
405        sys.stdout.write('\n')
406
407    if opts.timeTests:
408        # Collate, in case we repeated tests.
409        times = {}
410        for t in tests:
411            key = t.getFullName()
412            times[key] = times.get(key, 0.) + t.elapsed
413
414        byTime = list(times.items())
415        byTime.sort(key = lambda item: item[1])
416        if byTime:
417            lit.Util.printHistogram(byTime, title='Tests')
418
419    for name,code in (('Expected Passes    ', lit.Test.PASS),
420                      ('Expected Failures  ', lit.Test.XFAIL),
421                      ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
422                      ('Unresolved Tests   ', lit.Test.UNRESOLVED),
423                      ('Unexpected Passes  ', lit.Test.XPASS),
424                      ('Unexpected Failures', lit.Test.FAIL),):
425        if opts.quiet and not code.isFailure:
426            continue
427        N = len(byCode.get(code,[]))
428        if N:
429            print('  %s: %d' % (name,N))
430
431    # If we encountered any additional errors, exit abnormally.
432    if litConfig.numErrors:
433        sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
434        sys.exit(2)
435
436    # Warn about warnings.
437    if litConfig.numWarnings:
438        sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
439
440    if hasFailures:
441        sys.exit(1)
442    sys.exit(0)
443
444if __name__=='__main__':
445    main()
446