1#!/usr/bin/env python
2
3"""
4lit - LLVM Integrated Tester.
5
6See lit.pod for more information.
7"""
8
9from __future__ import absolute_import
10import math, os, platform, random, re, sys, time, threading, traceback
11
12import lit.ProgressBar
13import lit.LitConfig
14import lit.Test
15import lit.Util
16
17import lit.discovery
18
19class TestingProgressDisplay:
20    def __init__(self, opts, numTests, progressBar=None):
21        self.opts = opts
22        self.numTests = numTests
23        self.current = None
24        self.lock = threading.Lock()
25        self.progressBar = progressBar
26        self.completed = 0
27
28    def update(self, test):
29        # Avoid locking overhead in quiet mode
30        if self.opts.quiet and not test.result.isFailure:
31            self.completed += 1
32            return
33
34        # Output lock.
35        self.lock.acquire()
36        try:
37            self.handleUpdate(test)
38        finally:
39            self.lock.release()
40
41    def finish(self):
42        if self.progressBar:
43            self.progressBar.clear()
44        elif self.opts.quiet:
45            pass
46        elif self.opts.succinct:
47            sys.stdout.write('\n')
48
49    def handleUpdate(self, test):
50        self.completed += 1
51        if self.progressBar:
52            self.progressBar.update(float(self.completed)/self.numTests,
53                                    test.getFullName())
54
55        if self.opts.succinct and not test.result.isFailure:
56            return
57
58        if self.progressBar:
59            self.progressBar.clear()
60
61        print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
62                                     self.completed, self.numTests))
63
64        if test.result.isFailure and self.opts.showOutput:
65            print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
66                                              '*'*20))
67            print(test.output)
68            print("*" * 20)
69
70        sys.stdout.flush()
71
72class TestProvider:
73    def __init__(self, tests, maxTime):
74        self.maxTime = maxTime
75        self.iter = iter(tests)
76        self.lock = threading.Lock()
77        self.startTime = time.time()
78        self.canceled = False
79
80    def cancel(self):
81        self.lock.acquire()
82        self.canceled = True
83        self.lock.release()
84
85    def get(self):
86        # Check if we have run out of time.
87        if self.maxTime is not None:
88            if time.time() - self.startTime > self.maxTime:
89                return None
90
91        # Otherwise take the next test.
92        self.lock.acquire()
93        if self.canceled:
94          self.lock.release()
95          return None
96        for item in self.iter:
97            break
98        else:
99            item = None
100        self.lock.release()
101        return item
102
103class Tester(threading.Thread):
104    def __init__(self, litConfig, provider, display):
105        threading.Thread.__init__(self)
106        self.litConfig = litConfig
107        self.provider = provider
108        self.display = display
109
110    def run(self):
111        while 1:
112            item = self.provider.get()
113            if item is None:
114                break
115            self.runTest(item)
116
117    def runTest(self, test):
118        result = None
119        startTime = time.time()
120        try:
121            result, output = test.config.test_format.execute(test,
122                                                             self.litConfig)
123        except KeyboardInterrupt:
124            # This is a sad hack. Unfortunately subprocess goes
125            # bonkers with ctrl-c and we start forking merrily.
126            print('\nCtrl-C detected, goodbye.')
127            os.kill(0,9)
128        except:
129            if self.litConfig.debug:
130                raise
131            result = lit.Test.UNRESOLVED
132            output = 'Exception during script execution:\n'
133            output += traceback.format_exc()
134            output += '\n'
135        elapsed = time.time() - startTime
136
137        test.setResult(result, output, elapsed)
138        self.display.update(test)
139
140def runTests(numThreads, litConfig, provider, display):
141    # If only using one testing thread, don't use threads at all; this lets us
142    # profile, among other things.
143    if numThreads == 1:
144        t = Tester(litConfig, provider, display)
145        t.run()
146        return
147
148    # Otherwise spin up the testing threads and wait for them to finish.
149    testers = [Tester(litConfig, provider, display)
150               for i in range(numThreads)]
151    for t in testers:
152        t.start()
153    try:
154        for t in testers:
155            t.join()
156    except KeyboardInterrupt:
157        sys.exit(2)
158
159def main(builtinParameters = {}):
160    # Bump the GIL check interval, its more important to get any one thread to a
161    # blocking operation (hopefully exec) than to try and unblock other threads.
162    #
163    # FIXME: This is a hack.
164    import sys
165    sys.setcheckinterval(1000)
166
167    global options
168    from optparse import OptionParser, OptionGroup
169    parser = OptionParser("usage: %prog [options] {file-or-path}")
170
171    parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
172                      help="Number of testing threads",
173                      type=int, action="store", default=None)
174    parser.add_option("", "--config-prefix", dest="configPrefix",
175                      metavar="NAME", help="Prefix for 'lit' config files",
176                      action="store", default=None)
177    parser.add_option("", "--param", dest="userParameters",
178                      metavar="NAME=VAL",
179                      help="Add 'NAME' = 'VAL' to the user defined parameters",
180                      type=str, action="append", default=[])
181
182    group = OptionGroup(parser, "Output Format")
183    # FIXME: I find these names very confusing, although I like the
184    # functionality.
185    group.add_option("-q", "--quiet", dest="quiet",
186                     help="Suppress no error output",
187                     action="store_true", default=False)
188    group.add_option("-s", "--succinct", dest="succinct",
189                     help="Reduce amount of output",
190                     action="store_true", default=False)
191    group.add_option("-v", "--verbose", dest="showOutput",
192                     help="Show all test output",
193                     action="store_true", default=False)
194    group.add_option("", "--no-progress-bar", dest="useProgressBar",
195                     help="Do not use curses based progress bar",
196                     action="store_false", default=True)
197    parser.add_option_group(group)
198
199    group = OptionGroup(parser, "Test Execution")
200    group.add_option("", "--path", dest="path",
201                     help="Additional paths to add to testing environment",
202                     action="append", type=str, default=[])
203    group.add_option("", "--vg", dest="useValgrind",
204                     help="Run tests under valgrind",
205                     action="store_true", default=False)
206    group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
207                     help="Check for memory leaks under valgrind",
208                     action="store_true", default=False)
209    group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
210                     help="Specify an extra argument for valgrind",
211                     type=str, action="append", default=[])
212    group.add_option("", "--time-tests", dest="timeTests",
213                     help="Track elapsed wall time for each test",
214                     action="store_true", default=False)
215    group.add_option("", "--no-execute", dest="noExecute",
216                     help="Don't execute any tests (assume PASS)",
217                     action="store_true", default=False)
218    parser.add_option_group(group)
219
220    group = OptionGroup(parser, "Test Selection")
221    group.add_option("", "--max-tests", dest="maxTests", metavar="N",
222                     help="Maximum number of tests to run",
223                     action="store", type=int, default=None)
224    group.add_option("", "--max-time", dest="maxTime", metavar="N",
225                     help="Maximum time to spend testing (in seconds)",
226                     action="store", type=float, default=None)
227    group.add_option("", "--shuffle", dest="shuffle",
228                     help="Run tests in random order",
229                     action="store_true", default=False)
230    group.add_option("", "--filter", dest="filter", metavar="REGEX",
231                     help=("Only run tests with paths matching the given "
232                           "regular expression"),
233                     action="store", default=None)
234    parser.add_option_group(group)
235
236    group = OptionGroup(parser, "Debug and Experimental Options")
237    group.add_option("", "--debug", dest="debug",
238                      help="Enable debugging (for 'lit' development)",
239                      action="store_true", default=False)
240    group.add_option("", "--show-suites", dest="showSuites",
241                      help="Show discovered test suites",
242                      action="store_true", default=False)
243    group.add_option("", "--show-tests", dest="showTests",
244                      help="Show all discovered tests",
245                      action="store_true", default=False)
246    group.add_option("", "--repeat", dest="repeatTests", metavar="N",
247                      help="Repeat tests N times (for timing)",
248                      action="store", default=None, type=int)
249    parser.add_option_group(group)
250
251    (opts, args) = parser.parse_args()
252
253    if not args:
254        parser.error('No inputs specified')
255
256    if opts.numThreads is None:
257# Python <2.5 has a race condition causing lit to always fail with numThreads>1
258# http://bugs.python.org/issue1731717
259# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
260# threads by default there.
261       if sys.hexversion >= 0x2050200:
262               opts.numThreads = lit.Util.detectCPUs()
263       else:
264               opts.numThreads = 1
265
266    inputs = args
267
268    # Create the user defined parameters.
269    userParams = dict(builtinParameters)
270    for entry in opts.userParameters:
271        if '=' not in entry:
272            name,val = entry,''
273        else:
274            name,val = entry.split('=', 1)
275        userParams[name] = val
276
277    # Create the global config object.
278    litConfig = lit.LitConfig.LitConfig(
279        progname = os.path.basename(sys.argv[0]),
280        path = opts.path,
281        quiet = opts.quiet,
282        useValgrind = opts.useValgrind,
283        valgrindLeakCheck = opts.valgrindLeakCheck,
284        valgrindArgs = opts.valgrindArgs,
285        noExecute = opts.noExecute,
286        debug = opts.debug,
287        isWindows = (platform.system()=='Windows'),
288        params = userParams,
289        config_prefix = opts.configPrefix)
290
291    tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
292
293    if opts.showSuites or opts.showTests:
294        # Aggregate the tests by suite.
295        suitesAndTests = {}
296        for t in tests:
297            if t.suite not in suitesAndTests:
298                suitesAndTests[t.suite] = []
299            suitesAndTests[t.suite].append(t)
300        suitesAndTests = suitesAndTests.items()
301        suitesAndTests.sort(key = lambda item: item[0].name)
302
303        # Show the suites, if requested.
304        if opts.showSuites:
305            print('-- Test Suites --')
306            for ts,ts_tests in suitesAndTests:
307                print('  %s - %d tests' %(ts.name, len(ts_tests)))
308                print('    Source Root: %s' % ts.source_root)
309                print('    Exec Root  : %s' % ts.exec_root)
310
311        # Show the tests, if requested.
312        if opts.showTests:
313            print('-- Available Tests --')
314            for ts,ts_tests in suitesAndTests:
315                ts_tests.sort(key = lambda test: test.path_in_suite)
316                for test in ts_tests:
317                    print('  %s' % (test.getFullName(),))
318
319    # Select and order the tests.
320    numTotalTests = len(tests)
321
322    # First, select based on the filter expression if given.
323    if opts.filter:
324        try:
325            rex = re.compile(opts.filter)
326        except:
327            parser.error("invalid regular expression for --filter: %r" % (
328                    opts.filter))
329        tests = [t for t in tests
330                 if rex.search(t.getFullName())]
331
332    # Then select the order.
333    if opts.shuffle:
334        random.shuffle(tests)
335    else:
336        tests.sort(key = lambda t: t.getFullName())
337
338    # Finally limit the number of tests, if desired.
339    if opts.maxTests is not None:
340        tests = tests[:opts.maxTests]
341
342    # Don't create more threads than tests.
343    opts.numThreads = min(len(tests), opts.numThreads)
344
345    extra = ''
346    if len(tests) != numTotalTests:
347        extra = ' of %d' % numTotalTests
348    header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
349                                                      opts.numThreads)
350
351    if opts.repeatTests:
352        tests = [t.copyWithIndex(i)
353                 for t in tests
354                 for i in range(opts.repeatTests)]
355
356    progressBar = None
357    if not opts.quiet:
358        if opts.succinct and opts.useProgressBar:
359            try:
360                tc = lit.ProgressBar.TerminalController()
361                progressBar = lit.ProgressBar.ProgressBar(tc, header)
362            except ValueError:
363                print(header)
364                progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
365        else:
366            print(header)
367
368    startTime = time.time()
369    display = TestingProgressDisplay(opts, len(tests), progressBar)
370    provider = TestProvider(tests, opts.maxTime)
371
372    try:
373      import win32api
374    except ImportError:
375      pass
376    else:
377      def console_ctrl_handler(type):
378        provider.cancel()
379        return True
380      win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
381
382    runTests(opts.numThreads, litConfig, provider, display)
383    display.finish()
384
385    if not opts.quiet:
386        print('Testing Time: %.2fs'%(time.time() - startTime))
387
388    # Update results for any tests which weren't run.
389    for t in tests:
390        if t.result is None:
391            t.setResult(lit.Test.UNRESOLVED, '', 0.0)
392
393    # List test results organized by kind.
394    hasFailures = False
395    byCode = {}
396    for t in tests:
397        if t.result not in byCode:
398            byCode[t.result] = []
399        byCode[t.result].append(t)
400        if t.result.isFailure:
401            hasFailures = True
402
403    # FIXME: Show unresolved and (optionally) unsupported tests.
404    for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
405                       ('Failing Tests', lit.Test.FAIL)):
406        elts = byCode.get(code)
407        if not elts:
408            continue
409        print('*'*20)
410        print('%s (%d):' % (title, len(elts)))
411        for t in elts:
412            print('    %s' % t.getFullName())
413        sys.stdout.write('\n')
414
415    if opts.timeTests:
416        # Collate, in case we repeated tests.
417        times = {}
418        for t in tests:
419            key = t.getFullName()
420            times[key] = times.get(key, 0.) + t.elapsed
421
422        byTime = list(times.items())
423        byTime.sort(key = lambda item: item[1])
424        if byTime:
425            lit.Util.printHistogram(byTime, title='Tests')
426
427    for name,code in (('Expected Passes    ', lit.Test.PASS),
428                      ('Expected Failures  ', lit.Test.XFAIL),
429                      ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
430                      ('Unresolved Tests   ', lit.Test.UNRESOLVED),
431                      ('Unexpected Passes  ', lit.Test.XPASS),
432                      ('Unexpected Failures', lit.Test.FAIL),):
433        if opts.quiet and not code.isFailure:
434            continue
435        N = len(byCode.get(code,[]))
436        if N:
437            print('  %s: %d' % (name,N))
438
439    # If we encountered any additional errors, exit abnormally.
440    if litConfig.numErrors:
441        sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
442        sys.exit(2)
443
444    # Warn about warnings.
445    if litConfig.numWarnings:
446        sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
447
448    if hasFailures:
449        sys.exit(1)
450    sys.exit(0)
451
452if __name__=='__main__':
453    main()
454