main.py revision 47896090f30fe9f37a776b46e285b1d6d8b1ad9b 
1#!/usr/bin/env python
2
3"""
4lit - LLVM Integrated Tester.
5
6See lit.pod for more information.
7"""
8
9import math, os, platform, random, re, sys, time, threading, traceback
10
11import ProgressBar
12import TestRunner
13import Util
14
15import LitConfig
16import Test
17
18import lit.discovery
19
20class TestingProgressDisplay:
21    def __init__(self, opts, numTests, progressBar=None):
22        self.opts = opts
23        self.numTests = numTests
24        self.current = None
25        self.lock = threading.Lock()
26        self.progressBar = progressBar
27        self.completed = 0
28
29    def update(self, test):
30        # Avoid locking overhead in quiet mode
31        if self.opts.quiet and not test.result.isFailure:
32            self.completed += 1
33            return
34
35        # Output lock.
36        self.lock.acquire()
37        try:
38            self.handleUpdate(test)
39        finally:
40            self.lock.release()
41
42    def finish(self):
43        if self.progressBar:
44            self.progressBar.clear()
45        elif self.opts.quiet:
46            pass
47        elif self.opts.succinct:
48            sys.stdout.write('\n')
49
50    def handleUpdate(self, test):
51        self.completed += 1
52        if self.progressBar:
53            self.progressBar.update(float(self.completed)/self.numTests,
54                                    test.getFullName())
55
56        if self.opts.succinct and not test.result.isFailure:
57            return
58
59        if self.progressBar:
60            self.progressBar.clear()
61
62        print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
63                                     self.completed, self.numTests)
64
65        if test.result.isFailure and self.opts.showOutput:
66            print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
67                                              '*'*20)
68            print test.output
69            print "*" * 20
70
71        sys.stdout.flush()
72
73class TestProvider:
74    def __init__(self, tests, maxTime):
75        self.maxTime = maxTime
76        self.iter = iter(tests)
77        self.lock = threading.Lock()
78        self.startTime = time.time()
79
80    def get(self):
81        # Check if we have run out of time.
82        if self.maxTime is not None:
83            if time.time() - self.startTime > self.maxTime:
84                return None
85
86        # Otherwise take the next test.
87        self.lock.acquire()
88        try:
89            item = self.iter.next()
90        except StopIteration:
91            item = None
92        self.lock.release()
93        return item
94
95class Tester(threading.Thread):
96    def __init__(self, litConfig, provider, display):
97        threading.Thread.__init__(self)
98        self.litConfig = litConfig
99        self.provider = provider
100        self.display = display
101
102    def run(self):
103        while 1:
104            item = self.provider.get()
105            if item is None:
106                break
107            self.runTest(item)
108
109    def runTest(self, test):
110        result = None
111        startTime = time.time()
112        try:
113            result, output = test.config.test_format.execute(test,
114                                                             self.litConfig)
115        except KeyboardInterrupt:
116            # This is a sad hack. Unfortunately subprocess goes
117            # bonkers with ctrl-c and we start forking merrily.
118            print '\nCtrl-C detected, goodbye.'
119            os.kill(0,9)
120        except:
121            if self.litConfig.debug:
122                raise
123            result = Test.UNRESOLVED
124            output = 'Exception during script execution:\n'
125            output += traceback.format_exc()
126            output += '\n'
127        elapsed = time.time() - startTime
128
129        test.setResult(result, output, elapsed)
130        self.display.update(test)
131
132def runTests(numThreads, litConfig, provider, display):
133    # If only using one testing thread, don't use threads at all; this lets us
134    # profile, among other things.
135    if numThreads == 1:
136        t = Tester(litConfig, provider, display)
137        t.run()
138        return
139
140    # Otherwise spin up the testing threads and wait for them to finish.
141    testers = [Tester(litConfig, provider, display)
142               for i in range(numThreads)]
143    for t in testers:
144        t.start()
145    try:
146        for t in testers:
147            t.join()
148    except KeyboardInterrupt:
149        sys.exit(2)
150
151def main(builtinParameters = {}):
152    # Bump the GIL check interval, its more important to get any one thread to a
153    # blocking operation (hopefully exec) than to try and unblock other threads.
154    #
155    # FIXME: This is a hack.
156    import sys
157    sys.setcheckinterval(1000)
158
159    global options
160    from optparse import OptionParser, OptionGroup
161    parser = OptionParser("usage: %prog [options] {file-or-path}")
162
163    parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
164                      help="Number of testing threads",
165                      type=int, action="store", default=None)
166    parser.add_option("", "--config-prefix", dest="configPrefix",
167                      metavar="NAME", help="Prefix for 'lit' config files",
168                      action="store", default=None)
169    parser.add_option("", "--param", dest="userParameters",
170                      metavar="NAME=VAL",
171                      help="Add 'NAME' = 'VAL' to the user defined parameters",
172                      type=str, action="append", default=[])
173
174    group = OptionGroup(parser, "Output Format")
175    # FIXME: I find these names very confusing, although I like the
176    # functionality.
177    group.add_option("-q", "--quiet", dest="quiet",
178                     help="Suppress no error output",
179                     action="store_true", default=False)
180    group.add_option("-s", "--succinct", dest="succinct",
181                     help="Reduce amount of output",
182                     action="store_true", default=False)
183    group.add_option("-v", "--verbose", dest="showOutput",
184                     help="Show all test output",
185                     action="store_true", default=False)
186    group.add_option("", "--no-progress-bar", dest="useProgressBar",
187                     help="Do not use curses based progress bar",
188                     action="store_false", default=True)
189    parser.add_option_group(group)
190
191    group = OptionGroup(parser, "Test Execution")
192    group.add_option("", "--path", dest="path",
193                     help="Additional paths to add to testing environment",
194                     action="append", type=str, default=[])
195    group.add_option("", "--vg", dest="useValgrind",
196                     help="Run tests under valgrind",
197                     action="store_true", default=False)
198    group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
199                     help="Check for memory leaks under valgrind",
200                     action="store_true", default=False)
201    group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
202                     help="Specify an extra argument for valgrind",
203                     type=str, action="append", default=[])
204    group.add_option("", "--time-tests", dest="timeTests",
205                     help="Track elapsed wall time for each test",
206                     action="store_true", default=False)
207    group.add_option("", "--no-execute", dest="noExecute",
208                     help="Don't execute any tests (assume PASS)",
209                     action="store_true", default=False)
210    parser.add_option_group(group)
211
212    group = OptionGroup(parser, "Test Selection")
213    group.add_option("", "--max-tests", dest="maxTests", metavar="N",
214                     help="Maximum number of tests to run",
215                     action="store", type=int, default=None)
216    group.add_option("", "--max-time", dest="maxTime", metavar="N",
217                     help="Maximum time to spend testing (in seconds)",
218                     action="store", type=float, default=None)
219    group.add_option("", "--shuffle", dest="shuffle",
220                     help="Run tests in random order",
221                     action="store_true", default=False)
222    group.add_option("", "--filter", dest="filter", metavar="EXPRESSION",
223                     help=("Only run tests with paths matching the given "
224                           "regular expression"),
225                     action="store", default=None)
226    parser.add_option_group(group)
227
228    group = OptionGroup(parser, "Debug and Experimental Options")
229    group.add_option("", "--debug", dest="debug",
230                      help="Enable debugging (for 'lit' development)",
231                      action="store_true", default=False)
232    group.add_option("", "--show-suites", dest="showSuites",
233                      help="Show discovered test suites",
234                      action="store_true", default=False)
235    group.add_option("", "--repeat", dest="repeatTests", metavar="N",
236                      help="Repeat tests N times (for timing)",
237                      action="store", default=None, type=int)
238    parser.add_option_group(group)
239
240    (opts, args) = parser.parse_args()
241
242    if not args:
243        parser.error('No inputs specified')
244
245    if opts.numThreads is None:
246# Python <2.5 has a race condition causing lit to always fail with numThreads>1
247# http://bugs.python.org/issue1731717
248# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
249# threads by default there.
250       if sys.hexversion >= 0x2050200:
251               opts.numThreads = Util.detectCPUs()
252       else:
253               opts.numThreads = 1
254
255    inputs = args
256
257    # Create the user defined parameters.
258    userParams = dict(builtinParameters)
259    for entry in opts.userParameters:
260        if '=' not in entry:
261            name,val = entry,''
262        else:
263            name,val = entry.split('=', 1)
264        userParams[name] = val
265
266    # Create the global config object.
267    litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
268                                    path = opts.path,
269                                    quiet = opts.quiet,
270                                    useValgrind = opts.useValgrind,
271                                    valgrindLeakCheck = opts.valgrindLeakCheck,
272                                    valgrindArgs = opts.valgrindArgs,
273                                    noExecute = opts.noExecute,
274                                    ignoreStdErr = False,
275                                    debug = opts.debug,
276                                    isWindows = (platform.system()=='Windows'),
277                                    params = userParams,
278                                    config_prefix = opts.configPrefix)
279
280    tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
281
282    if opts.showSuites:
283        suitesAndTests = {}
284        for t in tests:
285            if t.suite not in suitesAndTests:
286                suitesAndTests[t.suite] = []
287            suitesAndTests[t.suite].append(t)
288
289        print '-- Test Suites --'
290        suitesAndTests = suitesAndTests.items()
291        suitesAndTests.sort(key = lambda (ts,_): ts.name)
292        for ts,ts_tests in suitesAndTests:
293            print '  %s - %d tests' %(ts.name, len(ts_tests))
294            print '    Source Root: %s' % ts.source_root
295            print '    Exec Root  : %s' % ts.exec_root
296
297    # Select and order the tests.
298    numTotalTests = len(tests)
299
300    # First, select based on the filter expression if given.
301    if opts.filter:
302        try:
303            rex = re.compile(opts.filter)
304        except:
305            parser.error("invalid regular expression for --filter: %r" % (
306                    opts.filter))
307        tests = [t for t in tests
308                 if rex.search(t.getFullName())]
309
310    # Then select the order.
311    if opts.shuffle:
312        random.shuffle(tests)
313    else:
314        tests.sort(key = lambda t: t.getFullName())
315
316    # Finally limit the number of tests, if desired.
317    if opts.maxTests is not None:
318        tests = tests[:opts.maxTests]
319
320    # Don't create more threads than tests.
321    opts.numThreads = min(len(tests), opts.numThreads)
322
323    extra = ''
324    if len(tests) != numTotalTests:
325        extra = ' of %d' % numTotalTests
326    header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
327                                                      opts.numThreads)
328
329    if opts.repeatTests:
330        tests = [t.copyWithIndex(i)
331                 for t in tests
332                 for i in range(opts.repeatTests)]
333
334    progressBar = None
335    if not opts.quiet:
336        if opts.succinct and opts.useProgressBar:
337            try:
338                tc = ProgressBar.TerminalController()
339                progressBar = ProgressBar.ProgressBar(tc, header)
340            except ValueError:
341                print header
342                progressBar = ProgressBar.SimpleProgressBar('Testing: ')
343        else:
344            print header
345
346    startTime = time.time()
347    display = TestingProgressDisplay(opts, len(tests), progressBar)
348    provider = TestProvider(tests, opts.maxTime)
349    runTests(opts.numThreads, litConfig, provider, display)
350    display.finish()
351
352    if not opts.quiet:
353        print 'Testing Time: %.2fs'%(time.time() - startTime)
354
355    # Update results for any tests which weren't run.
356    for t in tests:
357        if t.result is None:
358            t.setResult(Test.UNRESOLVED, '', 0.0)
359
360    # List test results organized by kind.
361    hasFailures = False
362    byCode = {}
363    for t in tests:
364        if t.result not in byCode:
365            byCode[t.result] = []
366        byCode[t.result].append(t)
367        if t.result.isFailure:
368            hasFailures = True
369
370    # FIXME: Show unresolved and (optionally) unsupported tests.
371    for title,code in (('Unexpected Passing Tests', Test.XPASS),
372                       ('Failing Tests', Test.FAIL)):
373        elts = byCode.get(code)
374        if not elts:
375            continue
376        print '*'*20
377        print '%s (%d):' % (title, len(elts))
378        for t in elts:
379            print '    %s' % t.getFullName()
380        print
381
382    if opts.timeTests:
383        # Collate, in case we repeated tests.
384        times = {}
385        for t in tests:
386            key = t.getFullName()
387            times[key] = times.get(key, 0.) + t.elapsed
388
389        byTime = list(times.items())
390        byTime.sort(key = lambda (name,elapsed): elapsed)
391        if byTime:
392            Util.printHistogram(byTime, title='Tests')
393
394    for name,code in (('Expected Passes    ', Test.PASS),
395                      ('Expected Failures  ', Test.XFAIL),
396                      ('Unsupported Tests  ', Test.UNSUPPORTED),
397                      ('Unresolved Tests   ', Test.UNRESOLVED),
398                      ('Unexpected Passes  ', Test.XPASS),
399                      ('Unexpected Failures', Test.FAIL),):
400        if opts.quiet and not code.isFailure:
401            continue
402        N = len(byCode.get(code,[]))
403        if N:
404            print '  %s: %d' % (name,N)
405
406    # If we encountered any additional errors, exit abnormally.
407    if litConfig.numErrors:
408        print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
409        sys.exit(2)
410
411    # Warn about warnings.
412    if litConfig.numWarnings:
413        print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
414
415    if hasFailures:
416        sys.exit(1)
417    sys.exit(0)
418
419if __name__=='__main__':
420    main()
421