dotest.py revision b5fe80c62f0de8b855e0ee4bf25d3325f52e1984 
1#!/usr/bin/env python
2
3"""
4A simple testing framework for lldb using python's unit testing framework.
5
6Tests for lldb are written as python scripts which take advantage of the script
7bridging provided by LLDB.framework to interact with lldb core.
8
9A specific naming pattern is followed by the .py script to be recognized as
10a module which implements a test scenario, namely, Test*.py.
11
12To specify the directories where "Test*.py" python test scripts are located,
13you need to pass in a list of directory names.  By default, the current
14working directory is searched if nothing is specified on the command line.
15
16Type:
17
18./dotest.py -h
19
20for available options.
21"""
22
23import os, signal, sys, time
24import unittest2
25
26def is_exe(fpath):
27    """Returns true if fpath is an executable."""
28    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
29
30def which(program):
31    """Returns the full path to a program; None otherwise."""
32    fpath, fname = os.path.split(program)
33    if fpath:
34        if is_exe(program):
35            return program
36    else:
37        for path in os.environ["PATH"].split(os.pathsep):
38            exe_file = os.path.join(path, program)
39            if is_exe(exe_file):
40                return exe_file
41    return None
42
43class _WritelnDecorator(object):
44    """Used to decorate file-like objects with a handy 'writeln' method"""
45    def __init__(self,stream):
46        self.stream = stream
47
48    def __getattr__(self, attr):
49        if attr in ('stream', '__getstate__'):
50            raise AttributeError(attr)
51        return getattr(self.stream,attr)
52
53    def writeln(self, arg=None):
54        if arg:
55            self.write(arg)
56        self.write('\n') # text-mode streams translate to \r\n if needed
57
58#
59# Global variables:
60#
61
62# The test suite.
63suite = unittest2.TestSuite()
64
65# By default, both command line and Python API tests are performed.
66# Use @python_api_test decorator, defined in lldbtest.py, to mark a test as
67# a Python API test.
68dont_do_python_api_test = False
69
70# By default, both command line and Python API tests are performed.
71just_do_python_api_test = False
72
73# The blacklist is optional (-b blacklistFile) and allows a central place to skip
74# testclass's and/or testclass.testmethod's.
75blacklist = None
76
77# The dictionary as a result of sourcing blacklistFile.
78blacklistConfig = {}
79
80# The config file is optional.
81configFile = None
82
83# Test suite repeat count.  Can be overwritten with '-# count'.
84count = 1
85
86# The dictionary as a result of sourcing configFile.
87config = {}
88
89# The 'archs' and 'compilers' can be specified via either command line or configFile,
90# with the command line overriding the configFile.  When specified, they should be
91# of the list type.  For example, "-A x86_64^i386" => archs=['x86_64', 'i386'] and
92# "-C gcc^clang" => compilers=['gcc', 'clang'].
93archs = None
94compilers = None
95
96# Delay startup in order for the debugger to attach.
97delay = False
98
99# Dump the Python sys.path variable.  Use '-D' to dump sys.path.
100dumpSysPath = False
101
102# By default, failfast is False.  Use '-F' to overwrite it.
103failfast = False
104
105# The filter (testclass.testmethod) used to admit tests into our test suite.
106filterspec = None
107
108# If '-g' is specified, the filterspec is not exclusive.  If a test module does
109# not contain testclass.testmethod which matches the filterspec, the whole test
110# module is still admitted into our test suite.  fs4all flag defaults to True.
111fs4all = True
112
113# Ignore the build search path relative to this script to locate the lldb.py module.
114ignore = False
115
116# By default, we skip long running test case.  Use '-l' option to override.
117skipLongRunningTest = True
118
119# The regular expression pattern to match against eligible filenames as our test cases.
120regexp = None
121
122# By default, tests are executed in place and cleanups are performed afterwards.
123# Use '-r dir' option to relocate the tests and their intermediate files to a
124# different directory and to forgo any cleanups.  The directory specified must
125# not exist yet.
126rdir = None
127
128# By default, recorded session info for errored/failed test are dumped into its
129# own file under a session directory named after the timestamp of the test suite
130# run.  Use '-s session-dir-name' to specify a specific dir name.
131sdir_name = None
132
133# Set this flag if there is any session info dumped during the test run.
134sdir_has_content = False
135
136# svn_info stores the output from 'svn info lldb.base.dir'.
137svn_info = ''
138
139# Default verbosity is 0.
140verbose = 0
141
142# By default, search from the current working directory.
143testdirs = [ os.getcwd() ]
144
145# Separator string.
146separator = '-' * 70
147
148
149def usage():
150    print """
151Usage: dotest.py [option] [args]
152where options:
153-h   : print this help message and exit.  Add '-v' for more detailed help.
154-A   : specify the architecture(s) to launch for the inferior process
155       -A i386 => launch inferior with i386 architecture
156       -A x86_64^i386 => launch inferior with x86_64 and i386 architectures
157-C   : specify the compiler(s) used to build the inferior executable
158       -C clang => build debuggee using clang compiler
159       -C clang^gcc => build debuggee using clang and gcc compilers
160-D   : dump the Python sys.path variable
161-a   : don't do lldb Python API tests
162       use @python_api_test to decorate a test case as lldb Python API test
163+a   : just do lldb Python API tests
164       do not specify both '-a' and '+a' at the same time
165-b   : read a blacklist file specified after this option
166-c   : read a config file specified after this option
167       the architectures and compilers (note the plurals) specified via '-A' and '-C'
168       will override those specified via a config file
169       (see also lldb-trunk/example/test/usage-config)
170-d   : delay startup for 10 seconds (in order for the debugger to attach)
171-F   : failfast, stop the test suite on the first error/failure
172-f   : specify a filter, which consists of the test class name, a dot, followed by
173       the test method, to only admit such test into the test suite
174       e.g., -f 'ClassTypesTestCase.test_with_dwarf_and_python_api'
175-g   : if specified, the filterspec by -f is not exclusive, i.e., if a test module
176       does not match the filterspec (testclass.testmethod), the whole module is
177       still admitted to the test suite
178-i   : ignore (don't bailout) if 'lldb.py' module cannot be located in the build
179       tree relative to this script; use PYTHONPATH to locate the module
180-l   : don't skip long running test
181-p   : specify a regexp filename pattern for inclusion in the test suite
182-r   : specify a dir to relocate the tests and their intermediate files to;
183       the directory must not exist before running this test driver;
184       no cleanup of intermediate test files is performed in this case
185-s   : specify the name of the dir created to store the session files of tests
186       with errored or failed status; if not specified, the test driver uses the
187       timestamp as the session dir name
188-t   : turn on tracing of lldb command and other detailed test executions
189-v   : do verbose mode of unittest framework (print out each test case invocation)
190-w   : insert some wait time (currently 0.5 sec) between consecutive test cases
191-#   : Repeat the test suite for a specified number of times
192
193and:
194args : specify a list of directory names to search for test modules named after
195       Test*.py (test discovery)
196       if empty, search from the curret working directory, instead
197"""
198
199    if verbose > 0:
200        print """
201Examples:
202
203This is an example of using the -f option to pinpoint to a specfic test class
204and test method to be run:
205
206$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
207----------------------------------------------------------------------
208Collected 1 test
209
210test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
211Test 'frame variable this' when stopped on a class constructor. ... ok
212
213----------------------------------------------------------------------
214Ran 1 test in 1.396s
215
216OK
217
218And this is an example of using the -p option to run a single file (the filename
219matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
220
221$ ./dotest.py -v -p ObjC
222----------------------------------------------------------------------
223Collected 4 tests
224
225test_break_with_dsym (TestObjCMethods.FoundationTestCase)
226Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
227test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
228Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
229test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
230Lookup objective-c data types and evaluate expressions. ... ok
231test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
232Lookup objective-c data types and evaluate expressions. ... ok
233
234----------------------------------------------------------------------
235Ran 4 tests in 16.661s
236
237OK
238
239Running of this script also sets up the LLDB_TEST environment variable so that
240individual test cases can locate their supporting files correctly.  The script
241tries to set up Python's search paths for modules by looking at the build tree
242relative to this script.  See also the '-i' option in the following example.
243
244Finally, this is an example of using the lldb.py module distributed/installed by
245Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
246option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
247as the architecture and CC=clang to specify the compiler used for the test run:
248
249$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
250
251Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
252----------------------------------------------------------------------
253Collected 2 tests
254
255test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
256Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
257test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
258Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
259
260----------------------------------------------------------------------
261Ran 2 tests in 5.659s
262
263OK
264
265The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
266notify the directory containing the session logs for test failures or errors.
267In case there is any test failure/error, a similar message is appended at the
268end of the stderr output for your convenience.
269
270Environment variables related to loggings:
271
272o LLDB_LOG: if defined, specifies the log file pathname for the 'lldb' subsystem
273  with a default option of 'event process' if LLDB_LOG_OPTION is not defined.
274
275o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the
276  'process.gdb-remote' subsystem with a default option of 'packets' if
277  GDB_REMOTE_LOG_OPTION is not defined.
278"""
279    sys.exit(0)
280
281
282def parseOptionsAndInitTestdirs():
283    """Initialize the list of directories containing our unittest scripts.
284
285    '-h/--help as the first option prints out usage info and exit the program.
286    """
287
288    global dont_do_python_api_test
289    global just_do_python_api_test
290    global blacklist
291    global blacklistConfig
292    global configFile
293    global archs
294    global compilers
295    global count
296    global delay
297    global dumpSysPath
298    global failfast
299    global filterspec
300    global fs4all
301    global ignore
302    global skipLongRunningTest
303    global regexp
304    global rdir
305    global sdir_name
306    global verbose
307    global testdirs
308
309    do_help = False
310
311    if len(sys.argv) == 1:
312        return
313
314    # Process possible trace and/or verbose flag, among other things.
315    index = 1
316    while index < len(sys.argv):
317        if sys.argv[index].startswith('-') or sys.argv[index].startswith('+'):
318            # We should continue processing...
319            pass
320        else:
321            # End of option processing.
322            break
323
324        if sys.argv[index].find('-h') != -1:
325            index += 1
326            do_help = True
327        elif sys.argv[index].startswith('-A'):
328            # Increment by 1 to fetch the ARCH spec.
329            index += 1
330            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
331                usage()
332            archs = sys.argv[index].split('^')
333            index += 1
334        elif sys.argv[index].startswith('-C'):
335            # Increment by 1 to fetch the CC spec.
336            index += 1
337            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
338                usage()
339            compilers = sys.argv[index].split('^')
340            index += 1
341        elif sys.argv[index].startswith('-D'):
342            dumpSysPath = True
343            index += 1
344        elif sys.argv[index].startswith('-a'):
345            dont_do_python_api_test = True
346            index += 1
347        elif sys.argv[index].startswith('+a'):
348            just_do_python_api_test = True
349            index += 1
350        elif sys.argv[index].startswith('-b'):
351            # Increment by 1 to fetch the blacklist file name option argument.
352            index += 1
353            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
354                usage()
355            blacklistFile = sys.argv[index]
356            if not os.path.isfile(blacklistFile):
357                print "Blacklist file:", blacklistFile, "does not exist!"
358                usage()
359            index += 1
360            # Now read the blacklist contents and assign it to blacklist.
361            execfile(blacklistFile, globals(), blacklistConfig)
362            blacklist = blacklistConfig.get('blacklist')
363        elif sys.argv[index].startswith('-c'):
364            # Increment by 1 to fetch the config file name option argument.
365            index += 1
366            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
367                usage()
368            configFile = sys.argv[index]
369            if not os.path.isfile(configFile):
370                print "Config file:", configFile, "does not exist!"
371                usage()
372            index += 1
373        elif sys.argv[index].startswith('-d'):
374            delay = True
375            index += 1
376        elif sys.argv[index].startswith('-F'):
377            failfast = True
378            index += 1
379        elif sys.argv[index].startswith('-f'):
380            # Increment by 1 to fetch the filter spec.
381            index += 1
382            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
383                usage()
384            filterspec = sys.argv[index]
385            index += 1
386        elif sys.argv[index].startswith('-g'):
387            fs4all = False
388            index += 1
389        elif sys.argv[index].startswith('-i'):
390            ignore = True
391            index += 1
392        elif sys.argv[index].startswith('-l'):
393            skipLongRunningTest = False
394            index += 1
395        elif sys.argv[index].startswith('-p'):
396            # Increment by 1 to fetch the reg exp pattern argument.
397            index += 1
398            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
399                usage()
400            regexp = sys.argv[index]
401            index += 1
402        elif sys.argv[index].startswith('-r'):
403            # Increment by 1 to fetch the relocated directory argument.
404            index += 1
405            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
406                usage()
407            rdir = os.path.abspath(sys.argv[index])
408            if os.path.exists(rdir):
409                print "Relocated directory:", rdir, "must not exist!"
410                usage()
411            index += 1
412        elif sys.argv[index].startswith('-s'):
413            # Increment by 1 to fetch the session dir name.
414            index += 1
415            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
416                usage()
417            sdir_name = sys.argv[index]
418            index += 1
419        elif sys.argv[index].startswith('-t'):
420            os.environ["LLDB_COMMAND_TRACE"] = "YES"
421            index += 1
422        elif sys.argv[index].startswith('-v'):
423            verbose = 2
424            index += 1
425        elif sys.argv[index].startswith('-w'):
426            os.environ["LLDB_WAIT_BETWEEN_TEST_CASES"] = 'YES'
427            index += 1
428        elif sys.argv[index].startswith('-#'):
429            # Increment by 1 to fetch the repeat count argument.
430            index += 1
431            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
432                usage()
433            count = int(sys.argv[index])
434            index += 1
435        else:
436            print "Unknown option: ", sys.argv[index]
437            usage()
438
439    if do_help == True:
440        usage()
441
442    # Do not specify both '-a' and '+a' at the same time.
443    if dont_do_python_api_test and just_do_python_api_test:
444        usage()
445
446    # Gather all the dirs passed on the command line.
447    if len(sys.argv) > index:
448        testdirs = map(os.path.abspath, sys.argv[index:])
449
450    # If '-r dir' is specified, the tests should be run under the relocated
451    # directory.  Let's copy the testdirs over.
452    if rdir:
453        from shutil import copytree, ignore_patterns
454
455        tmpdirs = []
456        for srcdir in testdirs:
457            dstdir = os.path.join(rdir, os.path.basename(srcdir))
458            # Don't copy the *.pyc and .svn stuffs.
459            copytree(srcdir, dstdir, ignore=ignore_patterns('*.pyc', '.svn'))
460            tmpdirs.append(dstdir)
461
462        # This will be our modified testdirs.
463        testdirs = tmpdirs
464
465        # With '-r dir' specified, there's no cleanup of intermediate test files.
466        os.environ["LLDB_DO_CLEANUP"] = 'NO'
467
468        # If testdirs is ['test'], the make directory has already been copied
469        # recursively and is contained within the rdir/test dir.  For anything
470        # else, we would need to copy over the make directory and its contents,
471        # so that, os.listdir(rdir) looks like, for example:
472        #
473        #     array_types conditional_break make
474        #
475        # where the make directory contains the Makefile.rules file.
476        if len(testdirs) != 1 or os.path.basename(testdirs[0]) != 'test':
477            # Don't copy the .svn stuffs.
478            copytree('make', os.path.join(rdir, 'make'),
479                     ignore=ignore_patterns('.svn'))
480
481    #print "testdirs:", testdirs
482
483    # Source the configFile if specified.
484    # The side effect, if any, will be felt from this point on.  An example
485    # config file may be these simple two lines:
486    #
487    # sys.stderr = open("/tmp/lldbtest-stderr", "w")
488    # sys.stdout = open("/tmp/lldbtest-stdout", "w")
489    #
490    # which will reassign the two file objects to sys.stderr and sys.stdout,
491    # respectively.
492    #
493    # See also lldb-trunk/example/test/usage-config.
494    global config
495    if configFile:
496        # Pass config (a dictionary) as the locals namespace for side-effect.
497        execfile(configFile, globals(), config)
498        #print "config:", config
499        #print "sys.stderr:", sys.stderr
500        #print "sys.stdout:", sys.stdout
501
502
503def setupSysPath():
504    """
505    Add LLDB.framework/Resources/Python to the search paths for modules.
506    As a side effect, we also discover the 'lldb' executable and export it here.
507    """
508
509    global rdir
510    global testdirs
511    global dumpSysPath
512    global svn_info
513
514    # Get the directory containing the current script.
515    if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
516        scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
517    else:
518        scriptPath = sys.path[0]
519    if not scriptPath.endswith('test'):
520        print "This script expects to reside in lldb's test directory."
521        sys.exit(-1)
522
523    if rdir:
524        # Set up the LLDB_TEST environment variable appropriately, so that the
525        # individual tests can be located relatively.
526        #
527        # See also lldbtest.TestBase.setUpClass(cls).
528        if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test':
529            os.environ["LLDB_TEST"] = os.path.join(rdir, 'test')
530        else:
531            os.environ["LLDB_TEST"] = rdir
532    else:
533        os.environ["LLDB_TEST"] = scriptPath
534    pluginPath = os.path.join(scriptPath, 'plugins')
535    pexpectPath = os.path.join(scriptPath, 'pexpect-2.4')
536
537    # Append script dir, plugin dir, and pexpect dir to the sys.path.
538    sys.path.append(scriptPath)
539    sys.path.append(pluginPath)
540    sys.path.append(pexpectPath)
541
542    # This is our base name component.
543    base = os.path.abspath(os.path.join(scriptPath, os.pardir))
544
545    # These are for xcode build directories.
546    xcode3_build_dir = ['build']
547    xcode4_build_dir = ['build', 'lldb', 'Build', 'Products']
548    dbg = ['Debug']
549    rel = ['Release']
550    bai = ['BuildAndIntegration']
551    python_resource_dir = ['LLDB.framework', 'Resources', 'Python']
552
553    # Some of the tests can invoke the 'lldb' command directly.
554    # We'll try to locate the appropriate executable right here.
555
556    executable = ['lldb']
557    dbgExec  = os.path.join(base, *(xcode3_build_dir + dbg + executable))
558    dbgExec2 = os.path.join(base, *(xcode4_build_dir + dbg + executable))
559    relExec  = os.path.join(base, *(xcode3_build_dir + rel + executable))
560    relExec2 = os.path.join(base, *(xcode4_build_dir + rel + executable))
561    baiExec  = os.path.join(base, *(xcode3_build_dir + bai + executable))
562    baiExec2 = os.path.join(base, *(xcode4_build_dir + bai + executable))
563
564    lldbExec = None
565    if is_exe(dbgExec):
566        lldbExec = dbgExec
567    elif is_exe(dbgExec2):
568        lldbExec = dbgExec2
569    elif is_exe(relExec):
570        lldbExec = relExec
571    elif is_exe(relExec2):
572        lldbExec = relExec2
573    elif is_exe(baiExec):
574        lldbExec = baiExec
575    elif is_exe(baiExec2):
576        lldbExec = baiExec2
577
578    if not lldbExec:
579        lldbExec = which('lldb')
580
581    if not lldbExec:
582        print "The 'lldb' executable cannot be located.  Some of the tests may not be run as a result."
583    else:
584        os.environ["LLDB_EXEC"] = lldbExec
585        #print "The 'lldb' executable path is", lldbExec
586        os.system('%s -v' % lldbExec)
587
588    import subprocess
589    pipe = subprocess.Popen(["svn", "info", base], stdout = subprocess.PIPE)
590    svn_info = pipe.stdout.read()
591    print svn_info
592
593    global ignore
594
595    # The '-i' option is used to skip looking for lldb.py in the build tree.
596    if ignore:
597        return
598
599    dbgPath  = os.path.join(base, *(xcode3_build_dir + dbg + python_resource_dir))
600    dbgPath2 = os.path.join(base, *(xcode4_build_dir + dbg + python_resource_dir))
601    relPath  = os.path.join(base, *(xcode3_build_dir + rel + python_resource_dir))
602    relPath2 = os.path.join(base, *(xcode4_build_dir + rel + python_resource_dir))
603    baiPath  = os.path.join(base, *(xcode3_build_dir + bai + python_resource_dir))
604    baiPath2 = os.path.join(base, *(xcode4_build_dir + bai + python_resource_dir))
605
606    lldbPath = None
607    if os.path.isfile(os.path.join(dbgPath, 'lldb.py')):
608        lldbPath = dbgPath
609    elif os.path.isfile(os.path.join(dbgPath2, 'lldb.py')):
610        lldbPath = dbgPath2
611    elif os.path.isfile(os.path.join(relPath, 'lldb.py')):
612        lldbPath = relPath
613    elif os.path.isfile(os.path.join(relPath2, 'lldb.py')):
614        lldbPath = relPath2
615    elif os.path.isfile(os.path.join(baiPath, 'lldb.py')):
616        lldbPath = baiPath
617    elif os.path.isfile(os.path.join(baiPath2, 'lldb.py')):
618        lldbPath = baiPath2
619
620    if not lldbPath:
621        print 'This script requires lldb.py to be in either ' + dbgPath + ',',
622        print relPath + ', or ' + baiPath
623        sys.exit(-1)
624
625    # This is to locate the lldb.py module.  Insert it right after sys.path[0].
626    sys.path[1:1] = [lldbPath]
627    if dumpSysPath:
628        print "sys.path:", sys.path
629
630
631def doDelay(delta):
632    """Delaying startup for delta-seconds to facilitate debugger attachment."""
633    def alarm_handler(*args):
634        raise Exception("timeout")
635
636    signal.signal(signal.SIGALRM, alarm_handler)
637    signal.alarm(delta)
638    sys.stdout.write("pid=%d\n" % os.getpid())
639    sys.stdout.write("Enter RET to proceed (or timeout after %d seconds):" %
640                     delta)
641    sys.stdout.flush()
642    try:
643        text = sys.stdin.readline()
644    except:
645        text = ""
646    signal.alarm(0)
647    sys.stdout.write("proceeding...\n")
648    pass
649
650
651def visit(prefix, dir, names):
652    """Visitor function for os.path.walk(path, visit, arg)."""
653
654    global suite
655    global regexp
656    global filterspec
657    global fs4all
658
659    for name in names:
660        if os.path.isdir(os.path.join(dir, name)):
661            continue
662
663        if '.py' == os.path.splitext(name)[1] and name.startswith(prefix):
664            # Try to match the regexp pattern, if specified.
665            if regexp:
666                import re
667                if re.search(regexp, name):
668                    #print "Filename: '%s' matches pattern: '%s'" % (name, regexp)
669                    pass
670                else:
671                    #print "Filename: '%s' does not match pattern: '%s'" % (name, regexp)
672                    continue
673
674            # We found a match for our test.  Add it to the suite.
675
676            # Update the sys.path first.
677            if not sys.path.count(dir):
678                sys.path.insert(0, dir)
679            base = os.path.splitext(name)[0]
680
681            # Thoroughly check the filterspec against the base module and admit
682            # the (base, filterspec) combination only when it makes sense.
683            if filterspec:
684                # Optimistically set the flag to True.
685                filtered = True
686                module = __import__(base)
687                parts = filterspec.split('.')
688                obj = module
689                for part in parts:
690                    try:
691                        parent, obj = obj, getattr(obj, part)
692                    except AttributeError:
693                        # The filterspec has failed.
694                        filtered = False
695                        break
696                # Forgo this module if the (base, filterspec) combo is invalid
697                # and no '-g' option is specified
698                if fs4all and not filtered:
699                    continue
700
701            # Add either the filtered test case or the entire test class.
702            if filterspec and filtered:
703                suite.addTests(
704                    unittest2.defaultTestLoader.loadTestsFromName(filterspec, module))
705            else:
706                # A simple case of just the module name.  Also the failover case
707                # from the filterspec branch when the (base, filterspec) combo
708                # doesn't make sense.
709                suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base))
710
711
712def lldbLoggings():
713    """Check and do lldb loggings if necessary."""
714
715    # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is
716    # defined.  Use ${LLDB_LOG} to specify the log file.
717    ci = lldb.DBG.GetCommandInterpreter()
718    res = lldb.SBCommandReturnObject()
719    if ("LLDB_LOG" in os.environ):
720        if ("LLDB_LOG_OPTION" in os.environ):
721            lldb_log_option = os.environ["LLDB_LOG_OPTION"]
722        else:
723            lldb_log_option = "event process expr state api"
724        ci.HandleCommand(
725            "log enable -n -f " + os.environ["LLDB_LOG"] + " lldb " + lldb_log_option,
726            res)
727        if not res.Succeeded():
728            raise Exception('log enable failed (check LLDB_LOG env variable.')
729    # Ditto for gdb-remote logging if ${GDB_REMOTE_LOG} environment variable is defined.
730    # Use ${GDB_REMOTE_LOG} to specify the log file.
731    if ("GDB_REMOTE_LOG" in os.environ):
732        if ("GDB_REMOTE_LOG_OPTION" in os.environ):
733            gdb_remote_log_option = os.environ["GDB_REMOTE_LOG_OPTION"]
734        else:
735            gdb_remote_log_option = "packets process"
736        ci.HandleCommand(
737            "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " process.gdb-remote "
738            + gdb_remote_log_option,
739            res)
740        if not res.Succeeded():
741            raise Exception('log enable failed (check GDB_REMOTE_LOG env variable.')
742
743def getMyCommandLine():
744    import subprocess
745    ps = subprocess.Popen(['ps', '-o', "command=CMD", str(os.getpid())], stdout=subprocess.PIPE).communicate()[0]
746    lines = ps.split('\n')
747    cmd_line = lines[1]
748    return cmd_line
749
750# ======================================== #
751#                                          #
752# Execution of the test driver starts here #
753#                                          #
754# ======================================== #
755
756#
757# Start the actions by first parsing the options while setting up the test
758# directories, followed by setting up the search paths for lldb utilities;
759# then, we walk the directory trees and collect the tests into our test suite.
760#
761parseOptionsAndInitTestdirs()
762setupSysPath()
763
764#
765# If '-d' is specified, do a delay of 10 seconds for the debugger to attach.
766#
767if delay:
768    doDelay(10)
769
770#
771# If '-l' is specified, do not skip the long running tests.
772if not skipLongRunningTest:
773    os.environ["LLDB_SKIP_LONG_RUNNING_TEST"] = "NO"
774
775#
776# Walk through the testdirs while collecting tests.
777#
778for testdir in testdirs:
779    os.path.walk(testdir, visit, 'Test')
780
781#
782# Now that we have loaded all the test cases, run the whole test suite.
783#
784
785# For the time being, let's bracket the test runner within the
786# lldb.SBDebugger.Initialize()/Terminate() pair.
787import lldb, atexit
788# Update: the act of importing lldb now executes lldb.SBDebugger.Initialize(),
789# there's no need to call it a second time.
790#lldb.SBDebugger.Initialize()
791atexit.register(lambda: lldb.SBDebugger.Terminate())
792
793# Create a singleton SBDebugger in the lldb namespace.
794lldb.DBG = lldb.SBDebugger.Create()
795
796# Put the blacklist in the lldb namespace, to be used by lldb.TestBase.
797lldb.blacklist = blacklist
798
799# Put dont/just_do_python_api_test in the lldb namespace, too.
800lldb.dont_do_python_api_test = dont_do_python_api_test
801lldb.just_do_python_api_test = just_do_python_api_test
802
803# Turn on lldb loggings if necessary.
804lldbLoggings()
805
806# Install the control-c handler.
807unittest2.signals.installHandler()
808
809# If sdir_name is not specified through the '-s sdir_name' option, get a
810# timestamp string and export it as LLDB_SESSION_DIR environment var.  This will
811# be used when/if we want to dump the session info of individual test cases
812# later on.
813#
814# See also TestBase.dumpSessionInfo() in lldbtest.py.
815if not sdir_name:
816    import datetime
817    # The windows platforms don't like ':' in the pathname.
818    timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
819    sdir_name = timestamp
820os.environ["LLDB_SESSION_DIRNAME"] = sdir_name
821
822sys.stderr.write("\nSession logs for test failures/errors/unexpected successes"
823                 " will go into directory '%s'\n" % sdir_name)
824sys.stderr.write("Command invoked: %s\n" % getMyCommandLine())
825
826if not os.path.isdir(sdir_name):
827    os.mkdir(sdir_name)
828fname = os.path.join(sdir_name, "svn-info")
829with open(fname, "w") as f:
830    print >> f, svn_info
831    print >> f, "Command invoked: %s\n" % getMyCommandLine()
832
833#
834# Invoke the default TextTestRunner to run the test suite, possibly iterating
835# over different configurations.
836#
837
838iterArchs = False
839iterCompilers = False
840
841if not archs and "archs" in config:
842    archs = config["archs"]
843
844if isinstance(archs, list) and len(archs) >= 1:
845    iterArchs = True
846
847if not compilers and "compilers" in config:
848    compilers = config["compilers"]
849
850if isinstance(compilers, list) and len(compilers) >= 1:
851    iterCompilers = True
852
853# Make a shallow copy of sys.path, we need to manipulate the search paths later.
854# This is only necessary if we are relocated and with different configurations.
855if rdir:
856    old_sys_path = sys.path[:]
857# If we iterate on archs or compilers, there is a chance we want to split stderr/stdout.
858if iterArchs or iterCompilers:
859    old_stderr = sys.stderr
860    old_stdout = sys.stdout
861    new_stderr = None
862    new_stdout = None
863
864# Iterating over all possible architecture and compiler combinations.
865for ia in range(len(archs) if iterArchs else 1):
866    archConfig = ""
867    if iterArchs:
868        os.environ["ARCH"] = archs[ia]
869        archConfig = "arch=%s" % archs[ia]
870    for ic in range(len(compilers) if iterCompilers else 1):
871        if iterCompilers:
872            os.environ["CC"] = compilers[ic]
873            configString = "%s compiler=%s" % (archConfig, compilers[ic])
874        else:
875            configString = archConfig
876
877        if iterArchs or iterCompilers:
878            # Translate ' ' to '-' for pathname component.
879            from string import maketrans
880            tbl = maketrans(' ', '-')
881            configPostfix = configString.translate(tbl)
882
883            # Check whether we need to split stderr/stdout into configuration
884            # specific files.
885            if old_stderr.name != '<stderr>' and config.get('split_stderr'):
886                if new_stderr:
887                    new_stderr.close()
888                new_stderr = open("%s.%s" % (old_stderr.name, configPostfix), "w")
889                sys.stderr = new_stderr
890            if old_stdout.name != '<stdout>' and config.get('split_stdout'):
891                if new_stdout:
892                    new_stdout.close()
893                new_stdout = open("%s.%s" % (old_stdout.name, configPostfix), "w")
894                sys.stdout = new_stdout
895
896            # If we specified a relocated directory to run the test suite, do
897            # the extra housekeeping to copy the testdirs to a configStringified
898            # directory and to update sys.path before invoking the test runner.
899            # The purpose is to separate the configuration-specific directories
900            # from each other.
901            if rdir:
902                from shutil import copytree, ignore_patterns
903
904                newrdir = "%s.%s" % (rdir, configPostfix)
905
906                # Copy the tree to a new directory with postfix name configPostfix.
907                copytree(rdir, newrdir, ignore=ignore_patterns('*.pyc', '*.o', '*.d'))
908
909               # Update the LLDB_TEST environment variable to reflect new top
910                # level test directory.
911                #
912                # See also lldbtest.TestBase.setUpClass(cls).
913                if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test':
914                    os.environ["LLDB_TEST"] = os.path.join(newrdir, 'test')
915                else:
916                    os.environ["LLDB_TEST"] = newrdir
917
918                # And update the Python search paths for modules.
919                sys.path = [x.replace(rdir, newrdir, 1) for x in old_sys_path]
920
921            # Output the configuration.
922            sys.stderr.write("\nConfiguration: " + configString + "\n")
923
924        #print "sys.stderr name is", sys.stderr.name
925        #print "sys.stdout name is", sys.stdout.name
926
927        # First, write out the number of collected test cases.
928        sys.stderr.write(separator + "\n")
929        sys.stderr.write("Collected %d test%s\n\n"
930                         % (suite.countTestCases(),
931                            suite.countTestCases() != 1 and "s" or ""))
932
933        class LLDBTestResult(unittest2.TextTestResult):
934            """
935            Enforce a singleton pattern to allow introspection of test progress.
936
937            Overwrite addError(), addFailure(), and addExpectedFailure() methods
938            to enable each test instance to track its failure/error status.  It
939            is used in the LLDB test framework to emit detailed trace messages
940            to a log file for easier human inspection of test failres/errors.
941            """
942            __singleton__ = None
943            __ignore_singleton__ = False
944
945            def __init__(self, *args):
946                if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
947                    raise Exception("LLDBTestResult instantiated more than once")
948                super(LLDBTestResult, self).__init__(*args)
949                LLDBTestResult.__singleton__ = self
950                # Now put this singleton into the lldb module namespace.
951                lldb.test_result = self
952                # Computes the format string for displaying the counter.
953                global suite
954                counterWidth = len(str(suite.countTestCases()))
955                self.fmt = "%" + str(counterWidth) + "d: "
956                self.indentation = ' ' * (counterWidth + 2)
957                # This counts from 1 .. suite.countTestCases().
958                self.counter = 0
959
960            def getDescription(self, test):
961                doc_first_line = test.shortDescription()
962                if self.descriptions and doc_first_line:
963                    return '\n'.join((str(test), self.indentation + doc_first_line))
964                else:
965                    return str(test)
966
967            def startTest(self, test):
968                self.counter += 1
969                if self.showAll:
970                    self.stream.write(self.fmt % self.counter)
971                super(LLDBTestResult, self).startTest(test)
972
973            def addError(self, test, err):
974                global sdir_has_content
975                sdir_has_content = True
976                super(LLDBTestResult, self).addError(test, err)
977                method = getattr(test, "markError", None)
978                if method:
979                    method()
980
981            def addFailure(self, test, err):
982                global sdir_has_content
983                sdir_has_content = True
984                super(LLDBTestResult, self).addFailure(test, err)
985                method = getattr(test, "markFailure", None)
986                if method:
987                    method()
988
989            def addExpectedFailure(self, test, err):
990                global sdir_has_content
991                sdir_has_content = True
992                super(LLDBTestResult, self).addExpectedFailure(test, err)
993                method = getattr(test, "markExpectedFailure", None)
994                if method:
995                    method()
996
997            def addUnexpectedSuccess(self, test):
998                global sdir_has_content
999                sdir_has_content = True
1000                super(LLDBTestResult, self).addUnexpectedSuccess(test)
1001                method = getattr(test, "markUnexpectedSuccess", None)
1002                if method:
1003                    method()
1004
1005        # Invoke the test runner.
1006        if count == 1:
1007            result = unittest2.TextTestRunner(stream=sys.stderr,
1008                                              verbosity=verbose,
1009                                              failfast=failfast,
1010                                              resultclass=LLDBTestResult).run(suite)
1011        else:
1012            # We are invoking the same test suite more than once.  In this case,
1013            # mark __ignore_singleton__ flag as True so the signleton pattern is
1014            # not enforced.
1015            LLDBTestResult.__ignore_singleton__ = True
1016            for i in range(count):
1017                result = unittest2.TextTestRunner(stream=sys.stderr,
1018                                                  verbosity=verbose,
1019                                                  failfast=failfast,
1020                                                  resultclass=LLDBTestResult).run(suite)
1021
1022
1023if sdir_has_content:
1024    sys.stderr.write("Session logs for test failures/errors/unexpected successes"
1025                     " can be found in directory '%s'\n" % sdir_name)
1026
1027# Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined.
1028# This should not be necessary now.
1029if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ):
1030    import subprocess
1031    print "Terminating Test suite..."
1032    subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())])
1033
1034# Exiting.
1035sys.exit(not result.wasSuccessful)
1036