dotest.py revision 4f347cbb039d9b5ef5f5772dcb672b5557feeb7e 
1#!/usr/bin/env python
2
3"""
4A simple testing framework for lldb using python's unit testing framework.
5
6Tests for lldb are written as python scripts which take advantage of the script
7bridging provided by LLDB.framework to interact with lldb core.
8
9A specific naming pattern is followed by the .py script to be recognized as
10a module which implements a test scenario, namely, Test*.py.
11
12To specify the directories where "Test*.py" python test scripts are located,
13you need to pass in a list of directory names.  By default, the current
14working directory is searched if nothing is specified on the command line.
15
16Type:
17
18./dotest.py -h
19
20for available options.
21"""
22
23import os, signal, sys, time
24import unittest2
25
26def is_exe(fpath):
27    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
28
29# Find the full path to a program, or return None.
30def which(program):
31    fpath, fname = os.path.split(program)
32    if fpath:
33        if is_exe(program):
34            return program
35    else:
36        for path in os.environ["PATH"].split(os.pathsep):
37            exe_file = os.path.join(path, program)
38            if is_exe(exe_file):
39                return exe_file
40    return None
41
42class _WritelnDecorator(object):
43    """Used to decorate file-like objects with a handy 'writeln' method"""
44    def __init__(self,stream):
45        self.stream = stream
46
47    def __getattr__(self, attr):
48        if attr in ('stream', '__getstate__'):
49            raise AttributeError(attr)
50        return getattr(self.stream,attr)
51
52    def writeln(self, arg=None):
53        if arg:
54            self.write(arg)
55        self.write('\n') # text-mode streams translate to \r\n if needed
56
57#
58# Global variables:
59#
60
61# The test suite.
62suite = unittest2.TestSuite()
63
64# By default, both command line and Python API tests are performed.
65# Use @python_api_test decorator, defined in lldbtest.py, to mark a test as
66# a Python API test.
67dont_do_python_api_test = False
68
69# By default, both command line and Python API tests are performed.
70just_do_python_api_test = False
71
72# The blacklist is optional (-b blacklistFile) and allows a central place to skip
73# testclass's and/or testclass.testmethod's.
74blacklist = None
75
76# The dictionary as a result of sourcing blacklistFile.
77blacklistConfig = {}
78
79# The config file is optional.
80configFile = None
81
82# Test suite repeat count.  Can be overwritten with '-# count'.
83count = 1
84
85# The dictionary as a result of sourcing configFile.
86config = {}
87
88# The 'archs' and 'compilers' can be specified via either command line or configFile,
89# with the command line overriding the configFile.  When specified, they should be
90# of the list type.  For example, "-A x86_64^i386" => archs=['x86_64', 'i386'] and
91# "-C gcc^clang" => compilers=['gcc', 'clang'].
92archs = None
93compilers = None
94
95# Delay startup in order for the debugger to attach.
96delay = False
97
98# Dump the Python sys.path variable.  Use '-D' to dump sys.path.
99dumpSysPath = False
100
101# By default, failfast is False.  Use '-F' to overwrite it.
102failfast = False
103
104# The filter (testclass.testmethod) used to admit tests into our test suite.
105filterspec = None
106
107# If '-g' is specified, the filterspec is not exclusive.  If a test module does
108# not contain testclass.testmethod which matches the filterspec, the whole test
109# module is still admitted into our test suite.  fs4all flag defaults to True.
110fs4all = True
111
112# Ignore the build search path relative to this script to locate the lldb.py module.
113ignore = False
114
115# By default, we skip long running test case.  Use '-l' option to override.
116skipLongRunningTest = True
117
118# The regular expression pattern to match against eligible filenames as our test cases.
119regexp = None
120
121# By default, tests are executed in place and cleanups are performed afterwards.
122# Use '-r dir' option to relocate the tests and their intermediate files to a
123# different directory and to forgo any cleanups.  The directory specified must
124# not exist yet.
125rdir = None
126
127# By default, recorded session info for errored/failed test are dumped into its
128# own file under a session directory named after the timestamp of the test suite
129# run.  Use '-s session-dir-name' to specify a specific dir name.
130sdir_name = None
131
132# Set this flag if there is any session info dumped during the test run.
133sdir_has_content = False
134
135# Default verbosity is 0.
136verbose = 0
137
138# By default, search from the current working directory.
139testdirs = [ os.getcwd() ]
140
141# Separator string.
142separator = '-' * 70
143
144
145def usage():
146    print """
147Usage: dotest.py [option] [args]
148where options:
149-h   : print this help message and exit.  Add '-v' for more detailed help.
150-A   : specify the architecture(s) to launch for the inferior process
151       -A i386 => launch inferior with i386 architecture
152       -A x86_64^i386 => launch inferior with x86_64 and i386 architectures
153-C   : specify the compiler(s) used to build the inferior executable
154       -C clang => build debuggee using clang compiler
155       -C clang^gcc => build debuggee using clang and gcc compilers
156-D   : dump the Python sys.path variable
157-a   : don't do lldb Python API tests
158       use @python_api_test to decorate a test case as lldb Python API test
159+a   : just do lldb Python API tests
160       do not specify both '-a' and '+a' at the same time
161-b   : read a blacklist file specified after this option
162-c   : read a config file specified after this option
163       the architectures and compilers (note the plurals) specified via '-A' and '-C'
164       will override those specified via a config file
165       (see also lldb-trunk/example/test/usage-config)
166-d   : delay startup for 10 seconds (in order for the debugger to attach)
167-F   : failfast, stop the test suite on the first error/failure
168-f   : specify a filter, which consists of the test class name, a dot, followed by
169       the test method, to only admit such test into the test suite
170       e.g., -f 'ClassTypesTestCase.test_with_dwarf_and_python_api'
171-g   : if specified, the filterspec by -f is not exclusive, i.e., if a test module
172       does not match the filterspec (testclass.testmethod), the whole module is
173       still admitted to the test suite
174-i   : ignore (don't bailout) if 'lldb.py' module cannot be located in the build
175       tree relative to this script; use PYTHONPATH to locate the module
176-l   : don't skip long running test
177-p   : specify a regexp filename pattern for inclusion in the test suite
178-r   : specify a dir to relocate the tests and their intermediate files to;
179       the directory must not exist before running this test driver;
180       no cleanup of intermediate test files is performed in this case
181-s   : specify the name of the dir created to store the session files of tests
182       with errored or failed status; if not specified, the test driver uses the
183       timestamp as the session dir name
184-t   : trace lldb command execution and result
185-v   : do verbose mode of unittest framework
186-w   : insert some wait time (currently 0.5 sec) between consecutive test cases
187-#   : Repeat the test suite for a specified number of times
188
189and:
190args : specify a list of directory names to search for test modules named after
191       Test*.py (test discovery)
192       if empty, search from the curret working directory, instead
193"""
194
195    if verbose > 0:
196        print """
197Examples:
198
199This is an example of using the -f option to pinpoint to a specfic test class
200and test method to be run:
201
202$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
203----------------------------------------------------------------------
204Collected 1 test
205
206test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
207Test 'frame variable this' when stopped on a class constructor. ... ok
208
209----------------------------------------------------------------------
210Ran 1 test in 1.396s
211
212OK
213
214And this is an example of using the -p option to run a single file (the filename
215matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
216
217$ ./dotest.py -v -p ObjC
218----------------------------------------------------------------------
219Collected 4 tests
220
221test_break_with_dsym (TestObjCMethods.FoundationTestCase)
222Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
223test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
224Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
225test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
226Lookup objective-c data types and evaluate expressions. ... ok
227test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
228Lookup objective-c data types and evaluate expressions. ... ok
229
230----------------------------------------------------------------------
231Ran 4 tests in 16.661s
232
233OK
234
235Running of this script also sets up the LLDB_TEST environment variable so that
236individual test cases can locate their supporting files correctly.  The script
237tries to set up Python's search paths for modules by looking at the build tree
238relative to this script.  See also the '-i' option in the following example.
239
240Finally, this is an example of using the lldb.py module distributed/installed by
241Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
242option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
243as the architecture and CC=clang to specify the compiler used for the test run:
244
245$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
246
247Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
248----------------------------------------------------------------------
249Collected 2 tests
250
251test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
252Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
253test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
254Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
255
256----------------------------------------------------------------------
257Ran 2 tests in 5.659s
258
259OK
260
261The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
262notify the directory containing the session logs for test failures or errors.
263In case there is any test failure/error, a similar message is appended at the
264end of the stderr output for your convenience.
265
266Environment variables related to loggings:
267
268o LLDB_LOG: if defined, specifies the log file pathname for the 'lldb' subsystem
269  with a default option of 'event process' if LLDB_LOG_OPTION is not defined.
270
271o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the
272  'process.gdb-remote' subsystem with a default option of 'packets' if
273  GDB_REMOTE_LOG_OPTION is not defined.
274"""
275    sys.exit(0)
276
277
278def parseOptionsAndInitTestdirs():
279    """Initialize the list of directories containing our unittest scripts.
280
281    '-h/--help as the first option prints out usage info and exit the program.
282    """
283
284    global dont_do_python_api_test
285    global just_do_python_api_test
286    global blacklist
287    global blacklistConfig
288    global configFile
289    global archs
290    global compilers
291    global count
292    global delay
293    global dumpSysPath
294    global failfast
295    global filterspec
296    global fs4all
297    global ignore
298    global skipLongRunningTest
299    global regexp
300    global rdir
301    global sdir_name
302    global verbose
303    global testdirs
304
305    do_help = False
306
307    if len(sys.argv) == 1:
308        return
309
310    # Process possible trace and/or verbose flag, among other things.
311    index = 1
312    while index < len(sys.argv):
313        if sys.argv[index].startswith('-') or sys.argv[index].startswith('+'):
314            # We should continue processing...
315            pass
316        else:
317            # End of option processing.
318            break
319
320        if sys.argv[index].find('-h') != -1:
321            index += 1
322            do_help = True
323        elif sys.argv[index].startswith('-A'):
324            # Increment by 1 to fetch the ARCH spec.
325            index += 1
326            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
327                usage()
328            archSpec = sys.argv[index]
329            if archSpec.find('^') != -1:
330                archs = archSpec.split('^')
331            else:
332                os.environ["ARCH"] = archSpec
333            index += 1
334        elif sys.argv[index].startswith('-C'):
335            # Increment by 1 to fetch the CC spec.
336            index += 1
337            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
338                usage()
339            ccSpec = sys.argv[index]
340            if ccSpec.find('^') != -1:
341                compilers = ccSpec.split('^')
342            else:
343                os.environ["CC"] = ccSpec
344            index += 1
345        elif sys.argv[index].startswith('-D'):
346            dumpSysPath = True
347            index += 1
348        elif sys.argv[index].startswith('-a'):
349            dont_do_python_api_test = True
350            index += 1
351        elif sys.argv[index].startswith('+a'):
352            just_do_python_api_test = True
353            index += 1
354        elif sys.argv[index].startswith('-b'):
355            # Increment by 1 to fetch the blacklist file name option argument.
356            index += 1
357            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
358                usage()
359            blacklistFile = sys.argv[index]
360            if not os.path.isfile(blacklistFile):
361                print "Blacklist file:", blacklistFile, "does not exist!"
362                usage()
363            index += 1
364            # Now read the blacklist contents and assign it to blacklist.
365            execfile(blacklistFile, globals(), blacklistConfig)
366            blacklist = blacklistConfig.get('blacklist')
367        elif sys.argv[index].startswith('-c'):
368            # Increment by 1 to fetch the config file name option argument.
369            index += 1
370            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
371                usage()
372            configFile = sys.argv[index]
373            if not os.path.isfile(configFile):
374                print "Config file:", configFile, "does not exist!"
375                usage()
376            index += 1
377        elif sys.argv[index].startswith('-d'):
378            delay = True
379            index += 1
380        elif sys.argv[index].startswith('-F'):
381            failfast = True
382            index += 1
383        elif sys.argv[index].startswith('-f'):
384            # Increment by 1 to fetch the filter spec.
385            index += 1
386            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
387                usage()
388            filterspec = sys.argv[index]
389            index += 1
390        elif sys.argv[index].startswith('-g'):
391            fs4all = False
392            index += 1
393        elif sys.argv[index].startswith('-i'):
394            ignore = True
395            index += 1
396        elif sys.argv[index].startswith('-l'):
397            skipLongRunningTest = False
398            index += 1
399        elif sys.argv[index].startswith('-p'):
400            # Increment by 1 to fetch the reg exp pattern argument.
401            index += 1
402            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
403                usage()
404            regexp = sys.argv[index]
405            index += 1
406        elif sys.argv[index].startswith('-r'):
407            # Increment by 1 to fetch the relocated directory argument.
408            index += 1
409            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
410                usage()
411            rdir = os.path.abspath(sys.argv[index])
412            if os.path.exists(rdir):
413                print "Relocated directory:", rdir, "must not exist!"
414                usage()
415            index += 1
416        elif sys.argv[index].startswith('-s'):
417            # Increment by 1 to fetch the session dir name.
418            index += 1
419            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
420                usage()
421            sdir_name = sys.argv[index]
422            index += 1
423        elif sys.argv[index].startswith('-t'):
424            os.environ["LLDB_COMMAND_TRACE"] = "YES"
425            index += 1
426        elif sys.argv[index].startswith('-v'):
427            verbose = 2
428            index += 1
429        elif sys.argv[index].startswith('-w'):
430            os.environ["LLDB_WAIT_BETWEEN_TEST_CASES"] = 'YES'
431            index += 1
432        elif sys.argv[index].startswith('-#'):
433            # Increment by 1 to fetch the repeat count argument.
434            index += 1
435            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
436                usage()
437            count = int(sys.argv[index])
438            index += 1
439        else:
440            print "Unknown option: ", sys.argv[index]
441            usage()
442
443    if do_help == True:
444        usage()
445
446    # Do not specify both '-a' and '+a' at the same time.
447    if dont_do_python_api_test and just_do_python_api_test:
448        usage()
449
450    # Gather all the dirs passed on the command line.
451    if len(sys.argv) > index:
452        testdirs = map(os.path.abspath, sys.argv[index:])
453
454    # If '-r dir' is specified, the tests should be run under the relocated
455    # directory.  Let's copy the testdirs over.
456    if rdir:
457        from shutil import copytree, ignore_patterns
458
459        tmpdirs = []
460        for srcdir in testdirs:
461            dstdir = os.path.join(rdir, os.path.basename(srcdir))
462            # Don't copy the *.pyc and .svn stuffs.
463            copytree(srcdir, dstdir, ignore=ignore_patterns('*.pyc', '.svn'))
464            tmpdirs.append(dstdir)
465
466        # This will be our modified testdirs.
467        testdirs = tmpdirs
468
469        # With '-r dir' specified, there's no cleanup of intermediate test files.
470        os.environ["LLDB_DO_CLEANUP"] = 'NO'
471
472        # If testdirs is ['test'], the make directory has already been copied
473        # recursively and is contained within the rdir/test dir.  For anything
474        # else, we would need to copy over the make directory and its contents,
475        # so that, os.listdir(rdir) looks like, for example:
476        #
477        #     array_types conditional_break make
478        #
479        # where the make directory contains the Makefile.rules file.
480        if len(testdirs) != 1 or os.path.basename(testdirs[0]) != 'test':
481            # Don't copy the .svn stuffs.
482            copytree('make', os.path.join(rdir, 'make'),
483                     ignore=ignore_patterns('.svn'))
484
485    #print "testdirs:", testdirs
486
487    # Source the configFile if specified.
488    # The side effect, if any, will be felt from this point on.  An example
489    # config file may be these simple two lines:
490    #
491    # sys.stderr = open("/tmp/lldbtest-stderr", "w")
492    # sys.stdout = open("/tmp/lldbtest-stdout", "w")
493    #
494    # which will reassign the two file objects to sys.stderr and sys.stdout,
495    # respectively.
496    #
497    # See also lldb-trunk/example/test/usage-config.
498    global config
499    if configFile:
500        # Pass config (a dictionary) as the locals namespace for side-effect.
501        execfile(configFile, globals(), config)
502        #print "config:", config
503        #print "sys.stderr:", sys.stderr
504        #print "sys.stdout:", sys.stdout
505
506
507def setupSysPath():
508    """
509    Add LLDB.framework/Resources/Python to the search paths for modules.
510    As a side effect, we also discover the 'lldb' executable and export it here.
511    """
512
513    global rdir
514    global testdirs
515    global dumpSysPath
516
517    # Get the directory containing the current script.
518    if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
519        scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
520    else:
521        scriptPath = sys.path[0]
522    if not scriptPath.endswith('test'):
523        print "This script expects to reside in lldb's test directory."
524        sys.exit(-1)
525
526    if rdir:
527        # Set up the LLDB_TEST environment variable appropriately, so that the
528        # individual tests can be located relatively.
529        #
530        # See also lldbtest.TestBase.setUpClass(cls).
531        if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test':
532            os.environ["LLDB_TEST"] = os.path.join(rdir, 'test')
533        else:
534            os.environ["LLDB_TEST"] = rdir
535    else:
536        os.environ["LLDB_TEST"] = scriptPath
537    pluginPath = os.path.join(scriptPath, 'plugins')
538    pexpectPath = os.path.join(scriptPath, 'pexpect-2.4')
539
540    # Append script dir, plugin dir, and pexpect dir to the sys.path.
541    sys.path.append(scriptPath)
542    sys.path.append(pluginPath)
543    sys.path.append(pexpectPath)
544
545    # This is our base name component.
546    base = os.path.abspath(os.path.join(scriptPath, os.pardir))
547
548    # These are for xcode build directories.
549    xcode3_build_dir = ['build']
550    xcode4_build_dir = ['build', 'lldb', 'Build', 'Products']
551    dbg = ['Debug']
552    rel = ['Release']
553    bai = ['BuildAndIntegration']
554    python_resource_dir = ['LLDB.framework', 'Resources', 'Python']
555
556    # Some of the tests can invoke the 'lldb' command directly.
557    # We'll try to locate the appropriate executable right here.
558
559    executable = ['lldb']
560    dbgExec  = os.path.join(base, *(xcode3_build_dir + dbg + executable))
561    dbgExec2 = os.path.join(base, *(xcode4_build_dir + dbg + executable))
562    relExec  = os.path.join(base, *(xcode3_build_dir + rel + executable))
563    relExec2 = os.path.join(base, *(xcode4_build_dir + rel + executable))
564    baiExec  = os.path.join(base, *(xcode3_build_dir + bai + executable))
565    baiExec2 = os.path.join(base, *(xcode4_build_dir + bai + executable))
566
567    lldbExec = None
568    if is_exe(dbgExec):
569        lldbExec = dbgExec
570    elif is_exe(dbgExec2):
571        lldbExec = dbgExec2
572    elif is_exe(relExec):
573        lldbExec = relExec
574    elif is_exe(relExec2):
575        lldbExec = relExec2
576    elif is_exe(baiExec):
577        lldbExec = baiExec
578    elif is_exe(baiExec2):
579        lldbExec = baiExec2
580
581    if not lldbExec:
582        lldbExec = which('lldb')
583
584    if not lldbExec:
585        print "The 'lldb' executable cannot be located.  Some of the tests may not be run as a result."
586    else:
587        os.environ["LLDB_EXEC"] = lldbExec
588        #print "The 'lldb' executable path is", lldbExec
589        os.system('%s -v' % lldbExec)
590
591    os.system('svn info %s' % base)
592
593    global ignore
594
595    # The '-i' option is used to skip looking for lldb.py in the build tree.
596    if ignore:
597        return
598
599    dbgPath  = os.path.join(base, *(xcode3_build_dir + dbg + python_resource_dir))
600    dbgPath2 = os.path.join(base, *(xcode4_build_dir + dbg + python_resource_dir))
601    relPath  = os.path.join(base, *(xcode3_build_dir + rel + python_resource_dir))
602    relPath2 = os.path.join(base, *(xcode4_build_dir + rel + python_resource_dir))
603    baiPath  = os.path.join(base, *(xcode3_build_dir + bai + python_resource_dir))
604    baiPath2 = os.path.join(base, *(xcode4_build_dir + bai + python_resource_dir))
605
606    lldbPath = None
607    if os.path.isfile(os.path.join(dbgPath, 'lldb.py')):
608        lldbPath = dbgPath
609    elif os.path.isfile(os.path.join(dbgPath2, 'lldb.py')):
610        lldbPath = dbgPath2
611    elif os.path.isfile(os.path.join(relPath, 'lldb.py')):
612        lldbPath = relPath
613    elif os.path.isfile(os.path.join(relPath2, 'lldb.py')):
614        lldbPath = relPath2
615    elif os.path.isfile(os.path.join(baiPath, 'lldb.py')):
616        lldbPath = baiPath
617    elif os.path.isfile(os.path.join(baiPath2, 'lldb.py')):
618        lldbPath = baiPath2
619
620    if not lldbPath:
621        print 'This script requires lldb.py to be in either ' + dbgPath + ',',
622        print relPath + ', or ' + baiPath
623        sys.exit(-1)
624
625    # This is to locate the lldb.py module.  Insert it right after sys.path[0].
626    sys.path[1:1] = [lldbPath]
627    if dumpSysPath:
628        print "sys.path:", sys.path
629
630
631def doDelay(delta):
632    """Delaying startup for delta-seconds to facilitate debugger attachment."""
633    def alarm_handler(*args):
634        raise Exception("timeout")
635
636    signal.signal(signal.SIGALRM, alarm_handler)
637    signal.alarm(delta)
638    sys.stdout.write("pid=%d\n" % os.getpid())
639    sys.stdout.write("Enter RET to proceed (or timeout after %d seconds):" %
640                     delta)
641    sys.stdout.flush()
642    try:
643        text = sys.stdin.readline()
644    except:
645        text = ""
646    signal.alarm(0)
647    sys.stdout.write("proceeding...\n")
648    pass
649
650
651def visit(prefix, dir, names):
652    """Visitor function for os.path.walk(path, visit, arg)."""
653
654    global suite
655    global regexp
656    global filterspec
657    global fs4all
658
659    for name in names:
660        if os.path.isdir(os.path.join(dir, name)):
661            continue
662
663        if '.py' == os.path.splitext(name)[1] and name.startswith(prefix):
664            # Try to match the regexp pattern, if specified.
665            if regexp:
666                import re
667                if re.search(regexp, name):
668                    #print "Filename: '%s' matches pattern: '%s'" % (name, regexp)
669                    pass
670                else:
671                    #print "Filename: '%s' does not match pattern: '%s'" % (name, regexp)
672                    continue
673
674            # We found a match for our test.  Add it to the suite.
675
676            # Update the sys.path first.
677            if not sys.path.count(dir):
678                sys.path.insert(0, dir)
679            base = os.path.splitext(name)[0]
680
681            # Thoroughly check the filterspec against the base module and admit
682            # the (base, filterspec) combination only when it makes sense.
683            if filterspec:
684                # Optimistically set the flag to True.
685                filtered = True
686                module = __import__(base)
687                parts = filterspec.split('.')
688                obj = module
689                for part in parts:
690                    try:
691                        parent, obj = obj, getattr(obj, part)
692                    except AttributeError:
693                        # The filterspec has failed.
694                        filtered = False
695                        break
696                # Forgo this module if the (base, filterspec) combo is invalid
697                # and no '-g' option is specified
698                if fs4all and not filtered:
699                    continue
700
701            # Add either the filtered test case or the entire test class.
702            if filterspec and filtered:
703                suite.addTests(
704                    unittest2.defaultTestLoader.loadTestsFromName(filterspec, module))
705            else:
706                # A simple case of just the module name.  Also the failover case
707                # from the filterspec branch when the (base, filterspec) combo
708                # doesn't make sense.
709                suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base))
710
711
712def lldbLoggings():
713    """Check and do lldb loggings if necessary."""
714
715    # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is
716    # defined.  Use ${LLDB_LOG} to specify the log file.
717    ci = lldb.DBG.GetCommandInterpreter()
718    res = lldb.SBCommandReturnObject()
719    if ("LLDB_LOG" in os.environ):
720        if ("LLDB_LOG_OPTION" in os.environ):
721            lldb_log_option = os.environ["LLDB_LOG_OPTION"]
722        else:
723            lldb_log_option = "event process expr state api"
724        ci.HandleCommand(
725            "log enable -n -f " + os.environ["LLDB_LOG"] + " lldb " + lldb_log_option,
726            res)
727        if not res.Succeeded():
728            raise Exception('log enable failed (check LLDB_LOG env variable.')
729    # Ditto for gdb-remote logging if ${GDB_REMOTE_LOG} environment variable is defined.
730    # Use ${GDB_REMOTE_LOG} to specify the log file.
731    if ("GDB_REMOTE_LOG" in os.environ):
732        if ("GDB_REMOTE_LOG_OPTION" in os.environ):
733            gdb_remote_log_option = os.environ["GDB_REMOTE_LOG_OPTION"]
734        else:
735            gdb_remote_log_option = "packets process"
736        ci.HandleCommand(
737            "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " process.gdb-remote "
738            + gdb_remote_log_option,
739            res)
740        if not res.Succeeded():
741            raise Exception('log enable failed (check GDB_REMOTE_LOG env variable.')
742
743def getMyCommandLine():
744    import subprocess
745    ps = subprocess.Popen(['ps', '-o', "command=CMD", str(os.getpid())], stdout=subprocess.PIPE).communicate()[0]
746    lines = ps.split('\n')
747    cmd_line = lines[1]
748    return cmd_line
749
750# ======================================== #
751#                                          #
752# Execution of the test driver starts here #
753#                                          #
754# ======================================== #
755
756#
757# Start the actions by first parsing the options while setting up the test
758# directories, followed by setting up the search paths for lldb utilities;
759# then, we walk the directory trees and collect the tests into our test suite.
760#
761parseOptionsAndInitTestdirs()
762setupSysPath()
763
764#
765# If '-d' is specified, do a delay of 10 seconds for the debugger to attach.
766#
767if delay:
768    doDelay(10)
769
770#
771# If '-l' is specified, do not skip the long running tests.
772if not skipLongRunningTest:
773    os.environ["LLDB_SKIP_LONG_RUNNING_TEST"] = "NO"
774
775#
776# Walk through the testdirs while collecting tests.
777#
778for testdir in testdirs:
779    os.path.walk(testdir, visit, 'Test')
780
781#
782# Now that we have loaded all the test cases, run the whole test suite.
783#
784
785# For the time being, let's bracket the test runner within the
786# lldb.SBDebugger.Initialize()/Terminate() pair.
787import lldb, atexit
788# Update: the act of importing lldb now executes lldb.SBDebugger.Initialize(),
789# there's no need to call it a second time.
790#lldb.SBDebugger.Initialize()
791atexit.register(lambda: lldb.SBDebugger.Terminate())
792
793# Create a singleton SBDebugger in the lldb namespace.
794lldb.DBG = lldb.SBDebugger.Create()
795
796# Put the blacklist in the lldb namespace, to be used by lldb.TestBase.
797lldb.blacklist = blacklist
798
799# Put dont/just_do_python_api_test in the lldb namespace, too.
800lldb.dont_do_python_api_test = dont_do_python_api_test
801lldb.just_do_python_api_test = just_do_python_api_test
802
803# Turn on lldb loggings if necessary.
804lldbLoggings()
805
806# Install the control-c handler.
807unittest2.signals.installHandler()
808
809# If sdir_name is not specified through the '-s sdir_name' option, get a
810# timestamp string and export it as LLDB_SESSION_DIR environment var.  This will
811# be used when/if we want to dump the session info of individual test cases
812# later on.
813#
814# See also TestBase.dumpSessionInfo() in lldbtest.py.
815if not sdir_name:
816    import datetime
817    # The windows platforms don't like ':' in the pathname.
818    timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
819    sdir_name = timestamp
820os.environ["LLDB_SESSION_DIRNAME"] = sdir_name
821
822sys.stderr.write("\nSession logs for test failures/errors will go into directory '%s'\n" % sdir_name)
823sys.stderr.write("Command invoked: %s\n" % getMyCommandLine())
824
825#
826# Invoke the default TextTestRunner to run the test suite, possibly iterating
827# over different configurations.
828#
829
830iterArchs = False
831iterCompilers = False
832
833if not archs and "archs" in config:
834    archs = config["archs"]
835
836if isinstance(archs, list) and len(archs) >= 1:
837    iterArchs = True
838
839if not compilers and "compilers" in config:
840    compilers = config["compilers"]
841
842if isinstance(compilers, list) and len(compilers) >= 1:
843    iterCompilers = True
844
845# Make a shallow copy of sys.path, we need to manipulate the search paths later.
846# This is only necessary if we are relocated and with different configurations.
847if rdir:
848    old_sys_path = sys.path[:]
849# If we iterate on archs or compilers, there is a chance we want to split stderr/stdout.
850if iterArchs or iterCompilers:
851    old_stderr = sys.stderr
852    old_stdout = sys.stdout
853    new_stderr = None
854    new_stdout = None
855
856# Iterating over all possible architecture and compiler combinations.
857for ia in range(len(archs) if iterArchs else 1):
858    archConfig = ""
859    if iterArchs:
860        os.environ["ARCH"] = archs[ia]
861        archConfig = "arch=%s" % archs[ia]
862    for ic in range(len(compilers) if iterCompilers else 1):
863        if iterCompilers:
864            os.environ["CC"] = compilers[ic]
865            configString = "%s compiler=%s" % (archConfig, compilers[ic])
866        else:
867            configString = archConfig
868
869        if iterArchs or iterCompilers:
870            # Translate ' ' to '-' for pathname component.
871            from string import maketrans
872            tbl = maketrans(' ', '-')
873            configPostfix = configString.translate(tbl)
874
875            # Check whether we need to split stderr/stdout into configuration
876            # specific files.
877            if old_stderr.name != '<stderr>' and config.get('split_stderr'):
878                if new_stderr:
879                    new_stderr.close()
880                new_stderr = open("%s.%s" % (old_stderr.name, configPostfix), "w")
881                sys.stderr = new_stderr
882            if old_stdout.name != '<stdout>' and config.get('split_stdout'):
883                if new_stdout:
884                    new_stdout.close()
885                new_stdout = open("%s.%s" % (old_stdout.name, configPostfix), "w")
886                sys.stdout = new_stdout
887
888            # If we specified a relocated directory to run the test suite, do
889            # the extra housekeeping to copy the testdirs to a configStringified
890            # directory and to update sys.path before invoking the test runner.
891            # The purpose is to separate the configuration-specific directories
892            # from each other.
893            if rdir:
894                from shutil import copytree, ignore_patterns
895
896                newrdir = "%s.%s" % (rdir, configPostfix)
897
898                # Copy the tree to a new directory with postfix name configPostfix.
899                copytree(rdir, newrdir, ignore=ignore_patterns('*.pyc', '*.o', '*.d'))
900
901               # Update the LLDB_TEST environment variable to reflect new top
902                # level test directory.
903                #
904                # See also lldbtest.TestBase.setUpClass(cls).
905                if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test':
906                    os.environ["LLDB_TEST"] = os.path.join(newrdir, 'test')
907                else:
908                    os.environ["LLDB_TEST"] = newrdir
909
910                # And update the Python search paths for modules.
911                sys.path = [x.replace(rdir, newrdir, 1) for x in old_sys_path]
912
913            # Output the configuration.
914            sys.stderr.write("\nConfiguration: " + configString + "\n")
915
916        #print "sys.stderr name is", sys.stderr.name
917        #print "sys.stdout name is", sys.stdout.name
918
919        # First, write out the number of collected test cases.
920        sys.stderr.write(separator + "\n")
921        sys.stderr.write("Collected %d test%s\n\n"
922                         % (suite.countTestCases(),
923                            suite.countTestCases() != 1 and "s" or ""))
924
925        class LLDBTestResult(unittest2.TextTestResult):
926            """
927            Enforce a singleton pattern to allow introspection of test progress.
928
929            Overwrite addError(), addFailure(), and addExpectedFailure() methods
930            to enable each test instance to track its failure/error status.  It
931            is used in the LLDB test framework to emit detailed trace messages
932            to a log file for easier human inspection of test failres/errors.
933            """
934            __singleton__ = None
935            __ignore_singleton__ = False
936
937            def __init__(self, *args):
938                if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
939                    raise Exception("LLDBTestResult instantiated more than once")
940                super(LLDBTestResult, self).__init__(*args)
941                LLDBTestResult.__singleton__ = self
942                # Now put this singleton into the lldb module namespace.
943                lldb.test_result = self
944                # Computes the format string for displaying the counter.
945                global suite
946                counterWidth = len(str(suite.countTestCases()))
947                self.fmt = "%" + str(counterWidth) + "d: "
948                self.indentation = ' ' * (counterWidth + 2)
949                # This counts from 1 .. suite.countTestCases().
950                self.counter = 0
951
952            def getDescription(self, test):
953                doc_first_line = test.shortDescription()
954                if self.descriptions and doc_first_line:
955                    return '\n'.join((str(test), self.indentation + doc_first_line))
956                else:
957                    return str(test)
958
959            def startTest(self, test):
960                self.counter += 1
961                if self.showAll:
962                    self.stream.write(self.fmt % self.counter)
963                super(LLDBTestResult, self).startTest(test)
964
965            def addError(self, test, err):
966                global sdir_has_content
967                sdir_has_content = True
968                super(LLDBTestResult, self).addError(test, err)
969                method = getattr(test, "markError", None)
970                if method:
971                    method()
972
973            def addFailure(self, test, err):
974                global sdir_has_content
975                sdir_has_content = True
976                super(LLDBTestResult, self).addFailure(test, err)
977                method = getattr(test, "markFailure", None)
978                if method:
979                    method()
980
981            def addExpectedFailure(self, test, err):
982                global sdir_has_content
983                sdir_has_content = True
984                super(LLDBTestResult, self).addExpectedFailure(test, err)
985                method = getattr(test, "markExpectedFailure", None)
986                if method:
987                    method()
988
989        # Invoke the test runner.
990        if count == 1:
991            result = unittest2.TextTestRunner(stream=sys.stderr,
992                                              verbosity=verbose,
993                                              failfast=failfast,
994                                              resultclass=LLDBTestResult).run(suite)
995        else:
996            # We are invoking the same test suite more than once.  In this case,
997            # mark __ignore_singleton__ flag as True so the signleton pattern is
998            # not enforced.
999            LLDBTestResult.__ignore_singleton__ = True
1000            for i in range(count):
1001                result = unittest2.TextTestRunner(stream=sys.stderr,
1002                                                  verbosity=verbose,
1003                                                  failfast=failfast,
1004                                                  resultclass=LLDBTestResult).run(suite)
1005
1006
1007if sdir_has_content:
1008    sys.stderr.write("Session logs for test failures/errors can be found in directory '%s'\n" % sdir_name)
1009
1010# Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined.
1011# This should not be necessary now.
1012if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ):
1013    import subprocess
1014    print "Terminating Test suite..."
1015    subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())])
1016
1017# Exiting.
1018sys.exit(not result.wasSuccessful)
1019