dotest.py revision c5fa005707f9378e03328fb0ee492b8a6c230057 
1#!/usr/bin/env python
2
3"""
4A simple testing framework for lldb using python's unit testing framework.
5
6Tests for lldb are written as python scripts which take advantage of the script
7bridging provided by LLDB.framework to interact with lldb core.
8
9A specific naming pattern is followed by the .py script to be recognized as
10a module which implements a test scenario, namely, Test*.py.
11
12To specify the directories where "Test*.py" python test scripts are located,
13you need to pass in a list of directory names.  By default, the current
14working directory is searched if nothing is specified on the command line.
15
16Type:
17
18./dotest.py -h
19
20for available options.
21"""
22
23import os, signal, sys, time
24import unittest2
25
26def is_exe(fpath):
27    """Returns true if fpath is an executable."""
28    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
29
30def which(program):
31    """Returns the full path to a program; None otherwise."""
32    fpath, fname = os.path.split(program)
33    if fpath:
34        if is_exe(program):
35            return program
36    else:
37        for path in os.environ["PATH"].split(os.pathsep):
38            exe_file = os.path.join(path, program)
39            if is_exe(exe_file):
40                return exe_file
41    return None
42
43class _WritelnDecorator(object):
44    """Used to decorate file-like objects with a handy 'writeln' method"""
45    def __init__(self,stream):
46        self.stream = stream
47
48    def __getattr__(self, attr):
49        if attr in ('stream', '__getstate__'):
50            raise AttributeError(attr)
51        return getattr(self.stream,attr)
52
53    def writeln(self, arg=None):
54        if arg:
55            self.write(arg)
56        self.write('\n') # text-mode streams translate to \r\n if needed
57
58#
59# Global variables:
60#
61
62# The test suite.
63suite = unittest2.TestSuite()
64
65# By default, both command line and Python API tests are performed.
66# Use @python_api_test decorator, defined in lldbtest.py, to mark a test as
67# a Python API test.
68dont_do_python_api_test = False
69
70# By default, both command line and Python API tests are performed.
71just_do_python_api_test = False
72
73# The blacklist is optional (-b blacklistFile) and allows a central place to skip
74# testclass's and/or testclass.testmethod's.
75blacklist = None
76
77# The dictionary as a result of sourcing blacklistFile.
78blacklistConfig = {}
79
80# The config file is optional.
81configFile = None
82
83# Test suite repeat count.  Can be overwritten with '-# count'.
84count = 1
85
86# The dictionary as a result of sourcing configFile.
87config = {}
88
89# The 'archs' and 'compilers' can be specified via either command line or configFile,
90# with the command line overriding the configFile.  When specified, they should be
91# of the list type.  For example, "-A x86_64^i386" => archs=['x86_64', 'i386'] and
92# "-C gcc^clang" => compilers=['gcc', 'clang'].
93archs = None
94compilers = None
95
96# Delay startup in order for the debugger to attach.
97delay = False
98
99# Dump the Python sys.path variable.  Use '-D' to dump sys.path.
100dumpSysPath = False
101
102# By default, failfast is False.  Use '-F' to overwrite it.
103failfast = False
104
105# The filters (testclass.testmethod) used to admit tests into our test suite.
106filters = []
107
108# If '-g' is specified, the filterspec is not exclusive.  If a test module does
109# not contain testclass.testmethod which matches the filterspec, the whole test
110# module is still admitted into our test suite.  fs4all flag defaults to True.
111fs4all = True
112
113# Ignore the build search path relative to this script to locate the lldb.py module.
114ignore = False
115
116# By default, we skip long running test case.  Use '-l' option to override.
117skipLongRunningTest = True
118
119# The regular expression pattern to match against eligible filenames as our test cases.
120regexp = None
121
122# By default, tests are executed in place and cleanups are performed afterwards.
123# Use '-r dir' option to relocate the tests and their intermediate files to a
124# different directory and to forgo any cleanups.  The directory specified must
125# not exist yet.
126rdir = None
127
128# By default, recorded session info for errored/failed test are dumped into its
129# own file under a session directory named after the timestamp of the test suite
130# run.  Use '-s session-dir-name' to specify a specific dir name.
131sdir_name = None
132
133# Set this flag if there is any session info dumped during the test run.
134sdir_has_content = False
135
136# svn_info stores the output from 'svn info lldb.base.dir'.
137svn_info = ''
138
139# Default verbosity is 0.
140verbose = 0
141
142# By default, search from the script directory.
143testdirs = [ sys.path[0] ]
144
145# Separator string.
146separator = '-' * 70
147
148
149def usage():
150    print """
151Usage: dotest.py [option] [args]
152where options:
153-h   : print this help message and exit.  Add '-v' for more detailed help.
154-A   : specify the architecture(s) to launch for the inferior process
155       -A i386 => launch inferior with i386 architecture
156       -A x86_64^i386 => launch inferior with x86_64 and i386 architectures
157-C   : specify the compiler(s) used to build the inferior executable
158       -C clang => build debuggee using clang compiler
159       -C clang^gcc => build debuggee using clang and gcc compilers
160-D   : dump the Python sys.path variable
161-a   : don't do lldb Python API tests
162       use @python_api_test to decorate a test case as lldb Python API test
163+a   : just do lldb Python API tests
164       do not specify both '-a' and '+a' at the same time
165-b   : read a blacklist file specified after this option
166-c   : read a config file specified after this option
167       the architectures and compilers (note the plurals) specified via '-A' and '-C'
168       will override those specified via a config file
169       (see also lldb-trunk/example/test/usage-config)
170-d   : delay startup for 10 seconds (in order for the debugger to attach)
171-F   : failfast, stop the test suite on the first error/failure
172-f   : specify a filter, which consists of the test class name, a dot, followed by
173       the test method, to only admit such test into the test suite
174       e.g., -f 'ClassTypesTestCase.test_with_dwarf_and_python_api'
175-g   : if specified, the filterspec by -f is not exclusive, i.e., if a test module
176       does not match the filterspec (testclass.testmethod), the whole module is
177       still admitted to the test suite
178-i   : ignore (don't bailout) if 'lldb.py' module cannot be located in the build
179       tree relative to this script; use PYTHONPATH to locate the module
180-l   : don't skip long running test
181-p   : specify a regexp filename pattern for inclusion in the test suite
182-r   : specify a dir to relocate the tests and their intermediate files to;
183       the directory must not exist before running this test driver;
184       no cleanup of intermediate test files is performed in this case
185-s   : specify the name of the dir created to store the session files of tests
186       with errored or failed status; if not specified, the test driver uses the
187       timestamp as the session dir name
188-t   : turn on tracing of lldb command and other detailed test executions
189-v   : do verbose mode of unittest framework (print out each test case invocation)
190-w   : insert some wait time (currently 0.5 sec) between consecutive test cases
191-#   : Repeat the test suite for a specified number of times
192
193and:
194args : specify a list of directory names to search for test modules named after
195       Test*.py (test discovery)
196       if empty, search from the current working directory, instead
197"""
198
199    if verbose > 0:
200        print """
201Examples:
202
203This is an example of using the -f option to pinpoint to a specfic test class
204and test method to be run:
205
206$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
207----------------------------------------------------------------------
208Collected 1 test
209
210test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
211Test 'frame variable this' when stopped on a class constructor. ... ok
212
213----------------------------------------------------------------------
214Ran 1 test in 1.396s
215
216OK
217
218And this is an example of using the -p option to run a single file (the filename
219matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
220
221$ ./dotest.py -v -p ObjC
222----------------------------------------------------------------------
223Collected 4 tests
224
225test_break_with_dsym (TestObjCMethods.FoundationTestCase)
226Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
227test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
228Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
229test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
230Lookup objective-c data types and evaluate expressions. ... ok
231test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
232Lookup objective-c data types and evaluate expressions. ... ok
233
234----------------------------------------------------------------------
235Ran 4 tests in 16.661s
236
237OK
238
239Running of this script also sets up the LLDB_TEST environment variable so that
240individual test cases can locate their supporting files correctly.  The script
241tries to set up Python's search paths for modules by looking at the build tree
242relative to this script.  See also the '-i' option in the following example.
243
244Finally, this is an example of using the lldb.py module distributed/installed by
245Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
246option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
247as the architecture and CC=clang to specify the compiler used for the test run:
248
249$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
250
251Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
252----------------------------------------------------------------------
253Collected 2 tests
254
255test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
256Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
257test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
258Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
259
260----------------------------------------------------------------------
261Ran 2 tests in 5.659s
262
263OK
264
265The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
266notify the directory containing the session logs for test failures or errors.
267In case there is any test failure/error, a similar message is appended at the
268end of the stderr output for your convenience.
269
270Environment variables related to loggings:
271
272o LLDB_LOG: if defined, specifies the log file pathname for the 'lldb' subsystem
273  with a default option of 'event process' if LLDB_LOG_OPTION is not defined.
274
275o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the
276  'process.gdb-remote' subsystem with a default option of 'packets' if
277  GDB_REMOTE_LOG_OPTION is not defined.
278"""
279    sys.exit(0)
280
281
282def parseOptionsAndInitTestdirs():
283    """Initialize the list of directories containing our unittest scripts.
284
285    '-h/--help as the first option prints out usage info and exit the program.
286    """
287
288    global dont_do_python_api_test
289    global just_do_python_api_test
290    global blacklist
291    global blacklistConfig
292    global configFile
293    global archs
294    global compilers
295    global count
296    global delay
297    global dumpSysPath
298    global failfast
299    global filters
300    global fs4all
301    global ignore
302    global skipLongRunningTest
303    global regexp
304    global rdir
305    global sdir_name
306    global verbose
307    global testdirs
308
309    do_help = False
310
311    if len(sys.argv) == 1:
312        return
313
314    # Process possible trace and/or verbose flag, among other things.
315    index = 1
316    while index < len(sys.argv):
317        if sys.argv[index].startswith('-') or sys.argv[index].startswith('+'):
318            # We should continue processing...
319            pass
320        else:
321            # End of option processing.
322            break
323
324        if sys.argv[index].find('-h') != -1:
325            index += 1
326            do_help = True
327        elif sys.argv[index].startswith('-A'):
328            # Increment by 1 to fetch the ARCH spec.
329            index += 1
330            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
331                usage()
332            archs = sys.argv[index].split('^')
333            index += 1
334        elif sys.argv[index].startswith('-C'):
335            # Increment by 1 to fetch the CC spec.
336            index += 1
337            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
338                usage()
339            compilers = sys.argv[index].split('^')
340            index += 1
341        elif sys.argv[index].startswith('-D'):
342            dumpSysPath = True
343            index += 1
344        elif sys.argv[index].startswith('-a'):
345            dont_do_python_api_test = True
346            index += 1
347        elif sys.argv[index].startswith('+a'):
348            just_do_python_api_test = True
349            index += 1
350        elif sys.argv[index].startswith('-b'):
351            # Increment by 1 to fetch the blacklist file name option argument.
352            index += 1
353            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
354                usage()
355            blacklistFile = sys.argv[index]
356            if not os.path.isfile(blacklistFile):
357                print "Blacklist file:", blacklistFile, "does not exist!"
358                usage()
359            index += 1
360            # Now read the blacklist contents and assign it to blacklist.
361            execfile(blacklistFile, globals(), blacklistConfig)
362            blacklist = blacklistConfig.get('blacklist')
363        elif sys.argv[index].startswith('-c'):
364            # Increment by 1 to fetch the config file name option argument.
365            index += 1
366            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
367                usage()
368            configFile = sys.argv[index]
369            if not os.path.isfile(configFile):
370                print "Config file:", configFile, "does not exist!"
371                usage()
372            index += 1
373        elif sys.argv[index].startswith('-d'):
374            delay = True
375            index += 1
376        elif sys.argv[index].startswith('-F'):
377            failfast = True
378            index += 1
379        elif sys.argv[index].startswith('-f'):
380            # Increment by 1 to fetch the filter spec.
381            index += 1
382            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
383                usage()
384            filters.append(sys.argv[index])
385            index += 1
386        elif sys.argv[index].startswith('-g'):
387            fs4all = False
388            index += 1
389        elif sys.argv[index].startswith('-i'):
390            ignore = True
391            index += 1
392        elif sys.argv[index].startswith('-l'):
393            skipLongRunningTest = False
394            index += 1
395        elif sys.argv[index].startswith('-p'):
396            # Increment by 1 to fetch the reg exp pattern argument.
397            index += 1
398            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
399                usage()
400            regexp = sys.argv[index]
401            index += 1
402        elif sys.argv[index].startswith('-r'):
403            # Increment by 1 to fetch the relocated directory argument.
404            index += 1
405            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
406                usage()
407            rdir = os.path.abspath(sys.argv[index])
408            if os.path.exists(rdir):
409                print "Relocated directory:", rdir, "must not exist!"
410                usage()
411            index += 1
412        elif sys.argv[index].startswith('-s'):
413            # Increment by 1 to fetch the session dir name.
414            index += 1
415            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
416                usage()
417            sdir_name = sys.argv[index]
418            index += 1
419        elif sys.argv[index].startswith('-t'):
420            os.environ["LLDB_COMMAND_TRACE"] = "YES"
421            index += 1
422        elif sys.argv[index].startswith('-v'):
423            verbose = 2
424            index += 1
425        elif sys.argv[index].startswith('-w'):
426            os.environ["LLDB_WAIT_BETWEEN_TEST_CASES"] = 'YES'
427            index += 1
428        elif sys.argv[index].startswith('-#'):
429            # Increment by 1 to fetch the repeat count argument.
430            index += 1
431            if index >= len(sys.argv) or sys.argv[index].startswith('-'):
432                usage()
433            count = int(sys.argv[index])
434            index += 1
435        else:
436            print "Unknown option: ", sys.argv[index]
437            usage()
438
439    if do_help == True:
440        usage()
441
442    # Do not specify both '-a' and '+a' at the same time.
443    if dont_do_python_api_test and just_do_python_api_test:
444        usage()
445
446    # Gather all the dirs passed on the command line.
447    if len(sys.argv) > index:
448        testdirs = map(os.path.abspath, sys.argv[index:])
449
450    # If '-r dir' is specified, the tests should be run under the relocated
451    # directory.  Let's copy the testdirs over.
452    if rdir:
453        from shutil import copytree, ignore_patterns
454
455        tmpdirs = []
456        for srcdir in testdirs:
457            dstdir = os.path.join(rdir, os.path.basename(srcdir))
458            # Don't copy the *.pyc and .svn stuffs.
459            copytree(srcdir, dstdir, ignore=ignore_patterns('*.pyc', '.svn'))
460            tmpdirs.append(dstdir)
461
462        # This will be our modified testdirs.
463        testdirs = tmpdirs
464
465        # With '-r dir' specified, there's no cleanup of intermediate test files.
466        os.environ["LLDB_DO_CLEANUP"] = 'NO'
467
468        # If testdirs is ['test'], the make directory has already been copied
469        # recursively and is contained within the rdir/test dir.  For anything
470        # else, we would need to copy over the make directory and its contents,
471        # so that, os.listdir(rdir) looks like, for example:
472        #
473        #     array_types conditional_break make
474        #
475        # where the make directory contains the Makefile.rules file.
476        if len(testdirs) != 1 or os.path.basename(testdirs[0]) != 'test':
477            # Don't copy the .svn stuffs.
478            copytree('make', os.path.join(rdir, 'make'),
479                     ignore=ignore_patterns('.svn'))
480
481    #print "testdirs:", testdirs
482
483    # Source the configFile if specified.
484    # The side effect, if any, will be felt from this point on.  An example
485    # config file may be these simple two lines:
486    #
487    # sys.stderr = open("/tmp/lldbtest-stderr", "w")
488    # sys.stdout = open("/tmp/lldbtest-stdout", "w")
489    #
490    # which will reassign the two file objects to sys.stderr and sys.stdout,
491    # respectively.
492    #
493    # See also lldb-trunk/example/test/usage-config.
494    global config
495    if configFile:
496        # Pass config (a dictionary) as the locals namespace for side-effect.
497        execfile(configFile, globals(), config)
498        #print "config:", config
499        #print "sys.stderr:", sys.stderr
500        #print "sys.stdout:", sys.stdout
501
502
503def setupSysPath():
504    """
505    Add LLDB.framework/Resources/Python to the search paths for modules.
506    As a side effect, we also discover the 'lldb' executable and export it here.
507    """
508
509    global rdir
510    global testdirs
511    global dumpSysPath
512    global svn_info
513
514    # Get the directory containing the current script.
515    if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
516        scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
517    else:
518        scriptPath = sys.path[0]
519    if not scriptPath.endswith('test'):
520        print "This script expects to reside in lldb's test directory."
521        sys.exit(-1)
522
523    if rdir:
524        # Set up the LLDB_TEST environment variable appropriately, so that the
525        # individual tests can be located relatively.
526        #
527        # See also lldbtest.TestBase.setUpClass(cls).
528        if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test':
529            os.environ["LLDB_TEST"] = os.path.join(rdir, 'test')
530        else:
531            os.environ["LLDB_TEST"] = rdir
532    else:
533        os.environ["LLDB_TEST"] = scriptPath
534
535    # Set up the LLDB_SRC environment variable, so that the tests can locate
536    # the LLDB source code.
537    os.environ["LLDB_SRC"] = os.path.join(sys.path[0], os.pardir)
538
539    pluginPath = os.path.join(scriptPath, 'plugins')
540    pexpectPath = os.path.join(scriptPath, 'pexpect-2.4')
541
542    # Append script dir, plugin dir, and pexpect dir to the sys.path.
543    sys.path.append(scriptPath)
544    sys.path.append(pluginPath)
545    sys.path.append(pexpectPath)
546
547    # This is our base name component.
548    base = os.path.abspath(os.path.join(scriptPath, os.pardir))
549
550    # These are for xcode build directories.
551    xcode3_build_dir = ['build']
552    xcode4_build_dir = ['build', 'lldb', 'Build', 'Products']
553    dbg = ['Debug']
554    rel = ['Release']
555    bai = ['BuildAndIntegration']
556    python_resource_dir = ['LLDB.framework', 'Resources', 'Python']
557
558    # Some of the tests can invoke the 'lldb' command directly.
559    # We'll try to locate the appropriate executable right here.
560
561    executable = ['lldb']
562    dbgExec  = os.path.join(base, *(xcode3_build_dir + dbg + executable))
563    dbgExec2 = os.path.join(base, *(xcode4_build_dir + dbg + executable))
564    relExec  = os.path.join(base, *(xcode3_build_dir + rel + executable))
565    relExec2 = os.path.join(base, *(xcode4_build_dir + rel + executable))
566    baiExec  = os.path.join(base, *(xcode3_build_dir + bai + executable))
567    baiExec2 = os.path.join(base, *(xcode4_build_dir + bai + executable))
568
569    lldbExec = None
570    if is_exe(dbgExec):
571        lldbExec = dbgExec
572    elif is_exe(dbgExec2):
573        lldbExec = dbgExec2
574    elif is_exe(relExec):
575        lldbExec = relExec
576    elif is_exe(relExec2):
577        lldbExec = relExec2
578    elif is_exe(baiExec):
579        lldbExec = baiExec
580    elif is_exe(baiExec2):
581        lldbExec = baiExec2
582
583    if not lldbExec:
584        lldbExec = which('lldb')
585
586    if not lldbExec:
587        print "The 'lldb' executable cannot be located.  Some of the tests may not be run as a result."
588    else:
589        os.environ["LLDB_EXEC"] = lldbExec
590        #print "The 'lldb' executable path is", lldbExec
591        os.system('%s -v' % lldbExec)
592
593    import subprocess
594    if os.path.isdir(os.path.join(base, '.svn')):
595        pipe = subprocess.Popen(["svn", "info", base], stdout = subprocess.PIPE)
596        svn_info = pipe.stdout.read()
597    elif os.path.isdir(os.path.join(base, '.git')):
598        pipe = subprocess.Popen(["git", "svn", "info", base], stdout = subprocess.PIPE)
599        svn_info = pipe.stdout.read()
600    print svn_info
601
602    global ignore
603
604    # The '-i' option is used to skip looking for lldb.py in the build tree.
605    if ignore:
606        return
607
608    dbgPath  = os.path.join(base, *(xcode3_build_dir + dbg + python_resource_dir))
609    dbgPath2 = os.path.join(base, *(xcode4_build_dir + dbg + python_resource_dir))
610    relPath  = os.path.join(base, *(xcode3_build_dir + rel + python_resource_dir))
611    relPath2 = os.path.join(base, *(xcode4_build_dir + rel + python_resource_dir))
612    baiPath  = os.path.join(base, *(xcode3_build_dir + bai + python_resource_dir))
613    baiPath2 = os.path.join(base, *(xcode4_build_dir + bai + python_resource_dir))
614
615    lldbPath = None
616    if os.path.isfile(os.path.join(dbgPath, 'lldb.py')):
617        lldbPath = dbgPath
618    elif os.path.isfile(os.path.join(dbgPath2, 'lldb.py')):
619        lldbPath = dbgPath2
620    elif os.path.isfile(os.path.join(relPath, 'lldb.py')):
621        lldbPath = relPath
622    elif os.path.isfile(os.path.join(relPath2, 'lldb.py')):
623        lldbPath = relPath2
624    elif os.path.isfile(os.path.join(baiPath, 'lldb.py')):
625        lldbPath = baiPath
626    elif os.path.isfile(os.path.join(baiPath2, 'lldb.py')):
627        lldbPath = baiPath2
628
629    if not lldbPath:
630        print 'This script requires lldb.py to be in either ' + dbgPath + ',',
631        print relPath + ', or ' + baiPath
632        sys.exit(-1)
633
634    # This is to locate the lldb.py module.  Insert it right after sys.path[0].
635    sys.path[1:1] = [lldbPath]
636    if dumpSysPath:
637        print "sys.path:", sys.path
638
639
640def doDelay(delta):
641    """Delaying startup for delta-seconds to facilitate debugger attachment."""
642    def alarm_handler(*args):
643        raise Exception("timeout")
644
645    signal.signal(signal.SIGALRM, alarm_handler)
646    signal.alarm(delta)
647    sys.stdout.write("pid=%d\n" % os.getpid())
648    sys.stdout.write("Enter RET to proceed (or timeout after %d seconds):" %
649                     delta)
650    sys.stdout.flush()
651    try:
652        text = sys.stdin.readline()
653    except:
654        text = ""
655    signal.alarm(0)
656    sys.stdout.write("proceeding...\n")
657    pass
658
659
660def visit(prefix, dir, names):
661    """Visitor function for os.path.walk(path, visit, arg)."""
662
663    global suite
664    global regexp
665    global filters
666    global fs4all
667
668    for name in names:
669        if os.path.isdir(os.path.join(dir, name)):
670            continue
671
672        if '.py' == os.path.splitext(name)[1] and name.startswith(prefix):
673            # Try to match the regexp pattern, if specified.
674            if regexp:
675                import re
676                if re.search(regexp, name):
677                    #print "Filename: '%s' matches pattern: '%s'" % (name, regexp)
678                    pass
679                else:
680                    #print "Filename: '%s' does not match pattern: '%s'" % (name, regexp)
681                    continue
682
683            # We found a match for our test.  Add it to the suite.
684
685            # Update the sys.path first.
686            if not sys.path.count(dir):
687                sys.path.insert(0, dir)
688            base = os.path.splitext(name)[0]
689
690            # Thoroughly check the filterspec against the base module and admit
691            # the (base, filterspec) combination only when it makes sense.
692            filterspec = None
693            for filterspec in filters:
694                # Optimistically set the flag to True.
695                filtered = True
696                module = __import__(base)
697                parts = filterspec.split('.')
698                obj = module
699                for part in parts:
700                    try:
701                        parent, obj = obj, getattr(obj, part)
702                    except AttributeError:
703                        # The filterspec has failed.
704                        filtered = False
705                        break
706
707                # If we reach here, we have a good filterspec.  Add it.
708                if filtered:
709                    break
710
711            # Forgo this module if the (base, filterspec) combo is invalid
712            # and no '-g' option is specified
713            if filters and fs4all and not filtered:
714                continue
715
716            # Add either the filtered test case or the entire test class.
717            if filterspec and filtered:
718                #print "adding filter spec %s to module %s" % (filterspec, module)
719                suite.addTests(
720                    unittest2.defaultTestLoader.loadTestsFromName(filterspec, module))
721            else:
722                # A simple case of just the module name.  Also the failover case
723                # from the filterspec branch when the (base, filterspec) combo
724                # doesn't make sense.
725                suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base))
726
727
728def lldbLoggings():
729    """Check and do lldb loggings if necessary."""
730
731    # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is
732    # defined.  Use ${LLDB_LOG} to specify the log file.
733    ci = lldb.DBG.GetCommandInterpreter()
734    res = lldb.SBCommandReturnObject()
735    if ("LLDB_LOG" in os.environ):
736        if ("LLDB_LOG_OPTION" in os.environ):
737            lldb_log_option = os.environ["LLDB_LOG_OPTION"]
738        else:
739            lldb_log_option = "event process expr state api"
740        ci.HandleCommand(
741            "log enable -n -f " + os.environ["LLDB_LOG"] + " lldb " + lldb_log_option,
742            res)
743        if not res.Succeeded():
744            raise Exception('log enable failed (check LLDB_LOG env variable.')
745    # Ditto for gdb-remote logging if ${GDB_REMOTE_LOG} environment variable is defined.
746    # Use ${GDB_REMOTE_LOG} to specify the log file.
747    if ("GDB_REMOTE_LOG" in os.environ):
748        if ("GDB_REMOTE_LOG_OPTION" in os.environ):
749            gdb_remote_log_option = os.environ["GDB_REMOTE_LOG_OPTION"]
750        else:
751            gdb_remote_log_option = "packets process"
752        ci.HandleCommand(
753            "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " gdb-remote "
754            + gdb_remote_log_option,
755            res)
756        if not res.Succeeded():
757            raise Exception('log enable failed (check GDB_REMOTE_LOG env variable.')
758
759def getMyCommandLine():
760    import subprocess
761    ps = subprocess.Popen(['ps', '-o', "command=CMD", str(os.getpid())], stdout=subprocess.PIPE).communicate()[0]
762    lines = ps.split('\n')
763    cmd_line = lines[1]
764    return cmd_line
765
766# ======================================== #
767#                                          #
768# Execution of the test driver starts here #
769#                                          #
770# ======================================== #
771
772#
773# Start the actions by first parsing the options while setting up the test
774# directories, followed by setting up the search paths for lldb utilities;
775# then, we walk the directory trees and collect the tests into our test suite.
776#
777parseOptionsAndInitTestdirs()
778setupSysPath()
779
780#
781# If '-d' is specified, do a delay of 10 seconds for the debugger to attach.
782#
783if delay:
784    doDelay(10)
785
786#
787# If '-l' is specified, do not skip the long running tests.
788if not skipLongRunningTest:
789    os.environ["LLDB_SKIP_LONG_RUNNING_TEST"] = "NO"
790
791#
792# Walk through the testdirs while collecting tests.
793#
794for testdir in testdirs:
795    os.path.walk(testdir, visit, 'Test')
796
797#
798# Now that we have loaded all the test cases, run the whole test suite.
799#
800
801# For the time being, let's bracket the test runner within the
802# lldb.SBDebugger.Initialize()/Terminate() pair.
803import lldb, atexit
804# Update: the act of importing lldb now executes lldb.SBDebugger.Initialize(),
805# there's no need to call it a second time.
806#lldb.SBDebugger.Initialize()
807atexit.register(lambda: lldb.SBDebugger.Terminate())
808
809# Create a singleton SBDebugger in the lldb namespace.
810lldb.DBG = lldb.SBDebugger.Create()
811
812# Put the blacklist in the lldb namespace, to be used by lldb.TestBase.
813lldb.blacklist = blacklist
814
815# Put dont/just_do_python_api_test in the lldb namespace, too.
816lldb.dont_do_python_api_test = dont_do_python_api_test
817lldb.just_do_python_api_test = just_do_python_api_test
818
819# Turn on lldb loggings if necessary.
820lldbLoggings()
821
822# Install the control-c handler.
823unittest2.signals.installHandler()
824
825# If sdir_name is not specified through the '-s sdir_name' option, get a
826# timestamp string and export it as LLDB_SESSION_DIR environment var.  This will
827# be used when/if we want to dump the session info of individual test cases
828# later on.
829#
830# See also TestBase.dumpSessionInfo() in lldbtest.py.
831if not sdir_name:
832    import datetime
833    # The windows platforms don't like ':' in the pathname.
834    timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
835    sdir_name = timestamp
836os.environ["LLDB_SESSION_DIRNAME"] = os.path.join(os.getcwd(), sdir_name)
837
838sys.stderr.write("\nSession logs for test failures/errors/unexpected successes"
839                 " will go into directory '%s'\n" % sdir_name)
840sys.stderr.write("Command invoked: %s\n" % getMyCommandLine())
841
842if not os.path.isdir(sdir_name):
843    os.mkdir(sdir_name)
844fname = os.path.join(sdir_name, "svn-info")
845with open(fname, "w") as f:
846    print >> f, svn_info
847    print >> f, "Command invoked: %s\n" % getMyCommandLine()
848
849#
850# Invoke the default TextTestRunner to run the test suite, possibly iterating
851# over different configurations.
852#
853
854iterArchs = False
855iterCompilers = False
856
857if not archs and "archs" in config:
858    archs = config["archs"]
859
860if isinstance(archs, list) and len(archs) >= 1:
861    iterArchs = True
862
863if not compilers and "compilers" in config:
864    compilers = config["compilers"]
865
866if isinstance(compilers, list) and len(compilers) >= 1:
867    iterCompilers = True
868
869# Make a shallow copy of sys.path, we need to manipulate the search paths later.
870# This is only necessary if we are relocated and with different configurations.
871if rdir:
872    old_sys_path = sys.path[:]
873# If we iterate on archs or compilers, there is a chance we want to split stderr/stdout.
874if iterArchs or iterCompilers:
875    old_stderr = sys.stderr
876    old_stdout = sys.stdout
877    new_stderr = None
878    new_stdout = None
879
880# Iterating over all possible architecture and compiler combinations.
881for ia in range(len(archs) if iterArchs else 1):
882    archConfig = ""
883    if iterArchs:
884        os.environ["ARCH"] = archs[ia]
885        archConfig = "arch=%s" % archs[ia]
886    for ic in range(len(compilers) if iterCompilers else 1):
887        if iterCompilers:
888            os.environ["CC"] = compilers[ic]
889            configString = "%s compiler=%s" % (archConfig, compilers[ic])
890        else:
891            configString = archConfig
892
893        if iterArchs or iterCompilers:
894            # Translate ' ' to '-' for pathname component.
895            from string import maketrans
896            tbl = maketrans(' ', '-')
897            configPostfix = configString.translate(tbl)
898
899            # Check whether we need to split stderr/stdout into configuration
900            # specific files.
901            if old_stderr.name != '<stderr>' and config.get('split_stderr'):
902                if new_stderr:
903                    new_stderr.close()
904                new_stderr = open("%s.%s" % (old_stderr.name, configPostfix), "w")
905                sys.stderr = new_stderr
906            if old_stdout.name != '<stdout>' and config.get('split_stdout'):
907                if new_stdout:
908                    new_stdout.close()
909                new_stdout = open("%s.%s" % (old_stdout.name, configPostfix), "w")
910                sys.stdout = new_stdout
911
912            # If we specified a relocated directory to run the test suite, do
913            # the extra housekeeping to copy the testdirs to a configStringified
914            # directory and to update sys.path before invoking the test runner.
915            # The purpose is to separate the configuration-specific directories
916            # from each other.
917            if rdir:
918                from shutil import copytree, ignore_patterns
919
920                newrdir = "%s.%s" % (rdir, configPostfix)
921
922                # Copy the tree to a new directory with postfix name configPostfix.
923                copytree(rdir, newrdir, ignore=ignore_patterns('*.pyc', '*.o', '*.d'))
924
925               # Update the LLDB_TEST environment variable to reflect new top
926                # level test directory.
927                #
928                # See also lldbtest.TestBase.setUpClass(cls).
929                if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test':
930                    os.environ["LLDB_TEST"] = os.path.join(newrdir, 'test')
931                else:
932                    os.environ["LLDB_TEST"] = newrdir
933
934                # And update the Python search paths for modules.
935                sys.path = [x.replace(rdir, newrdir, 1) for x in old_sys_path]
936
937            # Output the configuration.
938            sys.stderr.write("\nConfiguration: " + configString + "\n")
939
940        #print "sys.stderr name is", sys.stderr.name
941        #print "sys.stdout name is", sys.stdout.name
942
943        # First, write out the number of collected test cases.
944        sys.stderr.write(separator + "\n")
945        sys.stderr.write("Collected %d test%s\n\n"
946                         % (suite.countTestCases(),
947                            suite.countTestCases() != 1 and "s" or ""))
948
949        class LLDBTestResult(unittest2.TextTestResult):
950            """
951            Enforce a singleton pattern to allow introspection of test progress.
952
953            Overwrite addError(), addFailure(), and addExpectedFailure() methods
954            to enable each test instance to track its failure/error status.  It
955            is used in the LLDB test framework to emit detailed trace messages
956            to a log file for easier human inspection of test failres/errors.
957            """
958            __singleton__ = None
959            __ignore_singleton__ = False
960
961            def __init__(self, *args):
962                if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
963                    raise Exception("LLDBTestResult instantiated more than once")
964                super(LLDBTestResult, self).__init__(*args)
965                LLDBTestResult.__singleton__ = self
966                # Now put this singleton into the lldb module namespace.
967                lldb.test_result = self
968                # Computes the format string for displaying the counter.
969                global suite
970                counterWidth = len(str(suite.countTestCases()))
971                self.fmt = "%" + str(counterWidth) + "d: "
972                self.indentation = ' ' * (counterWidth + 2)
973                # This counts from 1 .. suite.countTestCases().
974                self.counter = 0
975
976            def getDescription(self, test):
977                doc_first_line = test.shortDescription()
978                if self.descriptions and doc_first_line:
979                    return '\n'.join((str(test), self.indentation + doc_first_line))
980                else:
981                    return str(test)
982
983            def startTest(self, test):
984                self.counter += 1
985                if self.showAll:
986                    self.stream.write(self.fmt % self.counter)
987                super(LLDBTestResult, self).startTest(test)
988
989            def addError(self, test, err):
990                global sdir_has_content
991                sdir_has_content = True
992                super(LLDBTestResult, self).addError(test, err)
993                method = getattr(test, "markError", None)
994                if method:
995                    method()
996
997            def addFailure(self, test, err):
998                global sdir_has_content
999                sdir_has_content = True
1000                super(LLDBTestResult, self).addFailure(test, err)
1001                method = getattr(test, "markFailure", None)
1002                if method:
1003                    method()
1004
1005            def addExpectedFailure(self, test, err):
1006                global sdir_has_content
1007                sdir_has_content = True
1008                super(LLDBTestResult, self).addExpectedFailure(test, err)
1009                method = getattr(test, "markExpectedFailure", None)
1010                if method:
1011                    method()
1012
1013            def addUnexpectedSuccess(self, test):
1014                global sdir_has_content
1015                sdir_has_content = True
1016                super(LLDBTestResult, self).addUnexpectedSuccess(test)
1017                method = getattr(test, "markUnexpectedSuccess", None)
1018                if method:
1019                    method()
1020
1021        # Invoke the test runner.
1022        if count == 1:
1023            result = unittest2.TextTestRunner(stream=sys.stderr,
1024                                              verbosity=verbose,
1025                                              failfast=failfast,
1026                                              resultclass=LLDBTestResult).run(suite)
1027        else:
1028            # We are invoking the same test suite more than once.  In this case,
1029            # mark __ignore_singleton__ flag as True so the signleton pattern is
1030            # not enforced.
1031            LLDBTestResult.__ignore_singleton__ = True
1032            for i in range(count):
1033                result = unittest2.TextTestRunner(stream=sys.stderr,
1034                                                  verbosity=verbose,
1035                                                  failfast=failfast,
1036                                                  resultclass=LLDBTestResult).run(suite)
1037
1038
1039if sdir_has_content:
1040    sys.stderr.write("Session logs for test failures/errors/unexpected successes"
1041                     " can be found in directory '%s'\n" % sdir_name)
1042
1043# Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined.
1044# This should not be necessary now.
1045if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ):
1046    import subprocess
1047    print "Terminating Test suite..."
1048    subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())])
1049
1050# Exiting.
1051sys.exit(not result.wasSuccessful)
1052