1#!/usr/bin/env python 2 3""" 4A simple testing framework for lldb using python's unit testing framework. 5 6Tests for lldb are written as python scripts which take advantage of the script 7bridging provided by LLDB.framework to interact with lldb core. 8 9A specific naming pattern is followed by the .py script to be recognized as 10a module which implements a test scenario, namely, Test*.py. 11 12To specify the directories where "Test*.py" python test scripts are located, 13you need to pass in a list of directory names. By default, the current 14working directory is searched if nothing is specified on the command line. 15 16Type: 17 18./dotest.py -h 19 20for available options. 21""" 22 23import os 24import platform 25import signal 26import subprocess 27import sys 28import textwrap 29import time 30import unittest2 31import progress 32 33if sys.version_info >= (2, 7): 34 argparse = __import__('argparse') 35else: 36 argparse = __import__('argparse_compat') 37 38def parse_args(parser): 39 """ Returns an argument object. LLDB_TEST_ARGUMENTS environment variable can 40 be used to pass additional arguments if a compatible (>=2.7) argparse 41 library is available. 42 """ 43 if sys.version_info >= (2, 7): 44 args = ArgParseNamespace() 45 46 if ('LLDB_TEST_ARGUMENTS' in os.environ): 47 print "Arguments passed through environment: '%s'" % os.environ['LLDB_TEST_ARGUMENTS'] 48 args = parser.parse_args([sys.argv[0]].__add__(os.environ['LLDB_TEST_ARGUMENTS'].split()),namespace=args) 49 50 return parser.parse_args(namespace=args) 51 else: 52 return parser.parse_args() 53 54def is_exe(fpath): 55 """Returns true if fpath is an executable.""" 56 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 57 58def which(program): 59 """Returns the full path to a program; None otherwise.""" 60 fpath, fname = os.path.split(program) 61 if fpath: 62 if is_exe(program): 63 return program 64 else: 65 for path in os.environ["PATH"].split(os.pathsep): 66 exe_file = os.path.join(path, program) 67 if is_exe(exe_file): 68 return exe_file 69 return None 70 71class _WritelnDecorator(object): 72 """Used to decorate file-like objects with a handy 'writeln' method""" 73 def __init__(self,stream): 74 self.stream = stream 75 76 def __getattr__(self, attr): 77 if attr in ('stream', '__getstate__'): 78 raise AttributeError(attr) 79 return getattr(self.stream,attr) 80 81 def writeln(self, arg=None): 82 if arg: 83 self.write(arg) 84 self.write('\n') # text-mode streams translate to \r\n if needed 85 86# 87# Global variables: 88# 89 90# Dictionary of categories 91# When you define a new category for your testcases, be sure to add it here, or the test suite 92# will gladly complain as soon as you try to use it. This allows us to centralize which categories 93# exist, and to provide a description for each one 94validCategories = { 95'dataformatters':'Tests related to the type command and the data formatters subsystem', 96'expression':'Tests related to the expression parser', 97'objc':'Tests related to the Objective-C programming language support', 98'pyapi':'Tests related to the Python API', 99'basic_process': 'Basic process execution sniff tests.', 100'cmdline' : 'Tests related to the LLDB command-line interface' 101} 102 103# The test suite. 104suite = unittest2.TestSuite() 105 106# By default, both command line and Python API tests are performed. 107# Use @python_api_test decorator, defined in lldbtest.py, to mark a test as 108# a Python API test. 109dont_do_python_api_test = False 110 111# By default, both command line and Python API tests are performed. 112just_do_python_api_test = False 113 114# By default, benchmarks tests are not run. 115just_do_benchmarks_test = False 116 117# By default, both dsym and dwarf tests are performed. 118# Use @dsym_test or @dwarf_test decorators, defined in lldbtest.py, to mark a test 119# as a dsym or dwarf test. Use '-N dsym' or '-N dwarf' to exclude dsym or dwarf 120# tests from running. 121dont_do_dsym_test = "linux" in sys.platform or "freebsd" in sys.platform 122dont_do_dwarf_test = False 123 124# The blacklist is optional (-b blacklistFile) and allows a central place to skip 125# testclass's and/or testclass.testmethod's. 126blacklist = None 127 128# The dictionary as a result of sourcing blacklistFile. 129blacklistConfig = {} 130 131# The list of categories we said we care about 132categoriesList = None 133# set to true if we are going to use categories for cherry-picking test cases 134useCategories = False 135# Categories we want to skip 136skipCategories = [] 137# use this to track per-category failures 138failuresPerCategory = {} 139 140# The path to LLDB.framework is optional. 141lldbFrameworkPath = None 142 143# The path to lldb is optional 144lldbExecutablePath = None 145 146# The config file is optional. 147configFile = None 148 149# Test suite repeat count. Can be overwritten with '-# count'. 150count = 1 151 152# The dictionary as a result of sourcing configFile. 153config = {} 154# The pre_flight and post_flight functions come from reading a config file. 155pre_flight = None 156post_flight = None 157 158# The 'archs' and 'compilers' can be specified via either command line or configFile, 159# with the command line overriding the configFile. The corresponding options can be 160# specified more than once. For example, "-A x86_64 -A i386" => archs=['x86_64', 'i386'] 161# and "-C gcc -C clang" => compilers=['gcc', 'clang']. 162archs = None # Must be initialized after option parsing 163compilers = None # Must be initialized after option parsing 164 165# The arch might dictate some specific CFLAGS to be passed to the toolchain to build 166# the inferior programs. The global variable cflags_extras provides a hook to do 167# just that. 168cflags_extras = '' 169 170# Delay startup in order for the debugger to attach. 171delay = False 172 173# Dump the Python sys.path variable. Use '-D' to dump sys.path. 174dumpSysPath = False 175 176# Full path of the benchmark executable, as specified by the '-e' option. 177bmExecutable = None 178# The breakpoint specification of bmExecutable, as specified by the '-x' option. 179bmBreakpointSpec = None 180# The benchamrk iteration count, as specified by the '-y' option. 181bmIterationCount = -1 182 183# By default, don't exclude any directories. Use '-X' to add one excluded directory. 184excluded = set(['.svn', '.git']) 185 186# By default, failfast is False. Use '-F' to overwrite it. 187failfast = False 188 189# The filters (testclass.testmethod) used to admit tests into our test suite. 190filters = [] 191 192# The runhooks is a list of lldb commands specifically for the debugger. 193# Use '-k' to specify a runhook. 194runHooks = [] 195 196# If '-g' is specified, the filterspec is not exclusive. If a test module does 197# not contain testclass.testmethod which matches the filterspec, the whole test 198# module is still admitted into our test suite. fs4all flag defaults to True. 199fs4all = True 200 201# Ignore the build search path relative to this script to locate the lldb.py module. 202ignore = False 203 204# By default, we do not skip build and cleanup. Use '-S' option to override. 205skip_build_and_cleanup = False 206 207# By default, we skip long running test case. Use '-l' option to override. 208skip_long_running_test = True 209 210# By default, we print the build dir, lldb version, and svn info. Use '-n' option to 211# turn it off. 212noHeaders = False 213 214# Parsable mode silences headers, and any other output this script might generate, and instead 215# prints machine-readable output similar to what clang tests produce. 216parsable = False 217 218# The regular expression pattern to match against eligible filenames as our test cases. 219regexp = None 220 221# By default, tests are executed in place and cleanups are performed afterwards. 222# Use '-r dir' option to relocate the tests and their intermediate files to a 223# different directory and to forgo any cleanups. The directory specified must 224# not exist yet. 225rdir = None 226 227# By default, recorded session info for errored/failed test are dumped into its 228# own file under a session directory named after the timestamp of the test suite 229# run. Use '-s session-dir-name' to specify a specific dir name. 230sdir_name = None 231 232# Set this flag if there is any session info dumped during the test run. 233sdir_has_content = False 234 235# svn_info stores the output from 'svn info lldb.base.dir'. 236svn_info = '' 237 238# svn_silent means do not try to obtain svn status 239svn_silent = True 240 241# Default verbosity is 0. 242verbose = 1 243 244# Set to True only if verbose is 0 and LLDB trace mode is off. 245progress_bar = False 246 247# By default, search from the script directory. 248testdirs = [ sys.path[0] ] 249 250# Separator string. 251separator = '-' * 70 252 253failed = False 254 255def usage(parser): 256 parser.print_help() 257 if verbose > 0: 258 print """ 259Examples: 260 261This is an example of using the -f option to pinpoint to a specfic test class 262and test method to be run: 263 264$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command 265---------------------------------------------------------------------- 266Collected 1 test 267 268test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase) 269Test 'frame variable this' when stopped on a class constructor. ... ok 270 271---------------------------------------------------------------------- 272Ran 1 test in 1.396s 273 274OK 275 276And this is an example of using the -p option to run a single file (the filename 277matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'): 278 279$ ./dotest.py -v -p ObjC 280---------------------------------------------------------------------- 281Collected 4 tests 282 283test_break_with_dsym (TestObjCMethods.FoundationTestCase) 284Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 285test_break_with_dwarf (TestObjCMethods.FoundationTestCase) 286Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 287test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase) 288Lookup objective-c data types and evaluate expressions. ... ok 289test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase) 290Lookup objective-c data types and evaluate expressions. ... ok 291 292---------------------------------------------------------------------- 293Ran 4 tests in 16.661s 294 295OK 296 297Running of this script also sets up the LLDB_TEST environment variable so that 298individual test cases can locate their supporting files correctly. The script 299tries to set up Python's search paths for modules by looking at the build tree 300relative to this script. See also the '-i' option in the following example. 301 302Finally, this is an example of using the lldb.py module distributed/installed by 303Xcode4 to run against the tests under the 'forward' directory, and with the '-w' 304option to add some delay between two tests. It uses ARCH=x86_64 to specify that 305as the architecture and CC=clang to specify the compiler used for the test run: 306 307$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward 308 309Session logs for test failures/errors will go into directory '2010-11-11-13_56_16' 310---------------------------------------------------------------------- 311Collected 2 tests 312 313test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 314Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 315test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 316Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 317 318---------------------------------------------------------------------- 319Ran 2 tests in 5.659s 320 321OK 322 323The 'Session ...' verbiage is recently introduced (see also the '-s' option) to 324notify the directory containing the session logs for test failures or errors. 325In case there is any test failure/error, a similar message is appended at the 326end of the stderr output for your convenience. 327 328Environment variables related to loggings: 329 330o LLDB_LOG: if defined, specifies the log file pathname for the 'lldb' subsystem 331 with a default option of 'event process' if LLDB_LOG_OPTION is not defined. 332 333o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the 334 'process.gdb-remote' subsystem with a default option of 'packets' if 335 GDB_REMOTE_LOG_OPTION is not defined. 336""" 337 sys.exit(0) 338 339 340def unique_string_match(yourentry,list): 341 candidate = None 342 for item in list: 343 if item.startswith(yourentry): 344 if candidate: 345 return None 346 candidate = item 347 return candidate 348 349class ArgParseNamespace(object): 350 pass 351 352def validate_categories(categories): 353 """For each category in categories, ensure that it's a valid category (or a prefix thereof). 354 If a category is invalid, print a message and quit. 355 If all categories are valid, return the list of categories. Prefixes are expanded in the 356 returned list. 357 """ 358 global validCategories 359 result = [] 360 for category in categories: 361 origCategory = category 362 if category not in validCategories: 363 category = unique_string_match(category, validCategories) 364 if (category not in validCategories) or category == None: 365 print "fatal error: category '" + origCategory + "' is not a valid category" 366 print "if you have added a new category, please edit dotest.py, adding your new category to validCategories" 367 print "else, please specify one or more of the following: " + str(validCategories.keys()) 368 sys.exit(1) 369 result.append(category) 370 return result 371 372def parseOptionsAndInitTestdirs(): 373 """Initialize the list of directories containing our unittest scripts. 374 375 '-h/--help as the first option prints out usage info and exit the program. 376 """ 377 378 global dont_do_python_api_test 379 global just_do_python_api_test 380 global just_do_benchmarks_test 381 global dont_do_dsym_test 382 global dont_do_dwarf_test 383 global blacklist 384 global blacklistConfig 385 global categoriesList 386 global validCategories 387 global useCategories 388 global skipCategories 389 global lldbFrameworkPath 390 global lldbExecutablePath 391 global configFile 392 global archs 393 global compilers 394 global count 395 global delay 396 global dumpSysPath 397 global bmExecutable 398 global bmBreakpointSpec 399 global bmIterationCount 400 global failfast 401 global filters 402 global fs4all 403 global ignore 404 global progress_bar 405 global runHooks 406 global skip_build_and_cleanup 407 global skip_long_running_test 408 global noHeaders 409 global parsable 410 global regexp 411 global rdir 412 global sdir_name 413 global svn_silent 414 global verbose 415 global testdirs 416 417 do_help = False 418 419 parser = argparse.ArgumentParser(description='description', prefix_chars='+-', add_help=False) 420 group = None 421 422 # Helper function for boolean options (group will point to the current group when executing X) 423 X = lambda optstr, helpstr, **kwargs: group.add_argument(optstr, help=helpstr, action='store_true', **kwargs) 424 425 group = parser.add_argument_group('Help') 426 group.add_argument('-h', '--help', dest='h', action='store_true', help="Print this help message and exit. Add '-v' for more detailed help.") 427 428 # C and Python toolchain options 429 group = parser.add_argument_group('Toolchain options') 430 group.add_argument('-A', '--arch', metavar='arch', action='append', dest='archs', help=textwrap.dedent('''Specify the architecture(s) to test. This option can be specified more than once''')) 431 group.add_argument('-C', '--compiler', metavar='compiler', dest='compilers', action='append', help=textwrap.dedent('''Specify the compiler(s) used to build the inferior executables. The compiler path can be an executable basename or a full path to a compiler executable. This option can be specified multiple times.''')) 432 # FIXME? This won't work for different extra flags according to each arch. 433 group.add_argument('-E', metavar='extra-flags', help=textwrap.dedent('''Specify the extra flags to be passed to the toolchain when building the inferior programs to be debugged 434 suggestions: do not lump the "-A arch1 -A arch2" together such that the -E option applies to only one of the architectures''')) 435 X('-D', 'Dump the Python sys.path variable') 436 437 # Test filtering options 438 group = parser.add_argument_group('Test filtering options') 439 group.add_argument('-N', choices=['dwarf', 'dsym'], help="Don't do test cases marked with the @dsym decorator by passing 'dsym' as the option arg, or don't do test cases marked with the @dwarf decorator by passing 'dwarf' as the option arg") 440 X('-a', "Don't do lldb Python API tests") 441 X('+a', "Just do lldb Python API tests. Do not specify along with '+a'", dest='plus_a') 442 X('+b', 'Just do benchmark tests', dest='plus_b') 443 group.add_argument('-b', metavar='blacklist', help='Read a blacklist file specified after this option') 444 group.add_argument('-f', metavar='filterspec', action='append', help='Specify a filter, which consists of the test class name, a dot, followed by the test method, to only admit such test into the test suite') # FIXME: Example? 445 X('-g', 'If specified, the filterspec by -f is not exclusive, i.e., if a test module does not match the filterspec (testclass.testmethod), the whole module is still admitted to the test suite') 446 X('-l', "Don't skip long running tests") 447 group.add_argument('-p', metavar='pattern', help='Specify a regexp filename pattern for inclusion in the test suite') 448 group.add_argument('-X', metavar='directory', help="Exclude a directory from consideration for test discovery. -X types => if 'types' appear in the pathname components of a potential testfile, it will be ignored") 449 group.add_argument('-G', '--category', metavar='category', action='append', dest='categoriesList', help=textwrap.dedent('''Specify categories of test cases of interest. Can be specified more than once.''')) 450 group.add_argument('--skip-category', metavar='category', action='append', dest='skipCategories', help=textwrap.dedent('''Specify categories of test cases to skip. Takes precedence over -G. Can be specified more than once.''')) 451 452 # Configuration options 453 group = parser.add_argument_group('Configuration options') 454 group.add_argument('-c', metavar='config-file', help='Read a config file specified after this option') # FIXME: additional doc. 455 group.add_argument('--framework', metavar='framework-path', help='The path to LLDB.framework') 456 group.add_argument('--executable', metavar='executable-path', help='The path to the lldb executable') 457 group.add_argument('--libcxx', metavar='directory', help='The path to custom libc++ library') 458 group.add_argument('-e', metavar='benchmark-exe', help='Specify the full path of an executable used for benchmark purposes (see also: -x)') 459 group.add_argument('-k', metavar='command', action='append', help="Specify a runhook, which is an lldb command to be executed by the debugger; The option can occur multiple times. The commands are executed one after the other to bring the debugger to a desired state, so that, for example, further benchmarking can be done") 460 group.add_argument('-R', metavar='dir', help='Specify a directory to relocate the tests and their intermediate files to. BE WARNED THAT the directory, if exists, will be deleted before running this test driver. No cleanup of intermediate test files is performed in this case') 461 group.add_argument('-r', metavar='dir', help="Similar to '-R', except that the directory must not exist before running this test driver") 462 group.add_argument('-s', metavar='name', help='Specify the name of the dir created to store the session files of tests with errored or failed status. If not specified, the test driver uses the timestamp as the session dir name') 463 group.add_argument('-x', metavar='breakpoint-spec', help='Specify the breakpoint specification for the benchmark executable') 464 group.add_argument('-y', type=int, metavar='count', help="Specify the iteration count used to collect our benchmarks. An example is the number of times to do 'thread step-over' to measure stepping speed.") 465 group.add_argument('-#', type=int, metavar='sharp', dest='sharp', help='Repeat the test suite for a specified number of times') 466 467 # Test-suite behaviour 468 group = parser.add_argument_group('Runtime behaviour options') 469 X('-d', 'Delay startup for 10 seconds (in order for the debugger to attach)') 470 X('-F', 'Fail fast. Stop the test suite on the first error/failure') 471 X('-i', "Ignore (don't bailout) if 'lldb.py' module cannot be located in the build tree relative to this script; use PYTHONPATH to locate the module") 472 X('-n', "Don't print the headers like build dir, lldb version, and svn info at all") 473 X('-P', "Use the graphic progress bar.") 474 X('-q', "Don't print extra output from this script.") 475 X('-S', "Skip the build and cleanup while running the test. Use this option with care as you would need to build the inferior(s) by hand and build the executable(s) with the correct name(s). This can be used with '-# n' to stress test certain test cases for n number of times") 476 X('-t', 'Turn on tracing of lldb command and other detailed test executions') 477 group.add_argument('-u', dest='unset_env_varnames', metavar='variable', action='append', help='Specify an environment variable to unset before running the test cases. e.g., -u DYLD_INSERT_LIBRARIES -u MallocScribble') 478 X('-v', 'Do verbose mode of unittest framework (print out each test case invocation)') 479 X('-w', 'Insert some wait time (currently 0.5 sec) between consecutive test cases') 480 X('-T', 'Obtain and dump svn information for this checkout of LLDB (off by default)') 481 482 # Remove the reference to our helper function 483 del X 484 485 group = parser.add_argument_group('Test directories') 486 group.add_argument('args', metavar='test-dir', nargs='*', help='Specify a list of directory names to search for test modules named after Test*.py (test discovery). If empty, search from the current working directory instead.') 487 488 args = parse_args(parser) 489 platform_system = platform.system() 490 platform_machine = platform.machine() 491 492 if args.unset_env_varnames: 493 for env_var in args.unset_env_varnames: 494 if env_var in os.environ: 495 # From Python Doc: When unsetenv() is supported, deletion of items in os.environ 496 # is automatically translated into a corresponding call to unsetenv(). 497 del os.environ[env_var] 498 #os.unsetenv(env_var) 499 500 # only print the args if being verbose (and parsable is off) 501 if args.v and not args.q: 502 print sys.argv 503 504 if args.h: 505 do_help = True 506 507 if args.archs: 508 archs = args.archs 509 else: 510 if platform_system == 'Darwin' and platform_machine == 'x86_64': 511 archs = ['x86_64', 'i386'] 512 else: 513 archs = [platform_machine] 514 515 if args.categoriesList: 516 categoriesList = set(validate_categories(args.categoriesList)) 517 useCategories = True 518 else: 519 categoriesList = [] 520 521 if args.skipCategories: 522 skipCategories = validate_categories(args.skipCategories) 523 524 if args.compilers: 525 compilers = args.compilers 526 else: 527 compilers = ['clang'] 528 529 if args.D: 530 dumpSysPath = True 531 532 if args.E: 533 cflags_extras = args.E 534 os.environ['CFLAGS_EXTRAS'] = cflags_extras 535 536 # argparse makes sure we have correct options 537 if args.N == 'dwarf': 538 dont_do_dwarf_test = True 539 elif args.N == 'dsym': 540 dont_do_dsym_test = True 541 542 if args.a: 543 dont_do_python_api_test = True 544 545 if args.plus_a: 546 if dont_do_python_api_test: 547 print "Warning: -a and +a can't both be specified! Using only -a" 548 else: 549 just_do_python_api_test = True 550 551 if args.plus_b: 552 just_do_benchmarks_test = True 553 554 if args.b: 555 if args.b.startswith('-'): 556 usage(parser) 557 blacklistFile = args.b 558 if not os.path.isfile(blacklistFile): 559 print 'Blacklist file:', blacklistFile, 'does not exist!' 560 usage(parser) 561 # Now read the blacklist contents and assign it to blacklist. 562 execfile(blacklistFile, globals(), blacklistConfig) 563 blacklist = blacklistConfig.get('blacklist') 564 565 if args.c: 566 if args.c.startswith('-'): 567 usage(parser) 568 configFile = args.c 569 if not os.path.isfile(configFile): 570 print 'Config file:', configFile, 'does not exist!' 571 usage(parser) 572 573 if args.d: 574 delay = True 575 576 if args.e: 577 if args.e.startswith('-'): 578 usage(parser) 579 bmExecutable = args.e 580 if not is_exe(bmExecutable): 581 usage(parser) 582 583 if args.F: 584 failfast = True 585 586 if args.f: 587 if any([x.startswith('-') for x in args.f]): 588 usage(parser) 589 filters.extend(args.f) 590 591 if args.g: 592 fs4all = False 593 594 if args.i: 595 ignore = True 596 597 if args.k: 598 runHooks.extend(args.k) 599 600 if args.l: 601 skip_long_running_test = False 602 603 if args.framework: 604 lldbFrameworkPath = args.framework 605 606 if args.executable: 607 lldbExecutablePath = args.executable 608 609 if args.libcxx: 610 os.environ["LIBCXX_PATH"] = args.libcxx 611 612 if args.n: 613 noHeaders = True 614 615 if args.p: 616 if args.p.startswith('-'): 617 usage(parser) 618 regexp = args.p 619 620 if args.q: 621 noHeaders = True 622 parsable = True 623 624 if args.P: 625 progress_bar = True 626 verbose = 0 627 628 if args.R: 629 if args.R.startswith('-'): 630 usage(parser) 631 rdir = os.path.abspath(args.R) 632 if os.path.exists(rdir): 633 import shutil 634 print 'Removing tree:', rdir 635 shutil.rmtree(rdir) 636 637 if args.r: 638 if args.r.startswith('-'): 639 usage(parser) 640 rdir = os.path.abspath(args.r) 641 if os.path.exists(rdir): 642 print 'Relocated directory:', rdir, 'must not exist!' 643 usage(parser) 644 645 if args.S: 646 skip_build_and_cleanup = True 647 648 if args.s: 649 if args.s.startswith('-'): 650 usage(parser) 651 sdir_name = args.s 652 653 if args.t: 654 os.environ['LLDB_COMMAND_TRACE'] = 'YES' 655 656 if args.T: 657 svn_silent = False 658 659 if args.v: 660 verbose = 2 661 662 if args.w: 663 os.environ['LLDB_WAIT_BETWEEN_TEST_CASES'] = 'YES' 664 665 if args.X: 666 if args.X.startswith('-'): 667 usage(parser) 668 excluded.add(args.X) 669 670 if args.x: 671 if args.x.startswith('-'): 672 usage(parser) 673 bmBreakpointSpec = args.x 674 675 # argparse makes sure we have a number 676 if args.y: 677 bmIterationCount = args.y 678 679 # argparse makes sure we have a number 680 if args.sharp: 681 count = args.sharp 682 683 if do_help == True: 684 usage(parser) 685 686 # Do not specify both '-a' and '+a' at the same time. 687 if dont_do_python_api_test and just_do_python_api_test: 688 usage(parser) 689 690 # Gather all the dirs passed on the command line. 691 if len(args.args) > 0: 692 testdirs = map(os.path.abspath, args.args) 693 694 # If '-r dir' is specified, the tests should be run under the relocated 695 # directory. Let's copy the testdirs over. 696 if rdir: 697 from shutil import copytree, ignore_patterns 698 699 tmpdirs = [] 700 orig_testdirs = testdirs[:] 701 for srcdir in testdirs: 702 # For example, /Volumes/data/lldb/svn/ToT/test/functionalities/watchpoint/hello_watchpoint 703 # shall be split into ['/Volumes/data/lldb/svn/ToT/', 'functionalities/watchpoint/hello_watchpoint']. 704 # Utilize the relative path to the 'test' directory to make our destination dir path. 705 if ("test" + os.sep) in srcdir: 706 to_split_on = "test" + os.sep 707 else: 708 to_split_on = "test" 709 dstdir = os.path.join(rdir, srcdir.split(to_split_on)[1]) 710 dstdir = dstdir.rstrip(os.sep) 711 # Don't copy the *.pyc and .svn stuffs. 712 copytree(srcdir, dstdir, ignore=ignore_patterns('*.pyc', '.svn')) 713 tmpdirs.append(dstdir) 714 715 # This will be our modified testdirs. 716 testdirs = tmpdirs 717 718 # With '-r dir' specified, there's no cleanup of intermediate test files. 719 os.environ["LLDB_DO_CLEANUP"] = 'NO' 720 721 # If the original testdirs is ['test'], the make directory has already been copied 722 # recursively and is contained within the rdir/test dir. For anything 723 # else, we would need to copy over the make directory and its contents, 724 # so that, os.listdir(rdir) looks like, for example: 725 # 726 # array_types conditional_break make 727 # 728 # where the make directory contains the Makefile.rules file. 729 if len(testdirs) != 1 or os.path.basename(orig_testdirs[0]) != 'test': 730 scriptdir = os.path.dirname(__file__) 731 # Don't copy the .svn stuffs. 732 copytree(os.path.join(scriptdir, 'make'), os.path.join(rdir, 'make'), 733 ignore=ignore_patterns('.svn')) 734 735 #print "testdirs:", testdirs 736 737 # Source the configFile if specified. 738 # The side effect, if any, will be felt from this point on. An example 739 # config file may be these simple two lines: 740 # 741 # sys.stderr = open("/tmp/lldbtest-stderr", "w") 742 # sys.stdout = open("/tmp/lldbtest-stdout", "w") 743 # 744 # which will reassign the two file objects to sys.stderr and sys.stdout, 745 # respectively. 746 # 747 # See also lldb-trunk/examples/test/usage-config. 748 global config, pre_flight, post_flight 749 if configFile: 750 # Pass config (a dictionary) as the locals namespace for side-effect. 751 execfile(configFile, globals(), config) 752 print "config:", config 753 if "pre_flight" in config: 754 pre_flight = config["pre_flight"] 755 if not callable(pre_flight): 756 print "fatal error: pre_flight is not callable, exiting." 757 sys.exit(1) 758 if "post_flight" in config: 759 post_flight = config["post_flight"] 760 if not callable(post_flight): 761 print "fatal error: post_flight is not callable, exiting." 762 sys.exit(1) 763 #print "sys.stderr:", sys.stderr 764 #print "sys.stdout:", sys.stdout 765 766 767def setupSysPath(): 768 """ 769 Add LLDB.framework/Resources/Python to the search paths for modules. 770 As a side effect, we also discover the 'lldb' executable and export it here. 771 """ 772 773 global rdir 774 global testdirs 775 global dumpSysPath 776 global noHeaders 777 global svn_info 778 global svn_silent 779 global lldbFrameworkPath 780 global lldbExecutablePath 781 782 # Get the directory containing the current script. 783 if ("DOTEST_PROFILE" in os.environ or "DOTEST_PDB" in os.environ) and "DOTEST_SCRIPT_DIR" in os.environ: 784 scriptPath = os.environ["DOTEST_SCRIPT_DIR"] 785 else: 786 scriptPath = sys.path[0] 787 if not scriptPath.endswith('test'): 788 print "This script expects to reside in lldb's test directory." 789 sys.exit(-1) 790 791 if rdir: 792 # Set up the LLDB_TEST environment variable appropriately, so that the 793 # individual tests can be located relatively. 794 # 795 # See also lldbtest.TestBase.setUpClass(cls). 796 if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test': 797 os.environ["LLDB_TEST"] = os.path.join(rdir, 'test') 798 else: 799 os.environ["LLDB_TEST"] = rdir 800 else: 801 os.environ["LLDB_TEST"] = scriptPath 802 803 # Set up the LLDB_SRC environment variable, so that the tests can locate 804 # the LLDB source code. 805 os.environ["LLDB_SRC"] = os.path.join(sys.path[0], os.pardir) 806 807 pluginPath = os.path.join(scriptPath, 'plugins') 808 pexpectPath = os.path.join(scriptPath, 'pexpect-2.4') 809 810 # Append script dir, plugin dir, and pexpect dir to the sys.path. 811 sys.path.append(scriptPath) 812 sys.path.append(pluginPath) 813 sys.path.append(pexpectPath) 814 815 # This is our base name component. 816 base = os.path.abspath(os.path.join(scriptPath, os.pardir)) 817 818 # These are for xcode build directories. 819 xcode3_build_dir = ['build'] 820 xcode4_build_dir = ['build', 'lldb', 'Build', 'Products'] 821 dbg = ['Debug'] 822 dbc = ['DebugClang'] 823 rel = ['Release'] 824 bai = ['BuildAndIntegration'] 825 python_resource_dir = ['LLDB.framework', 'Resources', 'Python'] 826 827 # Some of the tests can invoke the 'lldb' command directly. 828 # We'll try to locate the appropriate executable right here. 829 830 lldbExec = None 831 if lldbExecutablePath: 832 if is_exe(lldbExecutablePath): 833 lldbExec = lldbExecutablePath 834 lldbHere = lldbExec 835 else: 836 print lldbExecutablePath + " is not an executable" 837 sys.exit(-1) 838 else: 839 # First, you can define an environment variable LLDB_EXEC specifying the 840 # full pathname of the lldb executable. 841 if "LLDB_EXEC" in os.environ and is_exe(os.environ["LLDB_EXEC"]): 842 lldbExec = os.environ["LLDB_EXEC"] 843 else: 844 lldbExec = None 845 846 executable = ['lldb'] 847 dbgExec = os.path.join(base, *(xcode3_build_dir + dbg + executable)) 848 dbgExec2 = os.path.join(base, *(xcode4_build_dir + dbg + executable)) 849 dbcExec = os.path.join(base, *(xcode3_build_dir + dbc + executable)) 850 dbcExec2 = os.path.join(base, *(xcode4_build_dir + dbc + executable)) 851 relExec = os.path.join(base, *(xcode3_build_dir + rel + executable)) 852 relExec2 = os.path.join(base, *(xcode4_build_dir + rel + executable)) 853 baiExec = os.path.join(base, *(xcode3_build_dir + bai + executable)) 854 baiExec2 = os.path.join(base, *(xcode4_build_dir + bai + executable)) 855 856 # The 'lldb' executable built here in the source tree. 857 lldbHere = None 858 if is_exe(dbgExec): 859 lldbHere = dbgExec 860 elif is_exe(dbgExec2): 861 lldbHere = dbgExec2 862 elif is_exe(dbcExec): 863 lldbHere = dbcExec 864 elif is_exe(dbcExec2): 865 lldbHere = dbcExec2 866 elif is_exe(relExec): 867 lldbHere = relExec 868 elif is_exe(relExec2): 869 lldbHere = relExec2 870 elif is_exe(baiExec): 871 lldbHere = baiExec 872 elif is_exe(baiExec2): 873 lldbHere = baiExec2 874 elif lldbExec: 875 lldbHere = lldbExec 876 877 # One last chance to locate the 'lldb' executable. 878 if not lldbExec: 879 lldbExec = which('lldb') 880 if lldbHere and not lldbExec: 881 lldbExec = lldbHere 882 if lldbExec and not lldbHere: 883 lldbHere = lldbExec 884 885 if lldbHere: 886 os.environ["LLDB_HERE"] = lldbHere 887 os.environ["LLDB_LIB_DIR"] = os.path.split(lldbHere)[0] 888 if not noHeaders: 889 print "LLDB library dir:", os.environ["LLDB_LIB_DIR"] 890 os.system('%s -v' % lldbHere) 891 892 if not lldbExec: 893 print "The 'lldb' executable cannot be located. Some of the tests may not be run as a result." 894 else: 895 os.environ["LLDB_EXEC"] = lldbExec 896 #print "The 'lldb' from PATH env variable", lldbExec 897 898 # Skip printing svn/git information when running in parsable (lit-test compatibility) mode 899 if not svn_silent and not parsable: 900 if os.path.isdir(os.path.join(base, '.svn')) and which("svn") is not None: 901 pipe = subprocess.Popen([which("svn"), "info", base], stdout = subprocess.PIPE) 902 svn_info = pipe.stdout.read() 903 elif os.path.isdir(os.path.join(base, '.git')) and which("git") is not None: 904 pipe = subprocess.Popen([which("git"), "svn", "info", base], stdout = subprocess.PIPE) 905 svn_info = pipe.stdout.read() 906 if not noHeaders: 907 print svn_info 908 909 global ignore 910 911 lldbPath = None 912 if lldbFrameworkPath: 913 candidatePath = os.path.join(lldbFrameworkPath, 'Resources', 'Python') 914 if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')): 915 lldbPath = candidatePath 916 if not lldbPath: 917 print 'Resources/Python/lldb/__init__.py was not found in ' + lldbFrameworkPath 918 sys.exit(-1) 919 else: 920 # The '-i' option is used to skip looking for lldb.py in the build tree. 921 if ignore: 922 return 923 924 # If our lldb supports the -P option, use it to find the python path: 925 init_in_python_dir = 'lldb/__init__.py' 926 import pexpect 927 lldb_dash_p_result = None 928 929 if lldbHere: 930 lldb_dash_p_result = pexpect.run("%s -P"%(lldbHere)) 931 elif lldbExec: 932 lldb_dash_p_result = pexpect.run("%s -P"%(lldbExec)) 933 934 if lldb_dash_p_result and not lldb_dash_p_result.startswith(("<", "lldb: invalid option:")): 935 lines = lldb_dash_p_result.splitlines() 936 if len(lines) == 1 and os.path.isfile(os.path.join(lines[0], init_in_python_dir)): 937 lldbPath = lines[0] 938 if "linux" in sys.platform: 939 os.environ['LLDB_LIB_DIR'] = os.path.join(lldbPath, '..', '..') 940 941 if not lldbPath: 942 dbgPath = os.path.join(base, *(xcode3_build_dir + dbg + python_resource_dir)) 943 dbgPath2 = os.path.join(base, *(xcode4_build_dir + dbg + python_resource_dir)) 944 dbcPath = os.path.join(base, *(xcode3_build_dir + dbc + python_resource_dir)) 945 dbcPath2 = os.path.join(base, *(xcode4_build_dir + dbc + python_resource_dir)) 946 relPath = os.path.join(base, *(xcode3_build_dir + rel + python_resource_dir)) 947 relPath2 = os.path.join(base, *(xcode4_build_dir + rel + python_resource_dir)) 948 baiPath = os.path.join(base, *(xcode3_build_dir + bai + python_resource_dir)) 949 baiPath2 = os.path.join(base, *(xcode4_build_dir + bai + python_resource_dir)) 950 951 if os.path.isfile(os.path.join(dbgPath, init_in_python_dir)): 952 lldbPath = dbgPath 953 elif os.path.isfile(os.path.join(dbgPath2, init_in_python_dir)): 954 lldbPath = dbgPath2 955 elif os.path.isfile(os.path.join(dbcPath, init_in_python_dir)): 956 lldbPath = dbcPath 957 elif os.path.isfile(os.path.join(dbcPath2, init_in_python_dir)): 958 lldbPath = dbcPath2 959 elif os.path.isfile(os.path.join(relPath, init_in_python_dir)): 960 lldbPath = relPath 961 elif os.path.isfile(os.path.join(relPath2, init_in_python_dir)): 962 lldbPath = relPath2 963 elif os.path.isfile(os.path.join(baiPath, init_in_python_dir)): 964 lldbPath = baiPath 965 elif os.path.isfile(os.path.join(baiPath2, init_in_python_dir)): 966 lldbPath = baiPath2 967 968 if not lldbPath: 969 print 'This script requires lldb.py to be in either ' + dbgPath + ',', 970 print relPath + ', or ' + baiPath 971 sys.exit(-1) 972 973 # Some of the code that uses this path assumes it hasn't resolved the Versions... link. 974 # If the path we've constructed looks like that, then we'll strip out the Versions/A part. 975 (before, frameWithVersion, after) = lldbPath.rpartition("LLDB.framework/Versions/A") 976 if frameWithVersion != "" : 977 lldbPath = before + "LLDB.framework" + after 978 979 lldbPath = os.path.abspath(lldbPath) 980 981 # If tests need to find LLDB_FRAMEWORK, now they can do it 982 os.environ["LLDB_FRAMEWORK"] = os.path.dirname(os.path.dirname(lldbPath)) 983 984 # This is to locate the lldb.py module. Insert it right after sys.path[0]. 985 sys.path[1:1] = [lldbPath] 986 if dumpSysPath: 987 print "sys.path:", sys.path 988 989 990def doDelay(delta): 991 """Delaying startup for delta-seconds to facilitate debugger attachment.""" 992 def alarm_handler(*args): 993 raise Exception("timeout") 994 995 signal.signal(signal.SIGALRM, alarm_handler) 996 signal.alarm(delta) 997 sys.stdout.write("pid=%d\n" % os.getpid()) 998 sys.stdout.write("Enter RET to proceed (or timeout after %d seconds):" % 999 delta) 1000 sys.stdout.flush() 1001 try: 1002 text = sys.stdin.readline() 1003 except: 1004 text = "" 1005 signal.alarm(0) 1006 sys.stdout.write("proceeding...\n") 1007 pass 1008 1009 1010def visit(prefix, dir, names): 1011 """Visitor function for os.path.walk(path, visit, arg).""" 1012 1013 global suite 1014 global regexp 1015 global filters 1016 global fs4all 1017 global excluded 1018 1019 if set(dir.split(os.sep)).intersection(excluded): 1020 #print "Detected an excluded dir component: %s" % dir 1021 return 1022 1023 for name in names: 1024 if os.path.isdir(os.path.join(dir, name)): 1025 continue 1026 1027 if '.py' == os.path.splitext(name)[1] and name.startswith(prefix): 1028 # Try to match the regexp pattern, if specified. 1029 if regexp: 1030 import re 1031 if re.search(regexp, name): 1032 #print "Filename: '%s' matches pattern: '%s'" % (name, regexp) 1033 pass 1034 else: 1035 #print "Filename: '%s' does not match pattern: '%s'" % (name, regexp) 1036 continue 1037 1038 # We found a match for our test. Add it to the suite. 1039 1040 # Update the sys.path first. 1041 if not sys.path.count(dir): 1042 sys.path.insert(0, dir) 1043 base = os.path.splitext(name)[0] 1044 1045 # Thoroughly check the filterspec against the base module and admit 1046 # the (base, filterspec) combination only when it makes sense. 1047 filterspec = None 1048 for filterspec in filters: 1049 # Optimistically set the flag to True. 1050 filtered = True 1051 module = __import__(base) 1052 parts = filterspec.split('.') 1053 obj = module 1054 for part in parts: 1055 try: 1056 parent, obj = obj, getattr(obj, part) 1057 except AttributeError: 1058 # The filterspec has failed. 1059 filtered = False 1060 break 1061 1062 # If filtered, we have a good filterspec. Add it. 1063 if filtered: 1064 #print "adding filter spec %s to module %s" % (filterspec, module) 1065 suite.addTests( 1066 unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)) 1067 continue 1068 1069 # Forgo this module if the (base, filterspec) combo is invalid 1070 # and no '-g' option is specified 1071 if filters and fs4all and not filtered: 1072 continue 1073 1074 # Add either the filtered test case(s) (which is done before) or the entire test class. 1075 if not filterspec or not filtered: 1076 # A simple case of just the module name. Also the failover case 1077 # from the filterspec branch when the (base, filterspec) combo 1078 # doesn't make sense. 1079 suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base)) 1080 1081 1082def lldbLoggings(): 1083 """Check and do lldb loggings if necessary.""" 1084 1085 # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is 1086 # defined. Use ${LLDB_LOG} to specify the log file. 1087 ci = lldb.DBG.GetCommandInterpreter() 1088 res = lldb.SBCommandReturnObject() 1089 if ("LLDB_LOG" in os.environ): 1090 if ("LLDB_LOG_OPTION" in os.environ): 1091 lldb_log_option = os.environ["LLDB_LOG_OPTION"] 1092 else: 1093 lldb_log_option = "event process expr state api" 1094 ci.HandleCommand( 1095 "log enable -n -f " + os.environ["LLDB_LOG"] + " lldb " + lldb_log_option, 1096 res) 1097 if not res.Succeeded(): 1098 raise Exception('log enable failed (check LLDB_LOG env variable.') 1099 # Ditto for gdb-remote logging if ${GDB_REMOTE_LOG} environment variable is defined. 1100 # Use ${GDB_REMOTE_LOG} to specify the log file. 1101 if ("GDB_REMOTE_LOG" in os.environ): 1102 if ("GDB_REMOTE_LOG_OPTION" in os.environ): 1103 gdb_remote_log_option = os.environ["GDB_REMOTE_LOG_OPTION"] 1104 else: 1105 gdb_remote_log_option = "packets process" 1106 ci.HandleCommand( 1107 "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " gdb-remote " 1108 + gdb_remote_log_option, 1109 res) 1110 if not res.Succeeded(): 1111 raise Exception('log enable failed (check GDB_REMOTE_LOG env variable.') 1112 1113def getMyCommandLine(): 1114 ps = subprocess.Popen([which('ps'), '-o', "command=CMD", str(os.getpid())], stdout=subprocess.PIPE).communicate()[0] 1115 lines = ps.split('\n') 1116 cmd_line = lines[1] 1117 return cmd_line 1118 1119# ======================================== # 1120# # 1121# Execution of the test driver starts here # 1122# # 1123# ======================================== # 1124 1125def checkDsymForUUIDIsNotOn(): 1126 cmd = ["defaults", "read", "com.apple.DebugSymbols"] 1127 pipe = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) 1128 cmd_output = pipe.stdout.read() 1129 if cmd_output and "DBGFileMappedPaths = " in cmd_output: 1130 print "%s =>" % ' '.join(cmd) 1131 print cmd_output 1132 print "Disable automatic lookup and caching of dSYMs before running the test suite!" 1133 print "Exiting..." 1134 sys.exit(0) 1135 1136# On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults 1137# does not exist before proceeding to running the test suite. 1138if sys.platform.startswith("darwin"): 1139 checkDsymForUUIDIsNotOn() 1140 1141# 1142# Start the actions by first parsing the options while setting up the test 1143# directories, followed by setting up the search paths for lldb utilities; 1144# then, we walk the directory trees and collect the tests into our test suite. 1145# 1146parseOptionsAndInitTestdirs() 1147setupSysPath() 1148 1149# 1150# If '-d' is specified, do a delay of 10 seconds for the debugger to attach. 1151# 1152if delay: 1153 doDelay(10) 1154 1155# 1156# If '-l' is specified, do not skip the long running tests. 1157if not skip_long_running_test: 1158 os.environ["LLDB_SKIP_LONG_RUNNING_TEST"] = "NO" 1159 1160# 1161# Walk through the testdirs while collecting tests. 1162# 1163for testdir in testdirs: 1164 os.path.walk(testdir, visit, 'Test') 1165 1166# 1167# Now that we have loaded all the test cases, run the whole test suite. 1168# 1169 1170# For the time being, let's bracket the test runner within the 1171# lldb.SBDebugger.Initialize()/Terminate() pair. 1172import lldb, atexit 1173# Update: the act of importing lldb now executes lldb.SBDebugger.Initialize(), 1174# there's no need to call it a second time. 1175#lldb.SBDebugger.Initialize() 1176atexit.register(lambda: lldb.SBDebugger.Terminate()) 1177 1178# Create a singleton SBDebugger in the lldb namespace. 1179lldb.DBG = lldb.SBDebugger.Create() 1180 1181# Put the blacklist in the lldb namespace, to be used by lldb.TestBase. 1182lldb.blacklist = blacklist 1183 1184# The pre_flight and post_flight come from reading a config file. 1185lldb.pre_flight = pre_flight 1186lldb.post_flight = post_flight 1187def getsource_if_available(obj): 1188 """ 1189 Return the text of the source code for an object if available. Otherwise, 1190 a print representation is returned. 1191 """ 1192 import inspect 1193 try: 1194 return inspect.getsource(obj) 1195 except: 1196 return repr(obj) 1197 1198if not noHeaders: 1199 print "lldb.pre_flight:", getsource_if_available(lldb.pre_flight) 1200 print "lldb.post_flight:", getsource_if_available(lldb.post_flight) 1201 1202# Put all these test decorators in the lldb namespace. 1203lldb.dont_do_python_api_test = dont_do_python_api_test 1204lldb.just_do_python_api_test = just_do_python_api_test 1205lldb.just_do_benchmarks_test = just_do_benchmarks_test 1206lldb.dont_do_dsym_test = dont_do_dsym_test 1207lldb.dont_do_dwarf_test = dont_do_dwarf_test 1208 1209# Do we need to skip build and cleanup? 1210lldb.skip_build_and_cleanup = skip_build_and_cleanup 1211 1212# Put bmExecutable, bmBreakpointSpec, and bmIterationCount into the lldb namespace, too. 1213lldb.bmExecutable = bmExecutable 1214lldb.bmBreakpointSpec = bmBreakpointSpec 1215lldb.bmIterationCount = bmIterationCount 1216 1217# And don't forget the runHooks! 1218lldb.runHooks = runHooks 1219 1220# Turn on lldb loggings if necessary. 1221lldbLoggings() 1222 1223# Install the control-c handler. 1224unittest2.signals.installHandler() 1225 1226# If sdir_name is not specified through the '-s sdir_name' option, get a 1227# timestamp string and export it as LLDB_SESSION_DIR environment var. This will 1228# be used when/if we want to dump the session info of individual test cases 1229# later on. 1230# 1231# See also TestBase.dumpSessionInfo() in lldbtest.py. 1232import datetime 1233# The windows platforms don't like ':' in the pathname. 1234timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") 1235if not sdir_name: 1236 sdir_name = timestamp_started 1237os.environ["LLDB_SESSION_DIRNAME"] = os.path.join(os.getcwd(), sdir_name) 1238 1239if not noHeaders: 1240 sys.stderr.write("\nSession logs for test failures/errors/unexpected successes" 1241 " will go into directory '%s'\n" % sdir_name) 1242 sys.stderr.write("Command invoked: %s\n" % getMyCommandLine()) 1243 1244if not os.path.isdir(sdir_name): 1245 os.mkdir(sdir_name) 1246where_to_save_session = os.getcwd() 1247fname = os.path.join(sdir_name, "TestStarted") 1248with open(fname, "w") as f: 1249 print >> f, "Test started at: %s\n" % timestamp_started 1250 print >> f, svn_info 1251 print >> f, "Command invoked: %s\n" % getMyCommandLine() 1252 1253# 1254# Invoke the default TextTestRunner to run the test suite, possibly iterating 1255# over different configurations. 1256# 1257 1258iterArchs = False 1259iterCompilers = False 1260 1261if not archs and "archs" in config: 1262 archs = config["archs"] 1263 1264if isinstance(archs, list) and len(archs) >= 1: 1265 iterArchs = True 1266 1267if not compilers and "compilers" in config: 1268 compilers = config["compilers"] 1269 1270# 1271# Add some intervention here to sanity check that the compilers requested are sane. 1272# If found not to be an executable program, the invalid one is dropped from the list. 1273for i in range(len(compilers)): 1274 c = compilers[i] 1275 if which(c): 1276 continue 1277 else: 1278 if sys.platform.startswith("darwin"): 1279 pipe = subprocess.Popen(['xcrun', '-find', c], stdout = subprocess.PIPE, stderr = subprocess.STDOUT) 1280 cmd_output = pipe.stdout.read() 1281 if cmd_output: 1282 if "not found" in cmd_output: 1283 print "dropping %s from the compilers used" % c 1284 compilers.remove(i) 1285 else: 1286 compilers[i] = cmd_output.split('\n')[0] 1287 print "'xcrun -find %s' returning %s" % (c, compilers[i]) 1288 1289if not parsable: 1290 print "compilers=%s" % str(compilers) 1291 1292if not compilers or len(compilers) == 0: 1293 print "No eligible compiler found, exiting." 1294 sys.exit(1) 1295 1296if isinstance(compilers, list) and len(compilers) >= 1: 1297 iterCompilers = True 1298 1299# Make a shallow copy of sys.path, we need to manipulate the search paths later. 1300# This is only necessary if we are relocated and with different configurations. 1301if rdir: 1302 old_sys_path = sys.path[:] 1303# If we iterate on archs or compilers, there is a chance we want to split stderr/stdout. 1304if iterArchs or iterCompilers: 1305 old_stderr = sys.stderr 1306 old_stdout = sys.stdout 1307 new_stderr = None 1308 new_stdout = None 1309 1310# Iterating over all possible architecture and compiler combinations. 1311for ia in range(len(archs) if iterArchs else 1): 1312 archConfig = "" 1313 if iterArchs: 1314 os.environ["ARCH"] = archs[ia] 1315 archConfig = "arch=%s" % archs[ia] 1316 for ic in range(len(compilers) if iterCompilers else 1): 1317 if iterCompilers: 1318 os.environ["CC"] = compilers[ic] 1319 configString = "%s compiler=%s" % (archConfig, compilers[ic]) 1320 else: 1321 configString = archConfig 1322 1323 if iterArchs or iterCompilers: 1324 # Translate ' ' to '-' for pathname component. 1325 from string import maketrans 1326 tbl = maketrans(' ', '-') 1327 configPostfix = configString.translate(tbl) 1328 1329 # Check whether we need to split stderr/stdout into configuration 1330 # specific files. 1331 if old_stderr.name != '<stderr>' and config.get('split_stderr'): 1332 if new_stderr: 1333 new_stderr.close() 1334 new_stderr = open("%s.%s" % (old_stderr.name, configPostfix), "w") 1335 sys.stderr = new_stderr 1336 if old_stdout.name != '<stdout>' and config.get('split_stdout'): 1337 if new_stdout: 1338 new_stdout.close() 1339 new_stdout = open("%s.%s" % (old_stdout.name, configPostfix), "w") 1340 sys.stdout = new_stdout 1341 1342 # If we specified a relocated directory to run the test suite, do 1343 # the extra housekeeping to copy the testdirs to a configStringified 1344 # directory and to update sys.path before invoking the test runner. 1345 # The purpose is to separate the configuration-specific directories 1346 # from each other. 1347 if rdir: 1348 from shutil import copytree, rmtree, ignore_patterns 1349 1350 newrdir = "%s.%s" % (rdir, configPostfix) 1351 1352 # Copy the tree to a new directory with postfix name configPostfix. 1353 if os.path.exists(newrdir): 1354 rmtree(newrdir) 1355 copytree(rdir, newrdir, ignore=ignore_patterns('*.pyc', '*.o', '*.d')) 1356 1357 # Update the LLDB_TEST environment variable to reflect new top 1358 # level test directory. 1359 # 1360 # See also lldbtest.TestBase.setUpClass(cls). 1361 if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test': 1362 os.environ["LLDB_TEST"] = os.path.join(newrdir, 'test') 1363 else: 1364 os.environ["LLDB_TEST"] = newrdir 1365 1366 # And update the Python search paths for modules. 1367 sys.path = [x.replace(rdir, newrdir, 1) for x in old_sys_path] 1368 1369 # Output the configuration. 1370 if not parsable: 1371 sys.stderr.write("\nConfiguration: " + configString + "\n") 1372 1373 #print "sys.stderr name is", sys.stderr.name 1374 #print "sys.stdout name is", sys.stdout.name 1375 1376 # First, write out the number of collected test cases. 1377 if not parsable: 1378 sys.stderr.write(separator + "\n") 1379 sys.stderr.write("Collected %d test%s\n\n" 1380 % (suite.countTestCases(), 1381 suite.countTestCases() != 1 and "s" or "")) 1382 1383 class LLDBTestResult(unittest2.TextTestResult): 1384 """ 1385 Enforce a singleton pattern to allow introspection of test progress. 1386 1387 Overwrite addError(), addFailure(), and addExpectedFailure() methods 1388 to enable each test instance to track its failure/error status. It 1389 is used in the LLDB test framework to emit detailed trace messages 1390 to a log file for easier human inspection of test failres/errors. 1391 """ 1392 __singleton__ = None 1393 __ignore_singleton__ = False 1394 1395 @staticmethod 1396 def getTerminalSize(): 1397 import os 1398 env = os.environ 1399 def ioctl_GWINSZ(fd): 1400 try: 1401 import fcntl, termios, struct, os 1402 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, 1403 '1234')) 1404 except: 1405 return 1406 return cr 1407 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 1408 if not cr: 1409 try: 1410 fd = os.open(os.ctermid(), os.O_RDONLY) 1411 cr = ioctl_GWINSZ(fd) 1412 os.close(fd) 1413 except: 1414 pass 1415 if not cr: 1416 cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) 1417 return int(cr[1]), int(cr[0]) 1418 1419 def __init__(self, *args): 1420 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: 1421 raise Exception("LLDBTestResult instantiated more than once") 1422 super(LLDBTestResult, self).__init__(*args) 1423 LLDBTestResult.__singleton__ = self 1424 # Now put this singleton into the lldb module namespace. 1425 lldb.test_result = self 1426 # Computes the format string for displaying the counter. 1427 global suite 1428 counterWidth = len(str(suite.countTestCases())) 1429 self.fmt = "%" + str(counterWidth) + "d: " 1430 self.indentation = ' ' * (counterWidth + 2) 1431 # This counts from 1 .. suite.countTestCases(). 1432 self.counter = 0 1433 (width, height) = LLDBTestResult.getTerminalSize() 1434 self.progressbar = None 1435 global progress_bar 1436 if width > 10 and not parsable and progress_bar: 1437 try: 1438 self.progressbar = progress.ProgressWithEvents(stdout=self.stream,start=0,end=suite.countTestCases(),width=width-10) 1439 except: 1440 self.progressbar = None 1441 1442 def _config_string(self, test): 1443 compiler = getattr(test, "getCompiler", None) 1444 arch = getattr(test, "getArchitecture", None) 1445 return "%s-%s" % (compiler() if compiler else "", arch() if arch else "") 1446 1447 def _exc_info_to_string(self, err, test): 1448 """Overrides superclass TestResult's method in order to append 1449 our test config info string to the exception info string.""" 1450 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): 1451 return '%sConfig=%s-%s' % (super(LLDBTestResult, self)._exc_info_to_string(err, test), 1452 test.getArchitecture(), 1453 test.getCompiler()) 1454 else: 1455 return super(LLDBTestResult, self)._exc_info_to_string(err, test) 1456 1457 def getDescription(self, test): 1458 doc_first_line = test.shortDescription() 1459 if self.descriptions and doc_first_line: 1460 return '\n'.join((str(test), self.indentation + doc_first_line)) 1461 else: 1462 return str(test) 1463 1464 def getCategoriesForTest(self,test): 1465 if hasattr(test,"_testMethodName"): 1466 test_method = getattr(test,"_testMethodName") 1467 test_method = getattr(test,test_method) 1468 else: 1469 test_method = None 1470 if test_method != None and hasattr(test_method,"getCategories"): 1471 test_categories = test_method.getCategories(test) 1472 elif hasattr(test,"getCategories"): 1473 test_categories = test.getCategories() 1474 elif inspect.ismethod(test) and test.__self__ != None and hasattr(test.__self__,"getCategories"): 1475 test_categories = test.__self__.getCategories() 1476 else: 1477 test_categories = [] 1478 if test_categories == None: 1479 test_categories = [] 1480 return test_categories 1481 1482 def shouldSkipBecauseOfCategories(self,test): 1483 global useCategories 1484 import inspect 1485 if useCategories: 1486 global categoriesList 1487 test_categories = self.getCategoriesForTest(test) 1488 if len(test_categories) == 0 or len(categoriesList & set(test_categories)) == 0: 1489 return True 1490 1491 global skipCategories 1492 for category in skipCategories: 1493 if category in self.getCategoriesForTest(test): 1494 return True 1495 1496 return False 1497 1498 def hardMarkAsSkipped(self,test): 1499 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True 1500 getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 1501 test.__class__.__unittest_skip__ = True 1502 test.__class__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 1503 1504 def startTest(self, test): 1505 if self.shouldSkipBecauseOfCategories(test): 1506 self.hardMarkAsSkipped(test) 1507 self.counter += 1 1508 if self.showAll: 1509 self.stream.write(self.fmt % self.counter) 1510 super(LLDBTestResult, self).startTest(test) 1511 1512 def addSuccess(self, test): 1513 global parsable 1514 super(LLDBTestResult, self).addSuccess(test) 1515 if parsable: 1516 self.stream.write("PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1517 1518 def addError(self, test, err): 1519 global sdir_has_content 1520 global parsable 1521 sdir_has_content = True 1522 super(LLDBTestResult, self).addError(test, err) 1523 method = getattr(test, "markError", None) 1524 if method: 1525 method() 1526 if parsable: 1527 self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1528 1529 def addFailure(self, test, err): 1530 global sdir_has_content 1531 global failuresPerCategory 1532 global parsable 1533 sdir_has_content = True 1534 super(LLDBTestResult, self).addFailure(test, err) 1535 method = getattr(test, "markFailure", None) 1536 if method: 1537 method() 1538 if parsable: 1539 self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1540 if useCategories: 1541 test_categories = self.getCategoriesForTest(test) 1542 for category in test_categories: 1543 if category in failuresPerCategory: 1544 failuresPerCategory[category] = failuresPerCategory[category] + 1 1545 else: 1546 failuresPerCategory[category] = 1 1547 1548 def addExpectedFailure(self, test, err, bugnumber): 1549 global sdir_has_content 1550 global parsable 1551 sdir_has_content = True 1552 super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) 1553 method = getattr(test, "markExpectedFailure", None) 1554 if method: 1555 method(err, bugnumber) 1556 if parsable: 1557 self.stream.write("XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1558 1559 def addSkip(self, test, reason): 1560 global sdir_has_content 1561 global parsable 1562 sdir_has_content = True 1563 super(LLDBTestResult, self).addSkip(test, reason) 1564 method = getattr(test, "markSkippedTest", None) 1565 if method: 1566 method() 1567 if parsable: 1568 self.stream.write("UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % (self._config_string(test), str(test), reason)) 1569 1570 def addUnexpectedSuccess(self, test, bugnumber): 1571 global sdir_has_content 1572 global parsable 1573 sdir_has_content = True 1574 super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) 1575 method = getattr(test, "markUnexpectedSuccess", None) 1576 if method: 1577 method(bugnumber) 1578 if parsable: 1579 self.stream.write("XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1580 1581 if parsable: 1582 v = 0 1583 elif progress_bar: 1584 v = 1 1585 else: 1586 v = verbose 1587 1588 # Invoke the test runner. 1589 if count == 1: 1590 result = unittest2.TextTestRunner(stream=sys.stderr, 1591 verbosity=v, 1592 failfast=failfast, 1593 resultclass=LLDBTestResult).run(suite) 1594 else: 1595 # We are invoking the same test suite more than once. In this case, 1596 # mark __ignore_singleton__ flag as True so the signleton pattern is 1597 # not enforced. 1598 LLDBTestResult.__ignore_singleton__ = True 1599 for i in range(count): 1600 1601 result = unittest2.TextTestRunner(stream=sys.stderr, 1602 verbosity=v, 1603 failfast=failfast, 1604 resultclass=LLDBTestResult).run(suite) 1605 1606 failed = failed or not result.wasSuccessful() 1607 1608if sdir_has_content and not parsable: 1609 sys.stderr.write("Session logs for test failures/errors/unexpected successes" 1610 " can be found in directory '%s'\n" % sdir_name) 1611 1612if useCategories and len(failuresPerCategory) > 0: 1613 sys.stderr.write("Failures per category:\n") 1614 for category in failuresPerCategory: 1615 sys.stderr.write("%s - %d\n" % (category,failuresPerCategory[category])) 1616 1617os.chdir(where_to_save_session) 1618fname = os.path.join(sdir_name, "TestFinished") 1619with open(fname, "w") as f: 1620 print >> f, "Test finished at: %s\n" % datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") 1621 1622# Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined. 1623# This should not be necessary now. 1624if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ): 1625 print "Terminating Test suite..." 1626 subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())]) 1627 1628# Exiting. 1629sys.exit(failed) 1630