lldbtest.py revision 783ac95c2aeaeed69a508535c6e0951fd29a55c8
1""" 2LLDB module which provides the abstract base class of lldb test case. 3 4The concrete subclass can override lldbtest.TesBase in order to inherit the 5common behavior for unitest.TestCase.setUp/tearDown implemented in this file. 6 7The subclass should override the attribute mydir in order for the python runtime 8to locate the individual test cases when running as part of a large test suite 9or when running each test case as a separate python invocation. 10 11./dotest.py provides a test driver which sets up the environment to run the 12entire of part of the test suite . Example: 13 14# Exercises the test suite in the types directory.... 15/Volumes/data/lldb/svn/ToT/test $ ./dotest.py -A x86_64 types 16... 17 18Session logs for test failures/errors/unexpected successes will go into directory '2012-05-16-13_35_42' 19Command invoked: python ./dotest.py -A x86_64 types 20compilers=['clang'] 21 22Configuration: arch=x86_64 compiler=clang 23---------------------------------------------------------------------- 24Collected 72 tests 25 26........................................................................ 27---------------------------------------------------------------------- 28Ran 72 tests in 135.468s 29 30OK 31$ 32""" 33 34import os, sys, traceback 35import re 36from subprocess import * 37import StringIO 38import time 39import types 40import unittest2 41import lldb 42 43# See also dotest.parseOptionsAndInitTestdirs(), where the environment variables 44# LLDB_COMMAND_TRACE and LLDB_DO_CLEANUP are set from '-t' and '-r dir' options. 45 46# By default, traceAlways is False. 47if "LLDB_COMMAND_TRACE" in os.environ and os.environ["LLDB_COMMAND_TRACE"]=="YES": 48 traceAlways = True 49else: 50 traceAlways = False 51 52# By default, doCleanup is True. 53if "LLDB_DO_CLEANUP" in os.environ and os.environ["LLDB_DO_CLEANUP"]=="NO": 54 doCleanup = False 55else: 56 doCleanup = True 57 58 59# 60# Some commonly used assert messages. 61# 62 63COMMAND_FAILED_AS_EXPECTED = "Command has failed as expected" 64 65CURRENT_EXECUTABLE_SET = "Current executable set successfully" 66 67PROCESS_IS_VALID = "Process is valid" 68 69PROCESS_KILLED = "Process is killed successfully" 70 71PROCESS_EXITED = "Process exited successfully" 72 73PROCESS_STOPPED = "Process status should be stopped" 74 75RUN_SUCCEEDED = "Process is launched successfully" 76 77RUN_COMPLETED = "Process exited successfully" 78 79BACKTRACE_DISPLAYED_CORRECTLY = "Backtrace displayed correctly" 80 81BREAKPOINT_CREATED = "Breakpoint created successfully" 82 83BREAKPOINT_STATE_CORRECT = "Breakpoint state is correct" 84 85BREAKPOINT_PENDING_CREATED = "Pending breakpoint created successfully" 86 87BREAKPOINT_HIT_ONCE = "Breakpoint resolved with hit cout = 1" 88 89BREAKPOINT_HIT_TWICE = "Breakpoint resolved with hit cout = 2" 90 91BREAKPOINT_HIT_THRICE = "Breakpoint resolved with hit cout = 3" 92 93OBJECT_PRINTED_CORRECTLY = "Object printed correctly" 94 95SOURCE_DISPLAYED_CORRECTLY = "Source code displayed correctly" 96 97STEP_OUT_SUCCEEDED = "Thread step-out succeeded" 98 99STOPPED_DUE_TO_EXC_BAD_ACCESS = "Process should be stopped due to bad access exception" 100 101STOPPED_DUE_TO_BREAKPOINT = "Process should be stopped due to breakpoint" 102 103STOPPED_DUE_TO_BREAKPOINT_WITH_STOP_REASON_AS = "%s, %s" % ( 104 STOPPED_DUE_TO_BREAKPOINT, "instead, the actual stop reason is: '%s'") 105 106STOPPED_DUE_TO_BREAKPOINT_CONDITION = "Stopped due to breakpoint condition" 107 108STOPPED_DUE_TO_BREAKPOINT_IGNORE_COUNT = "Stopped due to breakpoint and ignore count" 109 110STOPPED_DUE_TO_SIGNAL = "Process state is stopped due to signal" 111 112STOPPED_DUE_TO_STEP_IN = "Process state is stopped due to step in" 113 114STOPPED_DUE_TO_WATCHPOINT = "Process should be stopped due to watchpoint" 115 116DATA_TYPES_DISPLAYED_CORRECTLY = "Data type(s) displayed correctly" 117 118VALID_BREAKPOINT = "Got a valid breakpoint" 119 120VALID_BREAKPOINT_LOCATION = "Got a valid breakpoint location" 121 122VALID_COMMAND_INTERPRETER = "Got a valid command interpreter" 123 124VALID_FILESPEC = "Got a valid filespec" 125 126VALID_MODULE = "Got a valid module" 127 128VALID_PROCESS = "Got a valid process" 129 130VALID_SYMBOL = "Got a valid symbol" 131 132VALID_TARGET = "Got a valid target" 133 134VALID_TYPE = "Got a valid type" 135 136VALID_VARIABLE = "Got a valid variable" 137 138VARIABLES_DISPLAYED_CORRECTLY = "Variable(s) displayed correctly" 139 140WATCHPOINT_CREATED = "Watchpoint created successfully" 141 142def CMD_MSG(str): 143 '''A generic "Command '%s' returns successfully" message generator.''' 144 return "Command '%s' returns successfully" % str 145 146def COMPLETION_MSG(str_before, str_after): 147 '''A generic message generator for the completion mechanism.''' 148 return "'%s' successfully completes to '%s'" % (str_before, str_after) 149 150def EXP_MSG(str, exe): 151 '''A generic "'%s' returns expected result" message generator if exe. 152 Otherwise, it generates "'%s' matches expected result" message.''' 153 return "'%s' %s expected result" % (str, 'returns' if exe else 'matches') 154 155def SETTING_MSG(setting): 156 '''A generic "Value of setting '%s' is correct" message generator.''' 157 return "Value of setting '%s' is correct" % setting 158 159def EnvArray(): 160 """Returns an env variable array from the os.environ map object.""" 161 return map(lambda k,v: k+"="+v, os.environ.keys(), os.environ.values()) 162 163def line_number(filename, string_to_match): 164 """Helper function to return the line number of the first matched string.""" 165 with open(filename, 'r') as f: 166 for i, line in enumerate(f): 167 if line.find(string_to_match) != -1: 168 # Found our match. 169 return i+1 170 raise Exception("Unable to find '%s' within file %s" % (string_to_match, filename)) 171 172def pointer_size(): 173 """Return the pointer size of the host system.""" 174 import ctypes 175 a_pointer = ctypes.c_void_p(0xffff) 176 return 8 * ctypes.sizeof(a_pointer) 177 178def is_exe(fpath): 179 """Returns true if fpath is an executable.""" 180 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 181 182def which(program): 183 """Returns the full path to a program; None otherwise.""" 184 fpath, fname = os.path.split(program) 185 if fpath: 186 if is_exe(program): 187 return program 188 else: 189 for path in os.environ["PATH"].split(os.pathsep): 190 exe_file = os.path.join(path, program) 191 if is_exe(exe_file): 192 return exe_file 193 return None 194 195class recording(StringIO.StringIO): 196 """ 197 A nice little context manager for recording the debugger interactions into 198 our session object. If trace flag is ON, it also emits the interactions 199 into the stderr. 200 """ 201 def __init__(self, test, trace): 202 """Create a StringIO instance; record the session obj and trace flag.""" 203 StringIO.StringIO.__init__(self) 204 # The test might not have undergone the 'setUp(self)' phase yet, so that 205 # the attribute 'session' might not even exist yet. 206 self.session = getattr(test, "session", None) if test else None 207 self.trace = trace 208 209 def __enter__(self): 210 """ 211 Context management protocol on entry to the body of the with statement. 212 Just return the StringIO object. 213 """ 214 return self 215 216 def __exit__(self, type, value, tb): 217 """ 218 Context management protocol on exit from the body of the with statement. 219 If trace is ON, it emits the recordings into stderr. Always add the 220 recordings to our session object. And close the StringIO object, too. 221 """ 222 if self.trace: 223 print >> sys.stderr, self.getvalue() 224 if self.session: 225 print >> self.session, self.getvalue() 226 self.close() 227 228# From 2.7's subprocess.check_output() convenience function. 229# Return a tuple (stdoutdata, stderrdata). 230def system(*popenargs, **kwargs): 231 r"""Run an os command with arguments and return its output as a byte string. 232 233 If the exit code was non-zero it raises a CalledProcessError. The 234 CalledProcessError object will have the return code in the returncode 235 attribute and output in the output attribute. 236 237 The arguments are the same as for the Popen constructor. Example: 238 239 >>> check_output(["ls", "-l", "/dev/null"]) 240 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 241 242 The stdout argument is not allowed as it is used internally. 243 To capture standard error in the result, use stderr=STDOUT. 244 245 >>> check_output(["/bin/sh", "-c", 246 ... "ls -l non_existent_file ; exit 0"], 247 ... stderr=STDOUT) 248 'ls: non_existent_file: No such file or directory\n' 249 """ 250 251 # Assign the sender object to variable 'test' and remove it from kwargs. 252 test = kwargs.pop('sender', None) 253 254 if 'stdout' in kwargs: 255 raise ValueError('stdout argument not allowed, it will be overridden.') 256 process = Popen(stdout=PIPE, stderr=PIPE, *popenargs, **kwargs) 257 pid = process.pid 258 output, error = process.communicate() 259 retcode = process.poll() 260 261 with recording(test, traceAlways) as sbuf: 262 if isinstance(popenargs, types.StringTypes): 263 args = [popenargs] 264 else: 265 args = list(popenargs) 266 print >> sbuf 267 print >> sbuf, "os command:", args 268 print >> sbuf, "with pid:", pid 269 print >> sbuf, "stdout:", output 270 print >> sbuf, "stderr:", error 271 print >> sbuf, "retcode:", retcode 272 print >> sbuf 273 274 if retcode: 275 cmd = kwargs.get("args") 276 if cmd is None: 277 cmd = popenargs[0] 278 raise CalledProcessError(retcode, cmd) 279 return (output, error) 280 281def getsource_if_available(obj): 282 """ 283 Return the text of the source code for an object if available. Otherwise, 284 a print representation is returned. 285 """ 286 import inspect 287 try: 288 return inspect.getsource(obj) 289 except: 290 return repr(obj) 291 292def builder_module(): 293 return __import__("builder_" + sys.platform) 294 295# 296# Decorators for categorizing test cases. 297# 298 299from functools import wraps 300def python_api_test(func): 301 """Decorate the item as a Python API only test.""" 302 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 303 raise Exception("@python_api_test can only be used to decorate a test method") 304 @wraps(func) 305 def wrapper(self, *args, **kwargs): 306 try: 307 if lldb.dont_do_python_api_test: 308 self.skipTest("python api tests") 309 except AttributeError: 310 pass 311 return func(self, *args, **kwargs) 312 313 # Mark this function as such to separate them from lldb command line tests. 314 wrapper.__python_api_test__ = True 315 return wrapper 316 317def benchmarks_test(func): 318 """Decorate the item as a benchmarks test.""" 319 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 320 raise Exception("@benchmarks_test can only be used to decorate a test method") 321 @wraps(func) 322 def wrapper(self, *args, **kwargs): 323 try: 324 if not lldb.just_do_benchmarks_test: 325 self.skipTest("benchmarks tests") 326 except AttributeError: 327 pass 328 return func(self, *args, **kwargs) 329 330 # Mark this function as such to separate them from the regular tests. 331 wrapper.__benchmarks_test__ = True 332 return wrapper 333 334def dsym_test(func): 335 """Decorate the item as a dsym test.""" 336 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 337 raise Exception("@dsym_test can only be used to decorate a test method") 338 @wraps(func) 339 def wrapper(self, *args, **kwargs): 340 try: 341 if lldb.dont_do_dsym_test: 342 self.skipTest("dsym tests") 343 except AttributeError: 344 pass 345 return func(self, *args, **kwargs) 346 347 # Mark this function as such to separate them from the regular tests. 348 wrapper.__dsym_test__ = True 349 return wrapper 350 351def dwarf_test(func): 352 """Decorate the item as a dwarf test.""" 353 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 354 raise Exception("@dwarf_test can only be used to decorate a test method") 355 @wraps(func) 356 def wrapper(self, *args, **kwargs): 357 try: 358 if lldb.dont_do_dwarf_test: 359 self.skipTest("dwarf tests") 360 except AttributeError: 361 pass 362 return func(self, *args, **kwargs) 363 364 # Mark this function as such to separate them from the regular tests. 365 wrapper.__dwarf_test__ = True 366 return wrapper 367 368def expectedFailureClang(func): 369 """Decorate the item as a Clang only expectedFailure.""" 370 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 371 raise Exception("@expectedFailureClang can only be used to decorate a test method") 372 @wraps(func) 373 def wrapper(*args, **kwargs): 374 from unittest2 import case 375 self = args[0] 376 compiler = self.getCompiler() 377 try: 378 func(*args, **kwargs) 379 except Exception: 380 if "clang" in compiler: 381 raise case._ExpectedFailure(sys.exc_info()) 382 else: 383 raise 384 385 if "clang" in compiler: 386 raise case._UnexpectedSuccess 387 return wrapper 388 389def expectedFailurei386(func): 390 """Decorate the item as an i386 only expectedFailure.""" 391 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 392 raise Exception("@expectedFailurei386 can only be used to decorate a test method") 393 @wraps(func) 394 def wrapper(*args, **kwargs): 395 from unittest2 import case 396 self = args[0] 397 arch = self.getArchitecture() 398 try: 399 func(*args, **kwargs) 400 except Exception: 401 if "i386" in arch: 402 raise case._ExpectedFailure(sys.exc_info()) 403 else: 404 raise 405 406 if "i386" in arch: 407 raise case._UnexpectedSuccess 408 return wrapper 409 410class Base(unittest2.TestCase): 411 """ 412 Abstract base for performing lldb (see TestBase) or other generic tests (see 413 BenchBase for one example). lldbtest.Base works with the test driver to 414 accomplish things. 415 416 """ 417 # The concrete subclass should override this attribute. 418 mydir = None 419 420 # Keep track of the old current working directory. 421 oldcwd = None 422 423 def TraceOn(self): 424 """Returns True if we are in trace mode (tracing detailed test execution).""" 425 return traceAlways 426 427 @classmethod 428 def setUpClass(cls): 429 """ 430 Python unittest framework class setup fixture. 431 Do current directory manipulation. 432 """ 433 434 # Fail fast if 'mydir' attribute is not overridden. 435 if not cls.mydir or len(cls.mydir) == 0: 436 raise Exception("Subclasses must override the 'mydir' attribute.") 437 # Save old working directory. 438 cls.oldcwd = os.getcwd() 439 440 # Change current working directory if ${LLDB_TEST} is defined. 441 # See also dotest.py which sets up ${LLDB_TEST}. 442 if ("LLDB_TEST" in os.environ): 443 if traceAlways: 444 print >> sys.stderr, "Change dir to:", os.path.join(os.environ["LLDB_TEST"], cls.mydir) 445 os.chdir(os.path.join(os.environ["LLDB_TEST"], cls.mydir)) 446 447 @classmethod 448 def tearDownClass(cls): 449 """ 450 Python unittest framework class teardown fixture. 451 Do class-wide cleanup. 452 """ 453 454 if doCleanup and not lldb.skip_build_and_cleanup: 455 # First, let's do the platform-specific cleanup. 456 module = builder_module() 457 if not module.cleanup(): 458 raise Exception("Don't know how to do cleanup") 459 460 # Subclass might have specific cleanup function defined. 461 if getattr(cls, "classCleanup", None): 462 if traceAlways: 463 print >> sys.stderr, "Call class-specific cleanup function for class:", cls 464 try: 465 cls.classCleanup() 466 except: 467 exc_type, exc_value, exc_tb = sys.exc_info() 468 traceback.print_exception(exc_type, exc_value, exc_tb) 469 470 # Restore old working directory. 471 if traceAlways: 472 print >> sys.stderr, "Restore dir to:", cls.oldcwd 473 os.chdir(cls.oldcwd) 474 475 @classmethod 476 def skipLongRunningTest(cls): 477 """ 478 By default, we skip long running test case. 479 This can be overridden by passing '-l' to the test driver (dotest.py). 480 """ 481 if "LLDB_SKIP_LONG_RUNNING_TEST" in os.environ and "NO" == os.environ["LLDB_SKIP_LONG_RUNNING_TEST"]: 482 return False 483 else: 484 return True 485 486 def setUp(self): 487 """Fixture for unittest test case setup. 488 489 It works with the test driver to conditionally skip tests and does other 490 initializations.""" 491 #import traceback 492 #traceback.print_stack() 493 494 if "LLDB_EXEC" in os.environ: 495 self.lldbExec = os.environ["LLDB_EXEC"] 496 else: 497 self.lldbExec = None 498 if "LLDB_HERE" in os.environ: 499 self.lldbHere = os.environ["LLDB_HERE"] 500 else: 501 self.lldbHere = None 502 # If we spawn an lldb process for test (via pexpect), do not load the 503 # init file unless told otherwise. 504 if "NO_LLDBINIT" in os.environ and "NO" == os.environ["NO_LLDBINIT"]: 505 self.lldbOption = "" 506 else: 507 self.lldbOption = "--no-lldbinit" 508 509 # Assign the test method name to self.testMethodName. 510 # 511 # For an example of the use of this attribute, look at test/types dir. 512 # There are a bunch of test cases under test/types and we don't want the 513 # module cacheing subsystem to be confused with executable name "a.out" 514 # used for all the test cases. 515 self.testMethodName = self._testMethodName 516 517 # Python API only test is decorated with @python_api_test, 518 # which also sets the "__python_api_test__" attribute of the 519 # function object to True. 520 try: 521 if lldb.just_do_python_api_test: 522 testMethod = getattr(self, self._testMethodName) 523 if getattr(testMethod, "__python_api_test__", False): 524 pass 525 else: 526 self.skipTest("non python api test") 527 except AttributeError: 528 pass 529 530 # Benchmarks test is decorated with @benchmarks_test, 531 # which also sets the "__benchmarks_test__" attribute of the 532 # function object to True. 533 try: 534 if lldb.just_do_benchmarks_test: 535 testMethod = getattr(self, self._testMethodName) 536 if getattr(testMethod, "__benchmarks_test__", False): 537 pass 538 else: 539 self.skipTest("non benchmarks test") 540 except AttributeError: 541 pass 542 543 # This is for the case of directly spawning 'lldb'/'gdb' and interacting 544 # with it using pexpect. 545 self.child = None 546 self.child_prompt = "(lldb) " 547 # If the child is interacting with the embedded script interpreter, 548 # there are two exits required during tear down, first to quit the 549 # embedded script interpreter and second to quit the lldb command 550 # interpreter. 551 self.child_in_script_interpreter = False 552 553 # These are for customized teardown cleanup. 554 self.dict = None 555 self.doTearDownCleanup = False 556 # And in rare cases where there are multiple teardown cleanups. 557 self.dicts = [] 558 self.doTearDownCleanups = False 559 560 # Create a string buffer to record the session info, to be dumped into a 561 # test case specific file if test failure is encountered. 562 self.session = StringIO.StringIO() 563 564 # Optimistically set __errored__, __failed__, __expected__ to False 565 # initially. If the test errored/failed, the session info 566 # (self.session) is then dumped into a session specific file for 567 # diagnosis. 568 self.__errored__ = False 569 self.__failed__ = False 570 self.__expected__ = False 571 # We are also interested in unexpected success. 572 self.__unexpected__ = False 573 # And skipped tests. 574 self.__skipped__ = False 575 576 # See addTearDownHook(self, hook) which allows the client to add a hook 577 # function to be run during tearDown() time. 578 self.hooks = [] 579 580 # See HideStdout(self). 581 self.sys_stdout_hidden = False 582 583 def runHooks(self, child=None, child_prompt=None, use_cmd_api=False): 584 """Perform the run hooks to bring lldb debugger to the desired state. 585 586 By default, expect a pexpect spawned child and child prompt to be 587 supplied (use_cmd_api=False). If use_cmd_api is true, ignore the child 588 and child prompt and use self.runCmd() to run the hooks one by one. 589 590 Note that child is a process spawned by pexpect.spawn(). If not, your 591 test case is mostly likely going to fail. 592 593 See also dotest.py where lldb.runHooks are processed/populated. 594 """ 595 if not lldb.runHooks: 596 self.skipTest("No runhooks specified for lldb, skip the test") 597 if use_cmd_api: 598 for hook in lldb.runhooks: 599 self.runCmd(hook) 600 else: 601 if not child or not child_prompt: 602 self.fail("Both child and child_prompt need to be defined.") 603 for hook in lldb.runHooks: 604 child.sendline(hook) 605 child.expect_exact(child_prompt) 606 607 def HideStdout(self): 608 """Hide output to stdout from the user. 609 610 During test execution, there might be cases where we don't want to show the 611 standard output to the user. For example, 612 613 self.runCmd(r'''sc print "\n\n\tHello!\n"''') 614 615 tests whether command abbreviation for 'script' works or not. There is no 616 need to show the 'Hello' output to the user as long as the 'script' command 617 succeeds and we are not in TraceOn() mode (see the '-t' option). 618 619 In this case, the test method calls self.HideStdout(self) to redirect the 620 sys.stdout to a null device, and restores the sys.stdout upon teardown. 621 622 Note that you should only call this method at most once during a test case 623 execution. Any subsequent call has no effect at all.""" 624 if self.sys_stdout_hidden: 625 return 626 627 self.sys_stdout_hidden = True 628 old_stdout = sys.stdout 629 sys.stdout = open(os.devnull, 'w') 630 def restore_stdout(): 631 sys.stdout = old_stdout 632 self.addTearDownHook(restore_stdout) 633 634 # ======================================================================= 635 # Methods for customized teardown cleanups as well as execution of hooks. 636 # ======================================================================= 637 638 def setTearDownCleanup(self, dictionary=None): 639 """Register a cleanup action at tearDown() time with a dictinary""" 640 self.dict = dictionary 641 self.doTearDownCleanup = True 642 643 def addTearDownCleanup(self, dictionary): 644 """Add a cleanup action at tearDown() time with a dictinary""" 645 self.dicts.append(dictionary) 646 self.doTearDownCleanups = True 647 648 def addTearDownHook(self, hook): 649 """ 650 Add a function to be run during tearDown() time. 651 652 Hooks are executed in a first come first serve manner. 653 """ 654 if callable(hook): 655 with recording(self, traceAlways) as sbuf: 656 print >> sbuf, "Adding tearDown hook:", getsource_if_available(hook) 657 self.hooks.append(hook) 658 659 def tearDown(self): 660 """Fixture for unittest test case teardown.""" 661 #import traceback 662 #traceback.print_stack() 663 664 # This is for the case of directly spawning 'lldb' and interacting with it 665 # using pexpect. 666 import pexpect 667 if self.child and self.child.isalive(): 668 with recording(self, traceAlways) as sbuf: 669 print >> sbuf, "tearing down the child process...." 670 if self.child_in_script_interpreter: 671 self.child.sendline('quit()') 672 self.child.expect_exact(self.child_prompt) 673 self.child.sendline('quit') 674 try: 675 self.child.expect(pexpect.EOF) 676 except: 677 pass 678 # Give it one final blow to make sure the child is terminated. 679 self.child.close() 680 681 # Check and run any hook functions. 682 for hook in reversed(self.hooks): 683 with recording(self, traceAlways) as sbuf: 684 print >> sbuf, "Executing tearDown hook:", getsource_if_available(hook) 685 hook() 686 687 del self.hooks 688 689 # Perform registered teardown cleanup. 690 if doCleanup and self.doTearDownCleanup: 691 self.cleanup(dictionary=self.dict) 692 693 # In rare cases where there are multiple teardown cleanups added. 694 if doCleanup and self.doTearDownCleanups: 695 if self.dicts: 696 for dict in reversed(self.dicts): 697 self.cleanup(dictionary=dict) 698 699 # Decide whether to dump the session info. 700 self.dumpSessionInfo() 701 702 # ========================================================= 703 # Various callbacks to allow introspection of test progress 704 # ========================================================= 705 706 def markError(self): 707 """Callback invoked when an error (unexpected exception) errored.""" 708 self.__errored__ = True 709 with recording(self, False) as sbuf: 710 # False because there's no need to write "ERROR" to the stderr twice. 711 # Once by the Python unittest framework, and a second time by us. 712 print >> sbuf, "ERROR" 713 714 def markFailure(self): 715 """Callback invoked when a failure (test assertion failure) occurred.""" 716 self.__failed__ = True 717 with recording(self, False) as sbuf: 718 # False because there's no need to write "FAIL" to the stderr twice. 719 # Once by the Python unittest framework, and a second time by us. 720 print >> sbuf, "FAIL" 721 722 def markExpectedFailure(self): 723 """Callback invoked when an expected failure/error occurred.""" 724 self.__expected__ = True 725 with recording(self, False) as sbuf: 726 # False because there's no need to write "expected failure" to the 727 # stderr twice. 728 # Once by the Python unittest framework, and a second time by us. 729 print >> sbuf, "expected failure" 730 731 def markSkippedTest(self): 732 """Callback invoked when a test is skipped.""" 733 self.__skipped__ = True 734 with recording(self, False) as sbuf: 735 # False because there's no need to write "skipped test" to the 736 # stderr twice. 737 # Once by the Python unittest framework, and a second time by us. 738 print >> sbuf, "skipped test" 739 740 def markUnexpectedSuccess(self): 741 """Callback invoked when an unexpected success occurred.""" 742 self.__unexpected__ = True 743 with recording(self, False) as sbuf: 744 # False because there's no need to write "unexpected success" to the 745 # stderr twice. 746 # Once by the Python unittest framework, and a second time by us. 747 print >> sbuf, "unexpected success" 748 749 def dumpSessionInfo(self): 750 """ 751 Dump the debugger interactions leading to a test error/failure. This 752 allows for more convenient postmortem analysis. 753 754 See also LLDBTestResult (dotest.py) which is a singlton class derived 755 from TextTestResult and overwrites addError, addFailure, and 756 addExpectedFailure methods to allow us to to mark the test instance as 757 such. 758 """ 759 760 # We are here because self.tearDown() detected that this test instance 761 # either errored or failed. The lldb.test_result singleton contains 762 # two lists (erros and failures) which get populated by the unittest 763 # framework. Look over there for stack trace information. 764 # 765 # The lists contain 2-tuples of TestCase instances and strings holding 766 # formatted tracebacks. 767 # 768 # See http://docs.python.org/library/unittest.html#unittest.TestResult. 769 if self.__errored__: 770 pairs = lldb.test_result.errors 771 prefix = 'Error' 772 elif self.__failed__: 773 pairs = lldb.test_result.failures 774 prefix = 'Failure' 775 elif self.__expected__: 776 pairs = lldb.test_result.expectedFailures 777 prefix = 'ExpectedFailure' 778 elif self.__skipped__: 779 prefix = 'SkippedTest' 780 elif self.__unexpected__: 781 prefix = "UnexpectedSuccess" 782 else: 783 # Simply return, there's no session info to dump! 784 return 785 786 if not self.__unexpected__ and not self.__skipped__: 787 for test, traceback in pairs: 788 if test is self: 789 print >> self.session, traceback 790 791 testMethod = getattr(self, self._testMethodName) 792 if getattr(testMethod, "__benchmarks_test__", False): 793 benchmarks = True 794 else: 795 benchmarks = False 796 797 # This records the compiler version used for the test. 798 system([self.getCompiler(), "-v"], sender=self) 799 800 dname = os.path.join(os.environ["LLDB_TEST"], 801 os.environ["LLDB_SESSION_DIRNAME"]) 802 if not os.path.isdir(dname): 803 os.mkdir(dname) 804 fname = os.path.join(dname, "%s-%s-%s-%s.log" % (prefix, self.getArchitecture(), "_".join(self.getCompiler().split('/')), self.id())) 805 with open(fname, "w") as f: 806 import datetime 807 print >> f, "Session info generated @", datetime.datetime.now().ctime() 808 print >> f, self.session.getvalue() 809 print >> f, "To rerun this test, issue the following command from the 'test' directory:\n" 810 print >> f, "./dotest.py %s -v %s -f %s.%s" % (self.getRunOptions(), 811 ('+b' if benchmarks else '-t'), 812 self.__class__.__name__, 813 self._testMethodName) 814 815 # ==================================================== 816 # Config. methods supported through a plugin interface 817 # (enables reading of the current test configuration) 818 # ==================================================== 819 820 def getArchitecture(self): 821 """Returns the architecture in effect the test suite is running with.""" 822 module = builder_module() 823 return module.getArchitecture() 824 825 def getCompiler(self): 826 """Returns the compiler in effect the test suite is running with.""" 827 module = builder_module() 828 return module.getCompiler() 829 830 def getRunOptions(self): 831 """Command line option for -A and -C to run this test again, called from 832 self.dumpSessionInfo().""" 833 arch = self.getArchitecture() 834 comp = self.getCompiler() 835 if arch: 836 option_str = "-A " + arch 837 else: 838 option_str = "" 839 if comp: 840 option_str += " -C " + comp 841 return option_str 842 843 # ================================================== 844 # Build methods supported through a plugin interface 845 # ================================================== 846 847 def buildDefault(self, architecture=None, compiler=None, dictionary=None, clean=True): 848 """Platform specific way to build the default binaries.""" 849 if lldb.skip_build_and_cleanup: 850 return 851 module = builder_module() 852 if not module.buildDefault(self, architecture, compiler, dictionary, clean): 853 raise Exception("Don't know how to build default binary") 854 855 def buildDsym(self, architecture=None, compiler=None, dictionary=None, clean=True): 856 """Platform specific way to build binaries with dsym info.""" 857 if lldb.skip_build_and_cleanup: 858 return 859 module = builder_module() 860 if not module.buildDsym(self, architecture, compiler, dictionary, clean): 861 raise Exception("Don't know how to build binary with dsym") 862 863 def buildDwarf(self, architecture=None, compiler=None, dictionary=None, clean=True): 864 """Platform specific way to build binaries with dwarf maps.""" 865 if lldb.skip_build_and_cleanup: 866 return 867 module = builder_module() 868 if not module.buildDwarf(self, architecture, compiler, dictionary, clean): 869 raise Exception("Don't know how to build binary with dwarf") 870 871 def cleanup(self, dictionary=None): 872 """Platform specific way to do cleanup after build.""" 873 if lldb.skip_build_and_cleanup: 874 return 875 module = builder_module() 876 if not module.cleanup(self, dictionary): 877 raise Exception("Don't know how to do cleanup with dictionary: "+dictionary) 878 879 880class TestBase(Base): 881 """ 882 This abstract base class is meant to be subclassed. It provides default 883 implementations for setUpClass(), tearDownClass(), setUp(), and tearDown(), 884 among other things. 885 886 Important things for test class writers: 887 888 - Overwrite the mydir class attribute, otherwise your test class won't 889 run. It specifies the relative directory to the top level 'test' so 890 the test harness can change to the correct working directory before 891 running your test. 892 893 - The setUp method sets up things to facilitate subsequent interactions 894 with the debugger as part of the test. These include: 895 - populate the test method name 896 - create/get a debugger set with synchronous mode (self.dbg) 897 - get the command interpreter from with the debugger (self.ci) 898 - create a result object for use with the command interpreter 899 (self.res) 900 - plus other stuffs 901 902 - The tearDown method tries to perform some necessary cleanup on behalf 903 of the test to return the debugger to a good state for the next test. 904 These include: 905 - execute any tearDown hooks registered by the test method with 906 TestBase.addTearDownHook(); examples can be found in 907 settings/TestSettings.py 908 - kill the inferior process associated with each target, if any, 909 and, then delete the target from the debugger's target list 910 - perform build cleanup before running the next test method in the 911 same test class; examples of registering for this service can be 912 found in types/TestIntegerTypes.py with the call: 913 - self.setTearDownCleanup(dictionary=d) 914 915 - Similarly setUpClass and tearDownClass perform classwise setup and 916 teardown fixtures. The tearDownClass method invokes a default build 917 cleanup for the entire test class; also, subclasses can implement the 918 classmethod classCleanup(cls) to perform special class cleanup action. 919 920 - The instance methods runCmd and expect are used heavily by existing 921 test cases to send a command to the command interpreter and to perform 922 string/pattern matching on the output of such command execution. The 923 expect method also provides a mode to peform string/pattern matching 924 without running a command. 925 926 - The build methods buildDefault, buildDsym, and buildDwarf are used to 927 build the binaries used during a particular test scenario. A plugin 928 should be provided for the sys.platform running the test suite. The 929 Mac OS X implementation is located in plugins/darwin.py. 930 """ 931 932 # Maximum allowed attempts when launching the inferior process. 933 # Can be overridden by the LLDB_MAX_LAUNCH_COUNT environment variable. 934 maxLaunchCount = 3; 935 936 # Time to wait before the next launching attempt in second(s). 937 # Can be overridden by the LLDB_TIME_WAIT_NEXT_LAUNCH environment variable. 938 timeWaitNextLaunch = 1.0; 939 940 def doDelay(self): 941 """See option -w of dotest.py.""" 942 if ("LLDB_WAIT_BETWEEN_TEST_CASES" in os.environ and 943 os.environ["LLDB_WAIT_BETWEEN_TEST_CASES"] == 'YES'): 944 waitTime = 1.0 945 if "LLDB_TIME_WAIT_BETWEEN_TEST_CASES" in os.environ: 946 waitTime = float(os.environ["LLDB_TIME_WAIT_BETWEEN_TEST_CASES"]) 947 time.sleep(waitTime) 948 949 # Returns the list of categories to which this test case belongs 950 # by default, look for a ".categories" file, and read its contents 951 # if no such file exists, traverse the hierarchy - we guarantee 952 # a .categories to exist at the top level directory so we do not end up 953 # looping endlessly - subclasses are free to define their own categories 954 # in whatever way makes sense to them 955 def getCategories(self): 956 import inspect 957 import os.path 958 folder = inspect.getfile(self.__class__) 959 folder = os.path.dirname(folder) 960 while folder != '/': 961 categories_file_name = os.path.join(folder,".categories") 962 if os.path.exists(categories_file_name): 963 categories_file = open(categories_file_name,'r') 964 categories = categories_file.readline() 965 categories_file.close() 966 categories = str.replace(categories,'\n','') 967 categories = str.replace(categories,'\r','') 968 return categories.split(',') 969 else: 970 folder = os.path.dirname(folder) 971 continue 972 973 def setUp(self): 974 #import traceback 975 #traceback.print_stack() 976 977 # Works with the test driver to conditionally skip tests via decorators. 978 Base.setUp(self) 979 980 try: 981 if lldb.blacklist: 982 className = self.__class__.__name__ 983 classAndMethodName = "%s.%s" % (className, self._testMethodName) 984 if className in lldb.blacklist: 985 self.skipTest(lldb.blacklist.get(className)) 986 elif classAndMethodName in lldb.blacklist: 987 self.skipTest(lldb.blacklist.get(classAndMethodName)) 988 except AttributeError: 989 pass 990 991 # Insert some delay between successive test cases if specified. 992 self.doDelay() 993 994 if "LLDB_MAX_LAUNCH_COUNT" in os.environ: 995 self.maxLaunchCount = int(os.environ["LLDB_MAX_LAUNCH_COUNT"]) 996 997 if "LLDB_TIME_WAIT_NEXT_LAUNCH" in os.environ: 998 self.timeWaitNextLaunch = float(os.environ["LLDB_TIME_WAIT_NEXT_LAUNCH"]) 999 1000 # Create the debugger instance if necessary. 1001 try: 1002 self.dbg = lldb.DBG 1003 except AttributeError: 1004 self.dbg = lldb.SBDebugger.Create() 1005 1006 if not self.dbg: 1007 raise Exception('Invalid debugger instance') 1008 1009 # We want our debugger to be synchronous. 1010 self.dbg.SetAsync(False) 1011 1012 # Retrieve the associated command interpreter instance. 1013 self.ci = self.dbg.GetCommandInterpreter() 1014 if not self.ci: 1015 raise Exception('Could not get the command interpreter') 1016 1017 # And the result object. 1018 self.res = lldb.SBCommandReturnObject() 1019 1020 # Run global pre-flight code, if defined via the config file. 1021 if lldb.pre_flight: 1022 lldb.pre_flight(self) 1023 1024 def tearDown(self): 1025 #import traceback 1026 #traceback.print_stack() 1027 1028 Base.tearDown(self) 1029 1030 # Delete the target(s) from the debugger as a general cleanup step. 1031 # This includes terminating the process for each target, if any. 1032 # We'd like to reuse the debugger for our next test without incurring 1033 # the initialization overhead. 1034 targets = [] 1035 for target in self.dbg: 1036 if target: 1037 targets.append(target) 1038 process = target.GetProcess() 1039 if process: 1040 rc = self.invoke(process, "Kill") 1041 self.assertTrue(rc.Success(), PROCESS_KILLED) 1042 for target in targets: 1043 self.dbg.DeleteTarget(target) 1044 1045 # Run global post-flight code, if defined via the config file. 1046 if lldb.post_flight: 1047 lldb.post_flight(self) 1048 1049 del self.dbg 1050 1051 def switch_to_thread_with_stop_reason(self, stop_reason): 1052 """ 1053 Run the 'thread list' command, and select the thread with stop reason as 1054 'stop_reason'. If no such thread exists, no select action is done. 1055 """ 1056 from lldbutil import stop_reason_to_str 1057 self.runCmd('thread list') 1058 output = self.res.GetOutput() 1059 thread_line_pattern = re.compile("^[ *] thread #([0-9]+):.*stop reason = %s" % 1060 stop_reason_to_str(stop_reason)) 1061 for line in output.splitlines(): 1062 matched = thread_line_pattern.match(line) 1063 if matched: 1064 self.runCmd('thread select %s' % matched.group(1)) 1065 1066 def runCmd(self, cmd, msg=None, check=True, trace=False): 1067 """ 1068 Ask the command interpreter to handle the command and then check its 1069 return status. 1070 """ 1071 # Fail fast if 'cmd' is not meaningful. 1072 if not cmd or len(cmd) == 0: 1073 raise Exception("Bad 'cmd' parameter encountered") 1074 1075 trace = (True if traceAlways else trace) 1076 1077 running = (cmd.startswith("run") or cmd.startswith("process launch")) 1078 1079 for i in range(self.maxLaunchCount if running else 1): 1080 self.ci.HandleCommand(cmd, self.res) 1081 1082 with recording(self, trace) as sbuf: 1083 print >> sbuf, "runCmd:", cmd 1084 if not check: 1085 print >> sbuf, "check of return status not required" 1086 if self.res.Succeeded(): 1087 print >> sbuf, "output:", self.res.GetOutput() 1088 else: 1089 print >> sbuf, "runCmd failed!" 1090 print >> sbuf, self.res.GetError() 1091 1092 if self.res.Succeeded(): 1093 break 1094 elif running: 1095 # For process launch, wait some time before possible next try. 1096 time.sleep(self.timeWaitNextLaunch) 1097 with recording(self, trace) as sbuf: 1098 print >> sbuf, "Command '" + cmd + "' failed!" 1099 1100 if check: 1101 self.assertTrue(self.res.Succeeded(), 1102 msg if msg else CMD_MSG(cmd)) 1103 1104 def match (self, str, patterns, msg=None, trace=False, error=False, matching=True, exe=True): 1105 """run command in str, and match the result against regexp in patterns returning the match object for the first matching pattern 1106 1107 Otherwise, all the arguments have the same meanings as for the expect function""" 1108 1109 trace = (True if traceAlways else trace) 1110 1111 if exe: 1112 # First run the command. If we are expecting error, set check=False. 1113 # Pass the assert message along since it provides more semantic info. 1114 self.runCmd(str, msg=msg, trace = (True if trace else False), check = not error) 1115 1116 # Then compare the output against expected strings. 1117 output = self.res.GetError() if error else self.res.GetOutput() 1118 1119 # If error is True, the API client expects the command to fail! 1120 if error: 1121 self.assertFalse(self.res.Succeeded(), 1122 "Command '" + str + "' is expected to fail!") 1123 else: 1124 # No execution required, just compare str against the golden input. 1125 output = str 1126 with recording(self, trace) as sbuf: 1127 print >> sbuf, "looking at:", output 1128 1129 # The heading says either "Expecting" or "Not expecting". 1130 heading = "Expecting" if matching else "Not expecting" 1131 1132 for pattern in patterns: 1133 # Match Objects always have a boolean value of True. 1134 match_object = re.search(pattern, output) 1135 matched = bool(match_object) 1136 with recording(self, trace) as sbuf: 1137 print >> sbuf, "%s pattern: %s" % (heading, pattern) 1138 print >> sbuf, "Matched" if matched else "Not matched" 1139 if matched: 1140 break 1141 1142 self.assertTrue(matched if matching else not matched, 1143 msg if msg else EXP_MSG(str, exe)) 1144 1145 return match_object 1146 1147 def expect(self, str, msg=None, patterns=None, startstr=None, endstr=None, substrs=None, trace=False, error=False, matching=True, exe=True): 1148 """ 1149 Similar to runCmd; with additional expect style output matching ability. 1150 1151 Ask the command interpreter to handle the command and then check its 1152 return status. The 'msg' parameter specifies an informational assert 1153 message. We expect the output from running the command to start with 1154 'startstr', matches the substrings contained in 'substrs', and regexp 1155 matches the patterns contained in 'patterns'. 1156 1157 If the keyword argument error is set to True, it signifies that the API 1158 client is expecting the command to fail. In this case, the error stream 1159 from running the command is retrieved and compared against the golden 1160 input, instead. 1161 1162 If the keyword argument matching is set to False, it signifies that the API 1163 client is expecting the output of the command not to match the golden 1164 input. 1165 1166 Finally, the required argument 'str' represents the lldb command to be 1167 sent to the command interpreter. In case the keyword argument 'exe' is 1168 set to False, the 'str' is treated as a string to be matched/not-matched 1169 against the golden input. 1170 """ 1171 trace = (True if traceAlways else trace) 1172 1173 if exe: 1174 # First run the command. If we are expecting error, set check=False. 1175 # Pass the assert message along since it provides more semantic info. 1176 self.runCmd(str, msg=msg, trace = (True if trace else False), check = not error) 1177 1178 # Then compare the output against expected strings. 1179 output = self.res.GetError() if error else self.res.GetOutput() 1180 1181 # If error is True, the API client expects the command to fail! 1182 if error: 1183 self.assertFalse(self.res.Succeeded(), 1184 "Command '" + str + "' is expected to fail!") 1185 else: 1186 # No execution required, just compare str against the golden input. 1187 output = str 1188 with recording(self, trace) as sbuf: 1189 print >> sbuf, "looking at:", output 1190 1191 # The heading says either "Expecting" or "Not expecting". 1192 heading = "Expecting" if matching else "Not expecting" 1193 1194 # Start from the startstr, if specified. 1195 # If there's no startstr, set the initial state appropriately. 1196 matched = output.startswith(startstr) if startstr else (True if matching else False) 1197 1198 if startstr: 1199 with recording(self, trace) as sbuf: 1200 print >> sbuf, "%s start string: %s" % (heading, startstr) 1201 print >> sbuf, "Matched" if matched else "Not matched" 1202 1203 # Look for endstr, if specified. 1204 keepgoing = matched if matching else not matched 1205 if endstr: 1206 matched = output.endswith(endstr) 1207 with recording(self, trace) as sbuf: 1208 print >> sbuf, "%s end string: %s" % (heading, endstr) 1209 print >> sbuf, "Matched" if matched else "Not matched" 1210 1211 # Look for sub strings, if specified. 1212 keepgoing = matched if matching else not matched 1213 if substrs and keepgoing: 1214 for str in substrs: 1215 matched = output.find(str) != -1 1216 with recording(self, trace) as sbuf: 1217 print >> sbuf, "%s sub string: %s" % (heading, str) 1218 print >> sbuf, "Matched" if matched else "Not matched" 1219 keepgoing = matched if matching else not matched 1220 if not keepgoing: 1221 break 1222 1223 # Search for regular expression patterns, if specified. 1224 keepgoing = matched if matching else not matched 1225 if patterns and keepgoing: 1226 for pattern in patterns: 1227 # Match Objects always have a boolean value of True. 1228 matched = bool(re.search(pattern, output)) 1229 with recording(self, trace) as sbuf: 1230 print >> sbuf, "%s pattern: %s" % (heading, pattern) 1231 print >> sbuf, "Matched" if matched else "Not matched" 1232 keepgoing = matched if matching else not matched 1233 if not keepgoing: 1234 break 1235 1236 self.assertTrue(matched if matching else not matched, 1237 msg if msg else EXP_MSG(str, exe)) 1238 1239 def invoke(self, obj, name, trace=False): 1240 """Use reflection to call a method dynamically with no argument.""" 1241 trace = (True if traceAlways else trace) 1242 1243 method = getattr(obj, name) 1244 import inspect 1245 self.assertTrue(inspect.ismethod(method), 1246 name + "is a method name of object: " + str(obj)) 1247 result = method() 1248 with recording(self, trace) as sbuf: 1249 print >> sbuf, str(method) + ":", result 1250 return result 1251 1252 # ================================================= 1253 # Misc. helper methods for debugging test execution 1254 # ================================================= 1255 1256 def DebugSBValue(self, val): 1257 """Debug print a SBValue object, if traceAlways is True.""" 1258 from lldbutil import value_type_to_str 1259 1260 if not traceAlways: 1261 return 1262 1263 err = sys.stderr 1264 err.write(val.GetName() + ":\n") 1265 err.write('\t' + "TypeName -> " + val.GetTypeName() + '\n') 1266 err.write('\t' + "ByteSize -> " + str(val.GetByteSize()) + '\n') 1267 err.write('\t' + "NumChildren -> " + str(val.GetNumChildren()) + '\n') 1268 err.write('\t' + "Value -> " + str(val.GetValue()) + '\n') 1269 err.write('\t' + "ValueAsUnsigned -> " + str(val.GetValueAsUnsigned())+ '\n') 1270 err.write('\t' + "ValueType -> " + value_type_to_str(val.GetValueType()) + '\n') 1271 err.write('\t' + "Summary -> " + str(val.GetSummary()) + '\n') 1272 err.write('\t' + "IsPointerType -> " + str(val.TypeIsPointerType()) + '\n') 1273 err.write('\t' + "Location -> " + val.GetLocation() + '\n') 1274 1275 def DebugSBType(self, type): 1276 """Debug print a SBType object, if traceAlways is True.""" 1277 if not traceAlways: 1278 return 1279 1280 err = sys.stderr 1281 err.write(type.GetName() + ":\n") 1282 err.write('\t' + "ByteSize -> " + str(type.GetByteSize()) + '\n') 1283 err.write('\t' + "IsPointerType -> " + str(type.IsPointerType()) + '\n') 1284 err.write('\t' + "IsReferenceType -> " + str(type.IsReferenceType()) + '\n') 1285 1286 def DebugPExpect(self, child): 1287 """Debug the spwaned pexpect object.""" 1288 if not traceAlways: 1289 return 1290 1291 print child 1292 1293 @classmethod 1294 def RemoveTempFile(cls, file): 1295 if os.path.exists(file): 1296 os.remove(file) 1297