lldbtest.py revision 65040cbce499d67cf9b92fd845b3cfb84efe98f0
1""" 2LLDB module which provides the abstract base class of lldb test case. 3 4The concrete subclass can override lldbtest.TesBase in order to inherit the 5common behavior for unitest.TestCase.setUp/tearDown implemented in this file. 6 7The subclass should override the attribute mydir in order for the python runtime 8to locate the individual test cases when running as part of a large test suite 9or when running each test case as a separate python invocation. 10 11./dotest.py provides a test driver which sets up the environment to run the 12entire test suite. Users who want to run a test case on its own can specify the 13LLDB_TEST and PYTHONPATH environment variables, for example: 14 15$ export LLDB_TEST=$PWD 16$ export PYTHONPATH=/Volumes/data/lldb/svn/trunk/build/Debug/LLDB.framework/Resources/Python:$LLDB_TEST:$LLDB_TEST/plugins:$LLDB_TEST/pexpect-2.4 17$ echo $LLDB_TEST 18/Volumes/data/lldb/svn/trunk/test 19$ echo $PYTHONPATH 20/Volumes/data/lldb/svn/trunk/build/Debug/LLDB.framework/Resources/Python:/Volumes/data/lldb/svn/trunk/test:/Volumes/data/lldb/svn/trunk/test/plugins 21$ python function_types/TestFunctionTypes.py 22. 23---------------------------------------------------------------------- 24Ran 1 test in 0.363s 25 26OK 27$ LLDB_COMMAND_TRACE=YES python array_types/TestArrayTypes.py 28 29... 30 31runCmd: breakpoint set -f main.c -l 42 32output: Breakpoint created: 1: file ='main.c', line = 42, locations = 1 33 34runCmd: run 35output: Launching '/Volumes/data/lldb/svn/trunk/test/array_types/a.out' (x86_64) 36 37... 38 39runCmd: frame variable strings 40output: (char *[4]) strings = { 41 (char *) strings[0] = 0x0000000100000f0c "Hello", 42 (char *) strings[1] = 0x0000000100000f12 "Hola", 43 (char *) strings[2] = 0x0000000100000f17 "Bonjour", 44 (char *) strings[3] = 0x0000000100000f1f "Guten Tag" 45} 46 47runCmd: frame variable char_16 48output: (char [16]) char_16 = { 49 (char) char_16[0] = 'H', 50 (char) char_16[1] = 'e', 51 (char) char_16[2] = 'l', 52 (char) char_16[3] = 'l', 53 (char) char_16[4] = 'o', 54 (char) char_16[5] = ' ', 55 (char) char_16[6] = 'W', 56 (char) char_16[7] = 'o', 57 (char) char_16[8] = 'r', 58 (char) char_16[9] = 'l', 59 (char) char_16[10] = 'd', 60 (char) char_16[11] = '\n', 61 (char) char_16[12] = '\0', 62 (char) char_16[13] = '\0', 63 (char) char_16[14] = '\0', 64 (char) char_16[15] = '\0' 65} 66 67runCmd: frame variable ushort_matrix 68output: (unsigned short [2][3]) ushort_matrix = { 69 (unsigned short [3]) ushort_matrix[0] = { 70 (unsigned short) ushort_matrix[0][0] = 0x0001, 71 (unsigned short) ushort_matrix[0][1] = 0x0002, 72 (unsigned short) ushort_matrix[0][2] = 0x0003 73 }, 74 (unsigned short [3]) ushort_matrix[1] = { 75 (unsigned short) ushort_matrix[1][0] = 0x000b, 76 (unsigned short) ushort_matrix[1][1] = 0x0016, 77 (unsigned short) ushort_matrix[1][2] = 0x0021 78 } 79} 80 81runCmd: frame variable long_6 82output: (long [6]) long_6 = { 83 (long) long_6[0] = 1, 84 (long) long_6[1] = 2, 85 (long) long_6[2] = 3, 86 (long) long_6[3] = 4, 87 (long) long_6[4] = 5, 88 (long) long_6[5] = 6 89} 90 91. 92---------------------------------------------------------------------- 93Ran 1 test in 0.349s 94 95OK 96$ 97""" 98 99import os, sys, traceback 100import re 101from subprocess import * 102import StringIO 103import time 104import types 105import unittest2 106import lldb 107 108# See also dotest.parseOptionsAndInitTestdirs(), where the environment variables 109# LLDB_COMMAND_TRACE and LLDB_DO_CLEANUP are set from '-t' and '-r dir' options. 110 111# By default, traceAlways is False. 112if "LLDB_COMMAND_TRACE" in os.environ and os.environ["LLDB_COMMAND_TRACE"]=="YES": 113 traceAlways = True 114else: 115 traceAlways = False 116 117# By default, doCleanup is True. 118if "LLDB_DO_CLEANUP" in os.environ and os.environ["LLDB_DO_CLEANUP"]=="NO": 119 doCleanup = False 120else: 121 doCleanup = True 122 123 124# 125# Some commonly used assert messages. 126# 127 128COMMAND_FAILED_AS_EXPECTED = "Command has failed as expected" 129 130CURRENT_EXECUTABLE_SET = "Current executable set successfully" 131 132PROCESS_IS_VALID = "Process is valid" 133 134PROCESS_KILLED = "Process is killed successfully" 135 136PROCESS_EXITED = "Process exited successfully" 137 138PROCESS_STOPPED = "Process status should be stopped" 139 140RUN_SUCCEEDED = "Process is launched successfully" 141 142RUN_COMPLETED = "Process exited successfully" 143 144BACKTRACE_DISPLAYED_CORRECTLY = "Backtrace displayed correctly" 145 146BREAKPOINT_CREATED = "Breakpoint created successfully" 147 148BREAKPOINT_STATE_CORRECT = "Breakpoint state is correct" 149 150BREAKPOINT_PENDING_CREATED = "Pending breakpoint created successfully" 151 152BREAKPOINT_HIT_ONCE = "Breakpoint resolved with hit cout = 1" 153 154BREAKPOINT_HIT_TWICE = "Breakpoint resolved with hit cout = 2" 155 156BREAKPOINT_HIT_THRICE = "Breakpoint resolved with hit cout = 3" 157 158OBJECT_PRINTED_CORRECTLY = "Object printed correctly" 159 160SOURCE_DISPLAYED_CORRECTLY = "Source code displayed correctly" 161 162STEP_OUT_SUCCEEDED = "Thread step-out succeeded" 163 164STOPPED_DUE_TO_EXC_BAD_ACCESS = "Process should be stopped due to bad access exception" 165 166STOPPED_DUE_TO_BREAKPOINT = "Process should be stopped due to breakpoint" 167 168STOPPED_DUE_TO_BREAKPOINT_WITH_STOP_REASON_AS = "%s, %s" % ( 169 STOPPED_DUE_TO_BREAKPOINT, "instead, the actual stop reason is: '%s'") 170 171STOPPED_DUE_TO_BREAKPOINT_CONDITION = "Stopped due to breakpoint condition" 172 173STOPPED_DUE_TO_BREAKPOINT_IGNORE_COUNT = "Stopped due to breakpoint and ignore count" 174 175STOPPED_DUE_TO_SIGNAL = "Process state is stopped due to signal" 176 177STOPPED_DUE_TO_STEP_IN = "Process state is stopped due to step in" 178 179DATA_TYPES_DISPLAYED_CORRECTLY = "Data type(s) displayed correctly" 180 181VALID_BREAKPOINT = "Got a valid breakpoint" 182 183VALID_BREAKPOINT_LOCATION = "Got a valid breakpoint location" 184 185VALID_COMMAND_INTERPRETER = "Got a valid command interpreter" 186 187VALID_FILESPEC = "Got a valid filespec" 188 189VALID_MODULE = "Got a valid module" 190 191VALID_PROCESS = "Got a valid process" 192 193VALID_SYMBOL = "Got a valid symbol" 194 195VALID_TARGET = "Got a valid target" 196 197VALID_VARIABLE = "Got a valid variable" 198 199VARIABLES_DISPLAYED_CORRECTLY = "Variable(s) displayed correctly" 200 201 202def CMD_MSG(str): 203 '''A generic "Command '%s' returns successfully" message generator.''' 204 return "Command '%s' returns successfully" % str 205 206def EXP_MSG(str, exe): 207 '''A generic "'%s' returns expected result" message generator if exe. 208 Otherwise, it generates "'%s' matches expected result" message.''' 209 return "'%s' %s expected result" % (str, 'returns' if exe else 'matches') 210 211def SETTING_MSG(setting): 212 '''A generic "Value of setting '%s' is correct" message generator.''' 213 return "Value of setting '%s' is correct" % setting 214 215def EnvArray(): 216 """Returns an env variable array from the os.environ map object.""" 217 return map(lambda k,v: k+"="+v, os.environ.keys(), os.environ.values()) 218 219def line_number(filename, string_to_match): 220 """Helper function to return the line number of the first matched string.""" 221 with open(filename, 'r') as f: 222 for i, line in enumerate(f): 223 if line.find(string_to_match) != -1: 224 # Found our match. 225 return i+1 226 raise Exception("Unable to find '%s' within file %s" % (string_to_match, filename)) 227 228def pointer_size(): 229 """Return the pointer size of the host system.""" 230 import ctypes 231 a_pointer = ctypes.c_void_p(0xffff) 232 return 8 * ctypes.sizeof(a_pointer) 233 234class recording(StringIO.StringIO): 235 """ 236 A nice little context manager for recording the debugger interactions into 237 our session object. If trace flag is ON, it also emits the interactions 238 into the stderr. 239 """ 240 def __init__(self, test, trace): 241 """Create a StringIO instance; record the session obj and trace flag.""" 242 StringIO.StringIO.__init__(self) 243 # The test might not have undergone the 'setUp(self)' phase yet, so that 244 # the attribute 'session' might not even exist yet. 245 self.session = getattr(test, "session", None) if test else None 246 self.trace = trace 247 248 def __enter__(self): 249 """ 250 Context management protocol on entry to the body of the with statement. 251 Just return the StringIO object. 252 """ 253 return self 254 255 def __exit__(self, type, value, tb): 256 """ 257 Context management protocol on exit from the body of the with statement. 258 If trace is ON, it emits the recordings into stderr. Always add the 259 recordings to our session object. And close the StringIO object, too. 260 """ 261 if self.trace: 262 print >> sys.stderr, self.getvalue() 263 if self.session: 264 print >> self.session, self.getvalue() 265 self.close() 266 267# From 2.7's subprocess.check_output() convenience function. 268# Return a tuple (stdoutdata, stderrdata). 269def system(*popenargs, **kwargs): 270 r"""Run command with arguments and return its output as a byte string. 271 272 If the exit code was non-zero it raises a CalledProcessError. The 273 CalledProcessError object will have the return code in the returncode 274 attribute and output in the output attribute. 275 276 The arguments are the same as for the Popen constructor. Example: 277 278 >>> check_output(["ls", "-l", "/dev/null"]) 279 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 280 281 The stdout argument is not allowed as it is used internally. 282 To capture standard error in the result, use stderr=STDOUT. 283 284 >>> check_output(["/bin/sh", "-c", 285 ... "ls -l non_existent_file ; exit 0"], 286 ... stderr=STDOUT) 287 'ls: non_existent_file: No such file or directory\n' 288 """ 289 290 # Assign the sender object to variable 'test' and remove it from kwargs. 291 test = kwargs.pop('sender', None) 292 293 if 'stdout' in kwargs: 294 raise ValueError('stdout argument not allowed, it will be overridden.') 295 process = Popen(stdout=PIPE, stderr=PIPE, *popenargs, **kwargs) 296 output, error = process.communicate() 297 retcode = process.poll() 298 299 with recording(test, traceAlways) as sbuf: 300 if isinstance(popenargs, types.StringTypes): 301 args = [popenargs] 302 else: 303 args = list(popenargs) 304 print >> sbuf 305 print >> sbuf, "os command:", args 306 print >> sbuf, "stdout:", output 307 print >> sbuf, "stderr:", error 308 print >> sbuf, "retcode:", retcode 309 print >> sbuf 310 311 if retcode: 312 cmd = kwargs.get("args") 313 if cmd is None: 314 cmd = popenargs[0] 315 raise CalledProcessError(retcode, cmd) 316 return (output, error) 317 318def getsource_if_available(obj): 319 """ 320 Return the text of the source code for an object if available. Otherwise, 321 a print representation is returned. 322 """ 323 import inspect 324 try: 325 return inspect.getsource(obj) 326 except: 327 return repr(obj) 328 329def builder_module(): 330 return __import__("builder_" + sys.platform) 331 332# 333# Decorators for categorizing test cases. 334# 335 336from functools import wraps 337def python_api_test(func): 338 """Decorate the item as a Python API only test.""" 339 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 340 raise Exception("@python_api_test can only be used to decorate a test method") 341 @wraps(func) 342 def wrapper(self, *args, **kwargs): 343 try: 344 if lldb.dont_do_python_api_test: 345 self.skipTest("python api tests") 346 except AttributeError: 347 pass 348 return func(self, *args, **kwargs) 349 350 # Mark this function as such to separate them from lldb command line tests. 351 wrapper.__python_api_test__ = True 352 return wrapper 353 354def benchmarks_test(func): 355 """Decorate the item as a benchmarks test.""" 356 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 357 raise Exception("@benchmarks_test can only be used to decorate a test method") 358 @wraps(func) 359 def wrapper(self, *args, **kwargs): 360 try: 361 if not lldb.just_do_benchmarks_test: 362 self.skipTest("benchmarks tests") 363 except AttributeError: 364 pass 365 return func(self, *args, **kwargs) 366 367 # Mark this function as such to separate them from the regular tests. 368 wrapper.__benchmarks_test__ = True 369 return wrapper 370 371def expectedFailureClang(func): 372 """Decorate the item as a Clang only expectedFailure.""" 373 if isinstance(func, type) and issubclass(func, unittest2.TestCase): 374 raise Exception("@expectedFailureClang can only be used to decorate a test method") 375 @wraps(func) 376 def wrapper(*args, **kwargs): 377 from unittest2 import case 378 self = args[0] 379 compiler = self.getCompiler() 380 try: 381 func(*args, **kwargs) 382 except Exception, e: 383 if "clang" in compiler: 384 raise case._ExpectedFailure(sys.exc_info()) 385 else: 386 raise e 387 388 if "clang" in compiler: 389 raise case._UnexpectedSuccess 390 return wrapper 391 392class Base(unittest2.TestCase): 393 """ 394 Abstract base for performing lldb (see TestBase) or other generic tests (see 395 BenchBase for one example). lldbtest.Base works with the test driver to 396 accomplish things. 397 398 """ 399 # The concrete subclass should override this attribute. 400 mydir = None 401 402 # Keep track of the old current working directory. 403 oldcwd = None 404 405 def TraceOn(self): 406 """Returns True if we are in trace mode (tracing detailed test execution).""" 407 return traceAlways 408 409 @classmethod 410 def setUpClass(cls): 411 """ 412 Python unittest framework class setup fixture. 413 Do current directory manipulation. 414 """ 415 416 # Fail fast if 'mydir' attribute is not overridden. 417 if not cls.mydir or len(cls.mydir) == 0: 418 raise Exception("Subclasses must override the 'mydir' attribute.") 419 # Save old working directory. 420 cls.oldcwd = os.getcwd() 421 422 # Change current working directory if ${LLDB_TEST} is defined. 423 # See also dotest.py which sets up ${LLDB_TEST}. 424 if ("LLDB_TEST" in os.environ): 425 if traceAlways: 426 print >> sys.stderr, "Change dir to:", os.path.join(os.environ["LLDB_TEST"], cls.mydir) 427 os.chdir(os.path.join(os.environ["LLDB_TEST"], cls.mydir)) 428 429 @classmethod 430 def tearDownClass(cls): 431 """ 432 Python unittest framework class teardown fixture. 433 Do class-wide cleanup. 434 """ 435 436 if doCleanup: 437 # First, let's do the platform-specific cleanup. 438 module = builder_module() 439 if not module.cleanup(): 440 raise Exception("Don't know how to do cleanup") 441 442 # Subclass might have specific cleanup function defined. 443 if getattr(cls, "classCleanup", None): 444 if traceAlways: 445 print >> sys.stderr, "Call class-specific cleanup function for class:", cls 446 try: 447 cls.classCleanup() 448 except: 449 exc_type, exc_value, exc_tb = sys.exc_info() 450 traceback.print_exception(exc_type, exc_value, exc_tb) 451 452 # Restore old working directory. 453 if traceAlways: 454 print >> sys.stderr, "Restore dir to:", cls.oldcwd 455 os.chdir(cls.oldcwd) 456 457 @classmethod 458 def skipLongRunningTest(cls): 459 """ 460 By default, we skip long running test case. 461 This can be overridden by passing '-l' to the test driver (dotest.py). 462 """ 463 if "LLDB_SKIP_LONG_RUNNING_TEST" in os.environ and "NO" == os.environ["LLDB_SKIP_LONG_RUNNING_TEST"]: 464 return False 465 else: 466 return True 467 468 def setUp(self): 469 """Fixture for unittest test case setup. 470 471 It works with the test driver to conditionally skip tests and does other 472 initializations.""" 473 #import traceback 474 #traceback.print_stack() 475 476 if "LLDB_EXEC" in os.environ: 477 self.lldbExec = os.environ["LLDB_EXEC"] 478 479 # Assign the test method name to self.testMethodName. 480 # 481 # For an example of the use of this attribute, look at test/types dir. 482 # There are a bunch of test cases under test/types and we don't want the 483 # module cacheing subsystem to be confused with executable name "a.out" 484 # used for all the test cases. 485 self.testMethodName = self._testMethodName 486 487 # Python API only test is decorated with @python_api_test, 488 # which also sets the "__python_api_test__" attribute of the 489 # function object to True. 490 try: 491 if lldb.just_do_python_api_test: 492 testMethod = getattr(self, self._testMethodName) 493 if getattr(testMethod, "__python_api_test__", False): 494 pass 495 else: 496 self.skipTest("non python api test") 497 except AttributeError: 498 pass 499 500 # Benchmarks test is decorated with @benchmarks_test, 501 # which also sets the "__benchmarks_test__" attribute of the 502 # function object to True. 503 try: 504 if lldb.just_do_benchmarks_test: 505 testMethod = getattr(self, self._testMethodName) 506 if getattr(testMethod, "__benchmarks_test__", False): 507 pass 508 else: 509 self.skipTest("non benchmarks test") 510 except AttributeError: 511 pass 512 513 # This is for the case of directly spawning 'lldb'/'gdb' and interacting 514 # with it using pexpect. 515 self.child = None 516 self.child_prompt = "(lldb) " 517 # If the child is interacting with the embedded script interpreter, 518 # there are two exits required during tear down, first to quit the 519 # embedded script interpreter and second to quit the lldb command 520 # interpreter. 521 self.child_in_script_interpreter = False 522 523 # These are for customized teardown cleanup. 524 self.dict = None 525 self.doTearDownCleanup = False 526 # And in rare cases where there are multiple teardown cleanups. 527 self.dicts = [] 528 self.doTearDownCleanups = False 529 530 # Create a string buffer to record the session info, to be dumped into a 531 # test case specific file if test failure is encountered. 532 self.session = StringIO.StringIO() 533 534 # Optimistically set __errored__, __failed__, __expected__ to False 535 # initially. If the test errored/failed, the session info 536 # (self.session) is then dumped into a session specific file for 537 # diagnosis. 538 self.__errored__ = False 539 self.__failed__ = False 540 self.__expected__ = False 541 # We are also interested in unexpected success. 542 self.__unexpected__ = False 543 # And skipped tests. 544 self.__skipped__ = False 545 546 # See addTearDownHook(self, hook) which allows the client to add a hook 547 # function to be run during tearDown() time. 548 self.hooks = [] 549 550 # See HideStdout(self). 551 self.sys_stdout_hidden = False 552 553 def HideStdout(self): 554 """Hide output to stdout from the user. 555 556 During test execution, there might be cases where we don't want to show the 557 standard output to the user. For example, 558 559 self.runCmd(r'''sc print "\n\n\tHello!\n"''') 560 561 tests whether command abbreviation for 'script' works or not. There is no 562 need to show the 'Hello' output to the user as long as the 'script' command 563 succeeds and we are not in TraceOn() mode (see the '-t' option). 564 565 In this case, the test method calls self.HideStdout(self) to redirect the 566 sys.stdout to a null device, and restores the sys.stdout upon teardown. 567 568 Note that you should only call this method at most once during a test case 569 execution. Any subsequent call has no effect at all.""" 570 if self.sys_stdout_hidden: 571 return 572 573 self.sys_stdout_hidden = True 574 old_stdout = sys.stdout 575 sys.stdout = open(os.devnull, 'w') 576 def restore_stdout(): 577 sys.stdout = old_stdout 578 self.addTearDownHook(restore_stdout) 579 580 # ======================================================================= 581 # Methods for customized teardown cleanups as well as execution of hooks. 582 # ======================================================================= 583 584 def setTearDownCleanup(self, dictionary=None): 585 """Register a cleanup action at tearDown() time with a dictinary""" 586 self.dict = dictionary 587 self.doTearDownCleanup = True 588 589 def addTearDownCleanup(self, dictionary): 590 """Add a cleanup action at tearDown() time with a dictinary""" 591 self.dicts.append(dictionary) 592 self.doTearDownCleanups = True 593 594 def addTearDownHook(self, hook): 595 """ 596 Add a function to be run during tearDown() time. 597 598 Hooks are executed in a first come first serve manner. 599 """ 600 if callable(hook): 601 with recording(self, traceAlways) as sbuf: 602 print >> sbuf, "Adding tearDown hook:", getsource_if_available(hook) 603 self.hooks.append(hook) 604 605 def tearDown(self): 606 """Fixture for unittest test case teardown.""" 607 #import traceback 608 #traceback.print_stack() 609 610 # This is for the case of directly spawning 'lldb' and interacting with it 611 # using pexpect. 612 import pexpect 613 if self.child and self.child.isalive(): 614 with recording(self, traceAlways) as sbuf: 615 print >> sbuf, "tearing down the child process...." 616 if self.child_in_script_interpreter: 617 self.child.sendline('quit()') 618 self.child.expect_exact(self.child_prompt) 619 self.child.sendline('quit') 620 try: 621 self.child.expect(pexpect.EOF) 622 except: 623 pass 624 625 # Check and run any hook functions. 626 for hook in reversed(self.hooks): 627 with recording(self, traceAlways) as sbuf: 628 print >> sbuf, "Executing tearDown hook:", getsource_if_available(hook) 629 hook() 630 631 del self.hooks 632 633 # Perform registered teardown cleanup. 634 if doCleanup and self.doTearDownCleanup: 635 module = builder_module() 636 if not module.cleanup(self, dictionary=self.dict): 637 raise Exception("Don't know how to do cleanup with dictionary: " + self.dict) 638 639 # In rare cases where there are multiple teardown cleanups added. 640 if doCleanup and self.doTearDownCleanups: 641 module = builder_module() 642 if self.dicts: 643 for dict in reversed(self.dicts): 644 if not module.cleanup(self, dictionary=dict): 645 raise Exception("Don't know how to do cleanup with dictionary: " + dict) 646 647 # Decide whether to dump the session info. 648 self.dumpSessionInfo() 649 650 # ========================================================= 651 # Various callbacks to allow introspection of test progress 652 # ========================================================= 653 654 def markError(self): 655 """Callback invoked when an error (unexpected exception) errored.""" 656 self.__errored__ = True 657 with recording(self, False) as sbuf: 658 # False because there's no need to write "ERROR" to the stderr twice. 659 # Once by the Python unittest framework, and a second time by us. 660 print >> sbuf, "ERROR" 661 662 def markFailure(self): 663 """Callback invoked when a failure (test assertion failure) occurred.""" 664 self.__failed__ = True 665 with recording(self, False) as sbuf: 666 # False because there's no need to write "FAIL" to the stderr twice. 667 # Once by the Python unittest framework, and a second time by us. 668 print >> sbuf, "FAIL" 669 670 def markExpectedFailure(self): 671 """Callback invoked when an expected failure/error occurred.""" 672 self.__expected__ = True 673 with recording(self, False) as sbuf: 674 # False because there's no need to write "expected failure" to the 675 # stderr twice. 676 # Once by the Python unittest framework, and a second time by us. 677 print >> sbuf, "expected failure" 678 679 def markSkippedTest(self): 680 """Callback invoked when a test is skipped.""" 681 self.__skipped__ = True 682 with recording(self, False) as sbuf: 683 # False because there's no need to write "skipped test" to the 684 # stderr twice. 685 # Once by the Python unittest framework, and a second time by us. 686 print >> sbuf, "skipped test" 687 688 def markUnexpectedSuccess(self): 689 """Callback invoked when an unexpected success occurred.""" 690 self.__unexpected__ = True 691 with recording(self, False) as sbuf: 692 # False because there's no need to write "unexpected success" to the 693 # stderr twice. 694 # Once by the Python unittest framework, and a second time by us. 695 print >> sbuf, "unexpected success" 696 697 def dumpSessionInfo(self): 698 """ 699 Dump the debugger interactions leading to a test error/failure. This 700 allows for more convenient postmortem analysis. 701 702 See also LLDBTestResult (dotest.py) which is a singlton class derived 703 from TextTestResult and overwrites addError, addFailure, and 704 addExpectedFailure methods to allow us to to mark the test instance as 705 such. 706 """ 707 708 # We are here because self.tearDown() detected that this test instance 709 # either errored or failed. The lldb.test_result singleton contains 710 # two lists (erros and failures) which get populated by the unittest 711 # framework. Look over there for stack trace information. 712 # 713 # The lists contain 2-tuples of TestCase instances and strings holding 714 # formatted tracebacks. 715 # 716 # See http://docs.python.org/library/unittest.html#unittest.TestResult. 717 if self.__errored__: 718 pairs = lldb.test_result.errors 719 prefix = 'Error' 720 elif self.__failed__: 721 pairs = lldb.test_result.failures 722 prefix = 'Failure' 723 elif self.__expected__: 724 pairs = lldb.test_result.expectedFailures 725 prefix = 'ExpectedFailure' 726 elif self.__skipped__: 727 prefix = 'SkippedTest' 728 elif self.__unexpected__: 729 prefix = "UnexpectedSuccess" 730 else: 731 # Simply return, there's no session info to dump! 732 return 733 734 if not self.__unexpected__ and not self.__skipped__: 735 for test, traceback in pairs: 736 if test is self: 737 print >> self.session, traceback 738 739 testMethod = getattr(self, self._testMethodName) 740 if getattr(testMethod, "__benchmarks_test__", False): 741 benchmarks = True 742 else: 743 benchmarks = False 744 745 dname = os.path.join(os.environ["LLDB_TEST"], 746 os.environ["LLDB_SESSION_DIRNAME"]) 747 if not os.path.isdir(dname): 748 os.mkdir(dname) 749 fname = os.path.join(dname, "%s-%s.log" % (prefix, self.id())) 750 with open(fname, "w") as f: 751 import datetime 752 print >> f, "Session info generated @", datetime.datetime.now().ctime() 753 print >> f, self.session.getvalue() 754 print >> f, "To rerun this test, issue the following command from the 'test' directory:\n" 755 print >> f, "./dotest.py %s -v %s -f %s.%s" % (self.getRunOptions(), 756 ('+b' if benchmarks else '-t'), 757 self.__class__.__name__, 758 self._testMethodName) 759 760 # ==================================================== 761 # Config. methods supported through a plugin interface 762 # (enables reading of the current test configuration) 763 # ==================================================== 764 765 def getArchitecture(self): 766 """Returns the architecture in effect the test suite is running with.""" 767 module = builder_module() 768 return module.getArchitecture() 769 770 def getCompiler(self): 771 """Returns the compiler in effect the test suite is running with.""" 772 module = builder_module() 773 return module.getCompiler() 774 775 def getRunOptions(self): 776 """Command line option for -A and -C to run this test again, called from 777 self.dumpSessionInfo().""" 778 arch = self.getArchitecture() 779 comp = self.getCompiler() 780 if not arch and not comp: 781 return "" 782 else: 783 return "%s %s" % ("-A "+arch if arch else "", 784 "-C "+comp if comp else "") 785 786 # ================================================== 787 # Build methods supported through a plugin interface 788 # ================================================== 789 790 def buildDefault(self, architecture=None, compiler=None, dictionary=None): 791 """Platform specific way to build the default binaries.""" 792 module = builder_module() 793 if not module.buildDefault(self, architecture, compiler, dictionary): 794 raise Exception("Don't know how to build default binary") 795 796 def buildDsym(self, architecture=None, compiler=None, dictionary=None): 797 """Platform specific way to build binaries with dsym info.""" 798 module = builder_module() 799 if not module.buildDsym(self, architecture, compiler, dictionary): 800 raise Exception("Don't know how to build binary with dsym") 801 802 def buildDwarf(self, architecture=None, compiler=None, dictionary=None): 803 """Platform specific way to build binaries with dwarf maps.""" 804 module = builder_module() 805 if not module.buildDwarf(self, architecture, compiler, dictionary): 806 raise Exception("Don't know how to build binary with dwarf") 807 808 def cleanup(self, dictionary=None): 809 """Platform specific way to do cleanup after build.""" 810 module = builder_module() 811 if not module.cleanup(self, dictionary): 812 raise Exception("Don't know how to do cleanup") 813 814 815class TestBase(Base): 816 """ 817 This abstract base class is meant to be subclassed. It provides default 818 implementations for setUpClass(), tearDownClass(), setUp(), and tearDown(), 819 among other things. 820 821 Important things for test class writers: 822 823 - Overwrite the mydir class attribute, otherwise your test class won't 824 run. It specifies the relative directory to the top level 'test' so 825 the test harness can change to the correct working directory before 826 running your test. 827 828 - The setUp method sets up things to facilitate subsequent interactions 829 with the debugger as part of the test. These include: 830 - populate the test method name 831 - create/get a debugger set with synchronous mode (self.dbg) 832 - get the command interpreter from with the debugger (self.ci) 833 - create a result object for use with the command interpreter 834 (self.res) 835 - plus other stuffs 836 837 - The tearDown method tries to perform some necessary cleanup on behalf 838 of the test to return the debugger to a good state for the next test. 839 These include: 840 - execute any tearDown hooks registered by the test method with 841 TestBase.addTearDownHook(); examples can be found in 842 settings/TestSettings.py 843 - kill the inferior process associated with each target, if any, 844 and, then delete the target from the debugger's target list 845 - perform build cleanup before running the next test method in the 846 same test class; examples of registering for this service can be 847 found in types/TestIntegerTypes.py with the call: 848 - self.setTearDownCleanup(dictionary=d) 849 850 - Similarly setUpClass and tearDownClass perform classwise setup and 851 teardown fixtures. The tearDownClass method invokes a default build 852 cleanup for the entire test class; also, subclasses can implement the 853 classmethod classCleanup(cls) to perform special class cleanup action. 854 855 - The instance methods runCmd and expect are used heavily by existing 856 test cases to send a command to the command interpreter and to perform 857 string/pattern matching on the output of such command execution. The 858 expect method also provides a mode to peform string/pattern matching 859 without running a command. 860 861 - The build methods buildDefault, buildDsym, and buildDwarf are used to 862 build the binaries used during a particular test scenario. A plugin 863 should be provided for the sys.platform running the test suite. The 864 Mac OS X implementation is located in plugins/darwin.py. 865 """ 866 867 # Maximum allowed attempts when launching the inferior process. 868 # Can be overridden by the LLDB_MAX_LAUNCH_COUNT environment variable. 869 maxLaunchCount = 3; 870 871 # Time to wait before the next launching attempt in second(s). 872 # Can be overridden by the LLDB_TIME_WAIT_NEXT_LAUNCH environment variable. 873 timeWaitNextLaunch = 1.0; 874 875 def doDelay(self): 876 """See option -w of dotest.py.""" 877 if ("LLDB_WAIT_BETWEEN_TEST_CASES" in os.environ and 878 os.environ["LLDB_WAIT_BETWEEN_TEST_CASES"] == 'YES'): 879 waitTime = 1.0 880 if "LLDB_TIME_WAIT_BETWEEN_TEST_CASES" in os.environ: 881 waitTime = float(os.environ["LLDB_TIME_WAIT_BETWEEN_TEST_CASES"]) 882 time.sleep(waitTime) 883 884 def setUp(self): 885 #import traceback 886 #traceback.print_stack() 887 888 # Works with the test driver to conditionally skip tests via decorators. 889 Base.setUp(self) 890 891 try: 892 if lldb.blacklist: 893 className = self.__class__.__name__ 894 classAndMethodName = "%s.%s" % (className, self._testMethodName) 895 if className in lldb.blacklist: 896 self.skipTest(lldb.blacklist.get(className)) 897 elif classAndMethodName in lldb.blacklist: 898 self.skipTest(lldb.blacklist.get(classAndMethodName)) 899 except AttributeError: 900 pass 901 902 # Insert some delay between successive test cases if specified. 903 self.doDelay() 904 905 if "LLDB_MAX_LAUNCH_COUNT" in os.environ: 906 self.maxLaunchCount = int(os.environ["LLDB_MAX_LAUNCH_COUNT"]) 907 908 if "LLDB_TIME_WAIT_NEXT_LAUNCH" in os.environ: 909 self.timeWaitNextLaunch = float(os.environ["LLDB_TIME_WAIT_NEXT_LAUNCH"]) 910 911 # Create the debugger instance if necessary. 912 try: 913 self.dbg = lldb.DBG 914 except AttributeError: 915 self.dbg = lldb.SBDebugger.Create() 916 917 if not self.dbg: 918 raise Exception('Invalid debugger instance') 919 920 # We want our debugger to be synchronous. 921 self.dbg.SetAsync(False) 922 923 # Retrieve the associated command interpreter instance. 924 self.ci = self.dbg.GetCommandInterpreter() 925 if not self.ci: 926 raise Exception('Could not get the command interpreter') 927 928 # And the result object. 929 self.res = lldb.SBCommandReturnObject() 930 931 def tearDown(self): 932 #import traceback 933 #traceback.print_stack() 934 935 Base.tearDown(self) 936 937 # Delete the target(s) from the debugger as a general cleanup step. 938 # This includes terminating the process for each target, if any. 939 # We'd like to reuse the debugger for our next test without incurring 940 # the initialization overhead. 941 targets = [] 942 for target in self.dbg: 943 if target: 944 targets.append(target) 945 process = target.GetProcess() 946 if process: 947 rc = self.invoke(process, "Kill") 948 self.assertTrue(rc.Success(), PROCESS_KILLED) 949 for target in targets: 950 self.dbg.DeleteTarget(target) 951 952 del self.dbg 953 954 def runCmd(self, cmd, msg=None, check=True, trace=False): 955 """ 956 Ask the command interpreter to handle the command and then check its 957 return status. 958 """ 959 # Fail fast if 'cmd' is not meaningful. 960 if not cmd or len(cmd) == 0: 961 raise Exception("Bad 'cmd' parameter encountered") 962 963 trace = (True if traceAlways else trace) 964 965 running = (cmd.startswith("run") or cmd.startswith("process launch")) 966 967 for i in range(self.maxLaunchCount if running else 1): 968 self.ci.HandleCommand(cmd, self.res) 969 970 with recording(self, trace) as sbuf: 971 print >> sbuf, "runCmd:", cmd 972 if not check: 973 print >> sbuf, "check of return status not required" 974 if self.res.Succeeded(): 975 print >> sbuf, "output:", self.res.GetOutput() 976 else: 977 print >> sbuf, "runCmd failed!" 978 print >> sbuf, self.res.GetError() 979 980 if self.res.Succeeded(): 981 break 982 elif running: 983 # For process launch, wait some time before possible next try. 984 time.sleep(self.timeWaitNextLaunch) 985 with recording(self, True) as sbuf: 986 print >> sbuf, "Command '" + cmd + "' failed!" 987 988 if check: 989 self.assertTrue(self.res.Succeeded(), 990 msg if msg else CMD_MSG(cmd)) 991 992 def expect(self, str, msg=None, patterns=None, startstr=None, substrs=None, trace=False, error=False, matching=True, exe=True): 993 """ 994 Similar to runCmd; with additional expect style output matching ability. 995 996 Ask the command interpreter to handle the command and then check its 997 return status. The 'msg' parameter specifies an informational assert 998 message. We expect the output from running the command to start with 999 'startstr', matches the substrings contained in 'substrs', and regexp 1000 matches the patterns contained in 'patterns'. 1001 1002 If the keyword argument error is set to True, it signifies that the API 1003 client is expecting the command to fail. In this case, the error stream 1004 from running the command is retrieved and compared against the golden 1005 input, instead. 1006 1007 If the keyword argument matching is set to False, it signifies that the API 1008 client is expecting the output of the command not to match the golden 1009 input. 1010 1011 Finally, the required argument 'str' represents the lldb command to be 1012 sent to the command interpreter. In case the keyword argument 'exe' is 1013 set to False, the 'str' is treated as a string to be matched/not-matched 1014 against the golden input. 1015 """ 1016 trace = (True if traceAlways else trace) 1017 1018 if exe: 1019 # First run the command. If we are expecting error, set check=False. 1020 # Pass the assert message along since it provides more semantic info. 1021 self.runCmd(str, msg=msg, trace = (True if trace else False), check = not error) 1022 1023 # Then compare the output against expected strings. 1024 output = self.res.GetError() if error else self.res.GetOutput() 1025 1026 # If error is True, the API client expects the command to fail! 1027 if error: 1028 self.assertFalse(self.res.Succeeded(), 1029 "Command '" + str + "' is expected to fail!") 1030 else: 1031 # No execution required, just compare str against the golden input. 1032 output = str 1033 with recording(self, trace) as sbuf: 1034 print >> sbuf, "looking at:", output 1035 1036 # The heading says either "Expecting" or "Not expecting". 1037 heading = "Expecting" if matching else "Not expecting" 1038 1039 # Start from the startstr, if specified. 1040 # If there's no startstr, set the initial state appropriately. 1041 matched = output.startswith(startstr) if startstr else (True if matching else False) 1042 1043 if startstr: 1044 with recording(self, trace) as sbuf: 1045 print >> sbuf, "%s start string: %s" % (heading, startstr) 1046 print >> sbuf, "Matched" if matched else "Not matched" 1047 1048 # Look for sub strings, if specified. 1049 keepgoing = matched if matching else not matched 1050 if substrs and keepgoing: 1051 for str in substrs: 1052 matched = output.find(str) != -1 1053 with recording(self, trace) as sbuf: 1054 print >> sbuf, "%s sub string: %s" % (heading, str) 1055 print >> sbuf, "Matched" if matched else "Not matched" 1056 keepgoing = matched if matching else not matched 1057 if not keepgoing: 1058 break 1059 1060 # Search for regular expression patterns, if specified. 1061 keepgoing = matched if matching else not matched 1062 if patterns and keepgoing: 1063 for pattern in patterns: 1064 # Match Objects always have a boolean value of True. 1065 matched = bool(re.search(pattern, output)) 1066 with recording(self, trace) as sbuf: 1067 print >> sbuf, "%s pattern: %s" % (heading, pattern) 1068 print >> sbuf, "Matched" if matched else "Not matched" 1069 keepgoing = matched if matching else not matched 1070 if not keepgoing: 1071 break 1072 1073 self.assertTrue(matched if matching else not matched, 1074 msg if msg else EXP_MSG(str, exe)) 1075 1076 def invoke(self, obj, name, trace=False): 1077 """Use reflection to call a method dynamically with no argument.""" 1078 trace = (True if traceAlways else trace) 1079 1080 method = getattr(obj, name) 1081 import inspect 1082 self.assertTrue(inspect.ismethod(method), 1083 name + "is a method name of object: " + str(obj)) 1084 result = method() 1085 with recording(self, trace) as sbuf: 1086 print >> sbuf, str(method) + ":", result 1087 return result 1088 1089 # ================================================= 1090 # Misc. helper methods for debugging test execution 1091 # ================================================= 1092 1093 def DebugSBValue(self, val): 1094 """Debug print a SBValue object, if traceAlways is True.""" 1095 from lldbutil import value_type_to_str 1096 1097 if not traceAlways: 1098 return 1099 1100 err = sys.stderr 1101 err.write(val.GetName() + ":\n") 1102 err.write('\t' + "TypeName -> " + val.GetTypeName() + '\n') 1103 err.write('\t' + "ByteSize -> " + str(val.GetByteSize()) + '\n') 1104 err.write('\t' + "NumChildren -> " + str(val.GetNumChildren()) + '\n') 1105 err.write('\t' + "Value -> " + str(val.GetValue()) + '\n') 1106 err.write('\t' + "ValueType -> " + value_type_to_str(val.GetValueType()) + '\n') 1107 err.write('\t' + "Summary -> " + str(val.GetSummary()) + '\n') 1108 err.write('\t' + "IsPointerType -> " + str(val.TypeIsPointerType()) + '\n') 1109 err.write('\t' + "Location -> " + val.GetLocation() + '\n') 1110 1111 def DebugSBType(self, type): 1112 """Debug print a SBType object, if traceAlways is True.""" 1113 if not traceAlways: 1114 return 1115 1116 err = sys.stderr 1117 err.write(type.GetName() + ":\n") 1118 err.write('\t' + "ByteSize -> " + str(type.GetByteSize()) + '\n') 1119 err.write('\t' + "IsPointerType -> " + str(type.IsPointerType()) + '\n') 1120 err.write('\t' + "IsReferenceType -> " + str(type.IsReferenceType()) + '\n') 1121 1122 def DebugPExpect(self, child): 1123 """Debug the spwaned pexpect object.""" 1124 if not traceAlways: 1125 return 1126 1127 print child 1128