doctest.py revision 195648000cd704e9d50dee0e7f082f3eb74d3bd3
1# Module doctest. 2# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). 3# Major enhancements and refactoring by: 4# Jim Fulton 5# Edward Loper 6 7# Provided as-is; use at your own risk; no warranty; no promises; enjoy! 8 9r"""Module doctest -- a framework for running examples in docstrings. 10 11In simplest use, end each module M to be tested with: 12 13def _test(): 14 import doctest 15 doctest.testmod() 16 17if __name__ == "__main__": 18 _test() 19 20Then running the module as a script will cause the examples in the 21docstrings to get executed and verified: 22 23python M.py 24 25This won't display anything unless an example fails, in which case the 26failing example(s) and the cause(s) of the failure(s) are printed to stdout 27(why not stderr? because stderr is a lame hack <0.2 wink>), and the final 28line of output is "Test failed.". 29 30Run it with the -v switch instead: 31 32python M.py -v 33 34and a detailed report of all examples tried is printed to stdout, along 35with assorted summaries at the end. 36 37You can force verbose mode by passing "verbose=True" to testmod, or prohibit 38it by passing "verbose=False". In either of those cases, sys.argv is not 39examined by testmod. 40 41There are a variety of other ways to run doctests, including integration 42with the unittest framework, and support for running non-Python text 43files containing doctests. There are also many ways to override parts 44of doctest's default behaviors. See the Library Reference Manual for 45details. 46""" 47 48__docformat__ = 'reStructuredText en' 49 50__all__ = [ 51 # 0, Option Flags 52 'register_optionflag', 53 'DONT_ACCEPT_TRUE_FOR_1', 54 'DONT_ACCEPT_BLANKLINE', 55 'NORMALIZE_WHITESPACE', 56 'ELLIPSIS', 57 'SKIP', 58 'IGNORE_EXCEPTION_DETAIL', 59 'COMPARISON_FLAGS', 60 'REPORT_UDIFF', 61 'REPORT_CDIFF', 62 'REPORT_NDIFF', 63 'REPORT_ONLY_FIRST_FAILURE', 64 'REPORTING_FLAGS', 65 # 1. Utility Functions 66 'is_private', 67 # 2. Example & DocTest 68 'Example', 69 'DocTest', 70 # 3. Doctest Parser 71 'DocTestParser', 72 # 4. Doctest Finder 73 'DocTestFinder', 74 # 5. Doctest Runner 75 'DocTestRunner', 76 'OutputChecker', 77 'DocTestFailure', 78 'UnexpectedException', 79 'DebugRunner', 80 # 6. Test Functions 81 'testmod', 82 'testfile', 83 'run_docstring_examples', 84 # 7. Tester 85 'Tester', 86 # 8. Unittest Support 87 'DocTestSuite', 88 'DocFileSuite', 89 'set_unittest_reportflags', 90 # 9. Debugging Support 91 'script_from_examples', 92 'testsource', 93 'debug_src', 94 'debug', 95] 96 97import __future__ 98 99import sys, traceback, inspect, linecache, os, re, types 100import unittest, difflib, pdb, tempfile 101import warnings 102from StringIO import StringIO 103 104# Don't whine about the deprecated is_private function in this 105# module's tests. 106warnings.filterwarnings("ignore", "is_private", DeprecationWarning, 107 __name__, 0) 108 109# There are 4 basic classes: 110# - Example: a <source, want> pair, plus an intra-docstring line number. 111# - DocTest: a collection of examples, parsed from a docstring, plus 112# info about where the docstring came from (name, filename, lineno). 113# - DocTestFinder: extracts DocTests from a given object's docstring and 114# its contained objects' docstrings. 115# - DocTestRunner: runs DocTest cases, and accumulates statistics. 116# 117# So the basic picture is: 118# 119# list of: 120# +------+ +---------+ +-------+ 121# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| 122# +------+ +---------+ +-------+ 123# | Example | 124# | ... | 125# | Example | 126# +---------+ 127 128# Option constants. 129 130OPTIONFLAGS_BY_NAME = {} 131def register_optionflag(name): 132 # Create a new flag unless `name` is already known. 133 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) 134 135DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') 136DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') 137NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') 138ELLIPSIS = register_optionflag('ELLIPSIS') 139SKIP = register_optionflag('SKIP') 140IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') 141 142COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | 143 DONT_ACCEPT_BLANKLINE | 144 NORMALIZE_WHITESPACE | 145 ELLIPSIS | 146 SKIP | 147 IGNORE_EXCEPTION_DETAIL) 148 149REPORT_UDIFF = register_optionflag('REPORT_UDIFF') 150REPORT_CDIFF = register_optionflag('REPORT_CDIFF') 151REPORT_NDIFF = register_optionflag('REPORT_NDIFF') 152REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') 153 154REPORTING_FLAGS = (REPORT_UDIFF | 155 REPORT_CDIFF | 156 REPORT_NDIFF | 157 REPORT_ONLY_FIRST_FAILURE) 158 159# Special string markers for use in `want` strings: 160BLANKLINE_MARKER = '<BLANKLINE>' 161ELLIPSIS_MARKER = '...' 162 163###################################################################### 164## Table of Contents 165###################################################################### 166# 1. Utility Functions 167# 2. Example & DocTest -- store test cases 168# 3. DocTest Parser -- extracts examples from strings 169# 4. DocTest Finder -- extracts test cases from objects 170# 5. DocTest Runner -- runs test cases 171# 6. Test Functions -- convenient wrappers for testing 172# 7. Tester Class -- for backwards compatibility 173# 8. Unittest Support 174# 9. Debugging Support 175# 10. Example Usage 176 177###################################################################### 178## 1. Utility Functions 179###################################################################### 180 181def is_private(prefix, base): 182 """prefix, base -> true iff name prefix + "." + base is "private". 183 184 Prefix may be an empty string, and base does not contain a period. 185 Prefix is ignored (although functions you write conforming to this 186 protocol may make use of it). 187 Return true iff base begins with an (at least one) underscore, but 188 does not both begin and end with (at least) two underscores. 189 190 >>> is_private("a.b", "my_func") 191 False 192 >>> is_private("____", "_my_func") 193 True 194 >>> is_private("someclass", "__init__") 195 False 196 >>> is_private("sometypo", "__init_") 197 True 198 >>> is_private("x.y.z", "_") 199 True 200 >>> is_private("_x.y.z", "__") 201 False 202 >>> is_private("", "") # senseless but consistent 203 False 204 """ 205 warnings.warn("is_private is deprecated; it wasn't useful; " 206 "examine DocTestFinder.find() lists instead", 207 DeprecationWarning, stacklevel=2) 208 return base[:1] == "_" and not base[:2] == "__" == base[-2:] 209 210def _extract_future_flags(globs): 211 """ 212 Return the compiler-flags associated with the future features that 213 have been imported into the given namespace (globs). 214 """ 215 flags = 0 216 for fname in __future__.all_feature_names: 217 feature = globs.get(fname, None) 218 if feature is getattr(__future__, fname): 219 flags |= feature.compiler_flag 220 return flags 221 222def _normalize_module(module, depth=2): 223 """ 224 Return the module specified by `module`. In particular: 225 - If `module` is a module, then return module. 226 - If `module` is a string, then import and return the 227 module with that name. 228 - If `module` is None, then return the calling module. 229 The calling module is assumed to be the module of 230 the stack frame at the given depth in the call stack. 231 """ 232 if inspect.ismodule(module): 233 return module 234 elif isinstance(module, (str, unicode)): 235 return __import__(module, globals(), locals(), ["*"]) 236 elif module is None: 237 return sys.modules[sys._getframe(depth).f_globals['__name__']] 238 else: 239 raise TypeError("Expected a module, string, or None") 240 241def _load_testfile(filename, package, module_relative): 242 if module_relative: 243 package = _normalize_module(package, 3) 244 filename = _module_relative_path(package, filename) 245 if hasattr(package, '__loader__'): 246 if hasattr(package.__loader__, 'get_data'): 247 return package.__loader__.get_data(filename), filename 248 return open(filename).read(), filename 249 250def _indent(s, indent=4): 251 """ 252 Add the given number of space characters to the beginning every 253 non-blank line in `s`, and return the result. 254 """ 255 # This regexp matches the start of non-blank lines: 256 return re.sub('(?m)^(?!$)', indent*' ', s) 257 258def _exception_traceback(exc_info): 259 """ 260 Return a string containing a traceback message for the given 261 exc_info tuple (as returned by sys.exc_info()). 262 """ 263 # Get a traceback message. 264 excout = StringIO() 265 exc_type, exc_val, exc_tb = exc_info 266 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) 267 return excout.getvalue() 268 269# Override some StringIO methods. 270class _SpoofOut(StringIO): 271 def getvalue(self): 272 result = StringIO.getvalue(self) 273 # If anything at all was written, make sure there's a trailing 274 # newline. There's no way for the expected output to indicate 275 # that a trailing newline is missing. 276 if result and not result.endswith("\n"): 277 result += "\n" 278 # Prevent softspace from screwing up the next test case, in 279 # case they used print with a trailing comma in an example. 280 if hasattr(self, "softspace"): 281 del self.softspace 282 return result 283 284 def truncate(self, size=None): 285 StringIO.truncate(self, size) 286 if hasattr(self, "softspace"): 287 del self.softspace 288 289# Worst-case linear-time ellipsis matching. 290def _ellipsis_match(want, got): 291 """ 292 Essentially the only subtle case: 293 >>> _ellipsis_match('aa...aa', 'aaa') 294 False 295 """ 296 if ELLIPSIS_MARKER not in want: 297 return want == got 298 299 # Find "the real" strings. 300 ws = want.split(ELLIPSIS_MARKER) 301 assert len(ws) >= 2 302 303 # Deal with exact matches possibly needed at one or both ends. 304 startpos, endpos = 0, len(got) 305 w = ws[0] 306 if w: # starts with exact match 307 if got.startswith(w): 308 startpos = len(w) 309 del ws[0] 310 else: 311 return False 312 w = ws[-1] 313 if w: # ends with exact match 314 if got.endswith(w): 315 endpos -= len(w) 316 del ws[-1] 317 else: 318 return False 319 320 if startpos > endpos: 321 # Exact end matches required more characters than we have, as in 322 # _ellipsis_match('aa...aa', 'aaa') 323 return False 324 325 # For the rest, we only need to find the leftmost non-overlapping 326 # match for each piece. If there's no overall match that way alone, 327 # there's no overall match period. 328 for w in ws: 329 # w may be '' at times, if there are consecutive ellipses, or 330 # due to an ellipsis at the start or end of `want`. That's OK. 331 # Search for an empty string succeeds, and doesn't change startpos. 332 startpos = got.find(w, startpos, endpos) 333 if startpos < 0: 334 return False 335 startpos += len(w) 336 337 return True 338 339def _comment_line(line): 340 "Return a commented form of the given line" 341 line = line.rstrip() 342 if line: 343 return '# '+line 344 else: 345 return '#' 346 347class _OutputRedirectingPdb(pdb.Pdb): 348 """ 349 A specialized version of the python debugger that redirects stdout 350 to a given stream when interacting with the user. Stdout is *not* 351 redirected when traced code is executed. 352 """ 353 def __init__(self, out): 354 self.__out = out 355 pdb.Pdb.__init__(self, stdout=out) 356 357 def trace_dispatch(self, *args): 358 # Redirect stdout to the given stream. 359 save_stdout = sys.stdout 360 sys.stdout = self.__out 361 # Call Pdb's trace dispatch method. 362 try: 363 return pdb.Pdb.trace_dispatch(self, *args) 364 finally: 365 sys.stdout = save_stdout 366 367# [XX] Normalize with respect to os.path.pardir? 368def _module_relative_path(module, path): 369 if not inspect.ismodule(module): 370 raise TypeError, 'Expected a module: %r' % module 371 if path.startswith('/'): 372 raise ValueError, 'Module-relative files may not have absolute paths' 373 374 # Find the base directory for the path. 375 if hasattr(module, '__file__'): 376 # A normal module/package 377 basedir = os.path.split(module.__file__)[0] 378 elif module.__name__ == '__main__': 379 # An interactive session. 380 if len(sys.argv)>0 and sys.argv[0] != '': 381 basedir = os.path.split(sys.argv[0])[0] 382 else: 383 basedir = os.curdir 384 else: 385 # A module w/o __file__ (this includes builtins) 386 raise ValueError("Can't resolve paths relative to the module " + 387 module + " (it has no __file__)") 388 389 # Combine the base directory and the path. 390 return os.path.join(basedir, *(path.split('/'))) 391 392###################################################################### 393## 2. Example & DocTest 394###################################################################### 395## - An "example" is a <source, want> pair, where "source" is a 396## fragment of source code, and "want" is the expected output for 397## "source." The Example class also includes information about 398## where the example was extracted from. 399## 400## - A "doctest" is a collection of examples, typically extracted from 401## a string (such as an object's docstring). The DocTest class also 402## includes information about where the string was extracted from. 403 404class Example: 405 """ 406 A single doctest example, consisting of source code and expected 407 output. `Example` defines the following attributes: 408 409 - source: A single Python statement, always ending with a newline. 410 The constructor adds a newline if needed. 411 412 - want: The expected output from running the source code (either 413 from stdout, or a traceback in case of exception). `want` ends 414 with a newline unless it's empty, in which case it's an empty 415 string. The constructor adds a newline if needed. 416 417 - exc_msg: The exception message generated by the example, if 418 the example is expected to generate an exception; or `None` if 419 it is not expected to generate an exception. This exception 420 message is compared against the return value of 421 `traceback.format_exception_only()`. `exc_msg` ends with a 422 newline unless it's `None`. The constructor adds a newline 423 if needed. 424 425 - lineno: The line number within the DocTest string containing 426 this Example where the Example begins. This line number is 427 zero-based, with respect to the beginning of the DocTest. 428 429 - indent: The example's indentation in the DocTest string. 430 I.e., the number of space characters that preceed the 431 example's first prompt. 432 433 - options: A dictionary mapping from option flags to True or 434 False, which is used to override default options for this 435 example. Any option flags not contained in this dictionary 436 are left at their default value (as specified by the 437 DocTestRunner's optionflags). By default, no options are set. 438 """ 439 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, 440 options=None): 441 # Normalize inputs. 442 if not source.endswith('\n'): 443 source += '\n' 444 if want and not want.endswith('\n'): 445 want += '\n' 446 if exc_msg is not None and not exc_msg.endswith('\n'): 447 exc_msg += '\n' 448 # Store properties. 449 self.source = source 450 self.want = want 451 self.lineno = lineno 452 self.indent = indent 453 if options is None: options = {} 454 self.options = options 455 self.exc_msg = exc_msg 456 457class DocTest: 458 """ 459 A collection of doctest examples that should be run in a single 460 namespace. Each `DocTest` defines the following attributes: 461 462 - examples: the list of examples. 463 464 - globs: The namespace (aka globals) that the examples should 465 be run in. 466 467 - name: A name identifying the DocTest (typically, the name of 468 the object whose docstring this DocTest was extracted from). 469 470 - filename: The name of the file that this DocTest was extracted 471 from, or `None` if the filename is unknown. 472 473 - lineno: The line number within filename where this DocTest 474 begins, or `None` if the line number is unavailable. This 475 line number is zero-based, with respect to the beginning of 476 the file. 477 478 - docstring: The string that the examples were extracted from, 479 or `None` if the string is unavailable. 480 """ 481 def __init__(self, examples, globs, name, filename, lineno, docstring): 482 """ 483 Create a new DocTest containing the given examples. The 484 DocTest's globals are initialized with a copy of `globs`. 485 """ 486 assert not isinstance(examples, basestring), \ 487 "DocTest no longer accepts str; use DocTestParser instead" 488 self.examples = examples 489 self.docstring = docstring 490 self.globs = globs.copy() 491 self.name = name 492 self.filename = filename 493 self.lineno = lineno 494 495 def __repr__(self): 496 if len(self.examples) == 0: 497 examples = 'no examples' 498 elif len(self.examples) == 1: 499 examples = '1 example' 500 else: 501 examples = '%d examples' % len(self.examples) 502 return ('<DocTest %s from %s:%s (%s)>' % 503 (self.name, self.filename, self.lineno, examples)) 504 505 506 # This lets us sort tests by name: 507 def __cmp__(self, other): 508 if not isinstance(other, DocTest): 509 return -1 510 return cmp((self.name, self.filename, self.lineno, id(self)), 511 (other.name, other.filename, other.lineno, id(other))) 512 513###################################################################### 514## 3. DocTestParser 515###################################################################### 516 517class DocTestParser: 518 """ 519 A class used to parse strings containing doctest examples. 520 """ 521 # This regular expression is used to find doctest examples in a 522 # string. It defines three groups: `source` is the source code 523 # (including leading indentation and prompts); `indent` is the 524 # indentation of the first (PS1) line of the source code; and 525 # `want` is the expected output (including leading indentation). 526 _EXAMPLE_RE = re.compile(r''' 527 # Source consists of a PS1 line followed by zero or more PS2 lines. 528 (?P<source> 529 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line 530 (?:\n [ ]* \.\.\. .*)*) # PS2 lines 531 \n? 532 # Want consists of any non-blank lines that do not start with PS1. 533 (?P<want> (?:(?![ ]*$) # Not a blank line 534 (?![ ]*>>>) # Not a line starting with PS1 535 .*$\n? # But any other line 536 )*) 537 ''', re.MULTILINE | re.VERBOSE) 538 539 # A regular expression for handling `want` strings that contain 540 # expected exceptions. It divides `want` into three pieces: 541 # - the traceback header line (`hdr`) 542 # - the traceback stack (`stack`) 543 # - the exception message (`msg`), as generated by 544 # traceback.format_exception_only() 545 # `msg` may have multiple lines. We assume/require that the 546 # exception message is the first non-indented line starting with a word 547 # character following the traceback header line. 548 _EXCEPTION_RE = re.compile(r""" 549 # Grab the traceback header. Different versions of Python have 550 # said different things on the first traceback line. 551 ^(?P<hdr> Traceback\ \( 552 (?: most\ recent\ call\ last 553 | innermost\ last 554 ) \) : 555 ) 556 \s* $ # toss trailing whitespace on the header. 557 (?P<stack> .*?) # don't blink: absorb stuff until... 558 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum. 559 """, re.VERBOSE | re.MULTILINE | re.DOTALL) 560 561 # A callable returning a true value iff its argument is a blank line 562 # or contains a single comment. 563 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match 564 565 def parse(self, string, name='<string>'): 566 """ 567 Divide the given string into examples and intervening text, 568 and return them as a list of alternating Examples and strings. 569 Line numbers for the Examples are 0-based. The optional 570 argument `name` is a name identifying this string, and is only 571 used for error messages. 572 """ 573 string = string.expandtabs() 574 # If all lines begin with the same indentation, then strip it. 575 min_indent = self._min_indent(string) 576 if min_indent > 0: 577 string = '\n'.join([l[min_indent:] for l in string.split('\n')]) 578 579 output = [] 580 charno, lineno = 0, 0 581 # Find all doctest examples in the string: 582 for m in self._EXAMPLE_RE.finditer(string): 583 # Add the pre-example text to `output`. 584 output.append(string[charno:m.start()]) 585 # Update lineno (lines before this example) 586 lineno += string.count('\n', charno, m.start()) 587 # Extract info from the regexp match. 588 (source, options, want, exc_msg) = \ 589 self._parse_example(m, name, lineno) 590 # Create an Example, and add it to the list. 591 if not self._IS_BLANK_OR_COMMENT(source): 592 output.append( Example(source, want, exc_msg, 593 lineno=lineno, 594 indent=min_indent+len(m.group('indent')), 595 options=options) ) 596 # Update lineno (lines inside this example) 597 lineno += string.count('\n', m.start(), m.end()) 598 # Update charno. 599 charno = m.end() 600 # Add any remaining post-example text to `output`. 601 output.append(string[charno:]) 602 return output 603 604 def get_doctest(self, string, globs, name, filename, lineno): 605 """ 606 Extract all doctest examples from the given string, and 607 collect them into a `DocTest` object. 608 609 `globs`, `name`, `filename`, and `lineno` are attributes for 610 the new `DocTest` object. See the documentation for `DocTest` 611 for more information. 612 """ 613 return DocTest(self.get_examples(string, name), globs, 614 name, filename, lineno, string) 615 616 def get_examples(self, string, name='<string>'): 617 """ 618 Extract all doctest examples from the given string, and return 619 them as a list of `Example` objects. Line numbers are 620 0-based, because it's most common in doctests that nothing 621 interesting appears on the same line as opening triple-quote, 622 and so the first interesting line is called \"line 1\" then. 623 624 The optional argument `name` is a name identifying this 625 string, and is only used for error messages. 626 """ 627 return [x for x in self.parse(string, name) 628 if isinstance(x, Example)] 629 630 def _parse_example(self, m, name, lineno): 631 """ 632 Given a regular expression match from `_EXAMPLE_RE` (`m`), 633 return a pair `(source, want)`, where `source` is the matched 634 example's source code (with prompts and indentation stripped); 635 and `want` is the example's expected output (with indentation 636 stripped). 637 638 `name` is the string's name, and `lineno` is the line number 639 where the example starts; both are used for error messages. 640 """ 641 # Get the example's indentation level. 642 indent = len(m.group('indent')) 643 644 # Divide source into lines; check that they're properly 645 # indented; and then strip their indentation & prompts. 646 source_lines = m.group('source').split('\n') 647 self._check_prompt_blank(source_lines, indent, name, lineno) 648 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) 649 source = '\n'.join([sl[indent+4:] for sl in source_lines]) 650 651 # Divide want into lines; check that it's properly indented; and 652 # then strip the indentation. Spaces before the last newline should 653 # be preserved, so plain rstrip() isn't good enough. 654 want = m.group('want') 655 want_lines = want.split('\n') 656 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): 657 del want_lines[-1] # forget final newline & spaces after it 658 self._check_prefix(want_lines, ' '*indent, name, 659 lineno + len(source_lines)) 660 want = '\n'.join([wl[indent:] for wl in want_lines]) 661 662 # If `want` contains a traceback message, then extract it. 663 m = self._EXCEPTION_RE.match(want) 664 if m: 665 exc_msg = m.group('msg') 666 else: 667 exc_msg = None 668 669 # Extract options from the source. 670 options = self._find_options(source, name, lineno) 671 672 return source, options, want, exc_msg 673 674 # This regular expression looks for option directives in the 675 # source code of an example. Option directives are comments 676 # starting with "doctest:". Warning: this may give false 677 # positives for string-literals that contain the string 678 # "#doctest:". Eliminating these false positives would require 679 # actually parsing the string; but we limit them by ignoring any 680 # line containing "#doctest:" that is *followed* by a quote mark. 681 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', 682 re.MULTILINE) 683 684 def _find_options(self, source, name, lineno): 685 """ 686 Return a dictionary containing option overrides extracted from 687 option directives in the given source string. 688 689 `name` is the string's name, and `lineno` is the line number 690 where the example starts; both are used for error messages. 691 """ 692 options = {} 693 # (note: with the current regexp, this will match at most once:) 694 for m in self._OPTION_DIRECTIVE_RE.finditer(source): 695 option_strings = m.group(1).replace(',', ' ').split() 696 for option in option_strings: 697 if (option[0] not in '+-' or 698 option[1:] not in OPTIONFLAGS_BY_NAME): 699 raise ValueError('line %r of the doctest for %s ' 700 'has an invalid option: %r' % 701 (lineno+1, name, option)) 702 flag = OPTIONFLAGS_BY_NAME[option[1:]] 703 options[flag] = (option[0] == '+') 704 if options and self._IS_BLANK_OR_COMMENT(source): 705 raise ValueError('line %r of the doctest for %s has an option ' 706 'directive on a line with no example: %r' % 707 (lineno, name, source)) 708 return options 709 710 # This regular expression finds the indentation of every non-blank 711 # line in a string. 712 _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE) 713 714 def _min_indent(self, s): 715 "Return the minimum indentation of any non-blank line in `s`" 716 indents = [len(indent) for indent in self._INDENT_RE.findall(s)] 717 if len(indents) > 0: 718 return min(indents) 719 else: 720 return 0 721 722 def _check_prompt_blank(self, lines, indent, name, lineno): 723 """ 724 Given the lines of a source string (including prompts and 725 leading indentation), check to make sure that every prompt is 726 followed by a space character. If any line is not followed by 727 a space character, then raise ValueError. 728 """ 729 for i, line in enumerate(lines): 730 if len(line) >= indent+4 and line[indent+3] != ' ': 731 raise ValueError('line %r of the docstring for %s ' 732 'lacks blank after %s: %r' % 733 (lineno+i+1, name, 734 line[indent:indent+3], line)) 735 736 def _check_prefix(self, lines, prefix, name, lineno): 737 """ 738 Check that every line in the given list starts with the given 739 prefix; if any line does not, then raise a ValueError. 740 """ 741 for i, line in enumerate(lines): 742 if line and not line.startswith(prefix): 743 raise ValueError('line %r of the docstring for %s has ' 744 'inconsistent leading whitespace: %r' % 745 (lineno+i+1, name, line)) 746 747 748###################################################################### 749## 4. DocTest Finder 750###################################################################### 751 752class DocTestFinder: 753 """ 754 A class used to extract the DocTests that are relevant to a given 755 object, from its docstring and the docstrings of its contained 756 objects. Doctests can currently be extracted from the following 757 object types: modules, functions, classes, methods, staticmethods, 758 classmethods, and properties. 759 """ 760 761 def __init__(self, verbose=False, parser=DocTestParser(), 762 recurse=True, _namefilter=None, exclude_empty=True): 763 """ 764 Create a new doctest finder. 765 766 The optional argument `parser` specifies a class or 767 function that should be used to create new DocTest objects (or 768 objects that implement the same interface as DocTest). The 769 signature for this factory function should match the signature 770 of the DocTest constructor. 771 772 If the optional argument `recurse` is false, then `find` will 773 only examine the given object, and not any contained objects. 774 775 If the optional argument `exclude_empty` is false, then `find` 776 will include tests for objects with empty docstrings. 777 """ 778 self._parser = parser 779 self._verbose = verbose 780 self._recurse = recurse 781 self._exclude_empty = exclude_empty 782 # _namefilter is undocumented, and exists only for temporary backward- 783 # compatibility support of testmod's deprecated isprivate mess. 784 self._namefilter = _namefilter 785 786 def find(self, obj, name=None, module=None, globs=None, 787 extraglobs=None): 788 """ 789 Return a list of the DocTests that are defined by the given 790 object's docstring, or by any of its contained objects' 791 docstrings. 792 793 The optional parameter `module` is the module that contains 794 the given object. If the module is not specified or is None, then 795 the test finder will attempt to automatically determine the 796 correct module. The object's module is used: 797 798 - As a default namespace, if `globs` is not specified. 799 - To prevent the DocTestFinder from extracting DocTests 800 from objects that are imported from other modules. 801 - To find the name of the file containing the object. 802 - To help find the line number of the object within its 803 file. 804 805 Contained objects whose module does not match `module` are ignored. 806 807 If `module` is False, no attempt to find the module will be made. 808 This is obscure, of use mostly in tests: if `module` is False, or 809 is None but cannot be found automatically, then all objects are 810 considered to belong to the (non-existent) module, so all contained 811 objects will (recursively) be searched for doctests. 812 813 The globals for each DocTest is formed by combining `globs` 814 and `extraglobs` (bindings in `extraglobs` override bindings 815 in `globs`). A new copy of the globals dictionary is created 816 for each DocTest. If `globs` is not specified, then it 817 defaults to the module's `__dict__`, if specified, or {} 818 otherwise. If `extraglobs` is not specified, then it defaults 819 to {}. 820 821 """ 822 # If name was not specified, then extract it from the object. 823 if name is None: 824 name = getattr(obj, '__name__', None) 825 if name is None: 826 raise ValueError("DocTestFinder.find: name must be given " 827 "when obj.__name__ doesn't exist: %r" % 828 (type(obj),)) 829 830 # Find the module that contains the given object (if obj is 831 # a module, then module=obj.). Note: this may fail, in which 832 # case module will be None. 833 if module is False: 834 module = None 835 elif module is None: 836 module = inspect.getmodule(obj) 837 838 # Read the module's source code. This is used by 839 # DocTestFinder._find_lineno to find the line number for a 840 # given object's docstring. 841 try: 842 file = inspect.getsourcefile(obj) or inspect.getfile(obj) 843 source_lines = linecache.getlines(file) 844 if not source_lines: 845 source_lines = None 846 except TypeError: 847 source_lines = None 848 849 # Initialize globals, and merge in extraglobs. 850 if globs is None: 851 if module is None: 852 globs = {} 853 else: 854 globs = module.__dict__.copy() 855 else: 856 globs = globs.copy() 857 if extraglobs is not None: 858 globs.update(extraglobs) 859 860 # Recursively expore `obj`, extracting DocTests. 861 tests = [] 862 self._find(tests, obj, name, module, source_lines, globs, {}) 863 return tests 864 865 def _filter(self, obj, prefix, base): 866 """ 867 Return true if the given object should not be examined. 868 """ 869 return (self._namefilter is not None and 870 self._namefilter(prefix, base)) 871 872 def _from_module(self, module, object): 873 """ 874 Return true if the given object is defined in the given 875 module. 876 """ 877 if module is None: 878 return True 879 elif inspect.isfunction(object): 880 return module.__dict__ is object.func_globals 881 elif inspect.isclass(object): 882 return module.__name__ == object.__module__ 883 elif inspect.getmodule(object) is not None: 884 return module is inspect.getmodule(object) 885 elif hasattr(object, '__module__'): 886 return module.__name__ == object.__module__ 887 elif isinstance(object, property): 888 return True # [XX] no way not be sure. 889 else: 890 raise ValueError("object must be a class or function") 891 892 def _find(self, tests, obj, name, module, source_lines, globs, seen): 893 """ 894 Find tests for the given object and any contained objects, and 895 add them to `tests`. 896 """ 897 if self._verbose: 898 print 'Finding tests in %s' % name 899 900 # If we've already processed this object, then ignore it. 901 if id(obj) in seen: 902 return 903 seen[id(obj)] = 1 904 905 # Find a test for this object, and add it to the list of tests. 906 test = self._get_test(obj, name, module, globs, source_lines) 907 if test is not None: 908 tests.append(test) 909 910 # Look for tests in a module's contained objects. 911 if inspect.ismodule(obj) and self._recurse: 912 for valname, val in obj.__dict__.items(): 913 # Check if this contained object should be ignored. 914 if self._filter(val, name, valname): 915 continue 916 valname = '%s.%s' % (name, valname) 917 # Recurse to functions & classes. 918 if ((inspect.isfunction(val) or inspect.isclass(val)) and 919 self._from_module(module, val)): 920 self._find(tests, val, valname, module, source_lines, 921 globs, seen) 922 923 # Look for tests in a module's __test__ dictionary. 924 if inspect.ismodule(obj) and self._recurse: 925 for valname, val in getattr(obj, '__test__', {}).items(): 926 if not isinstance(valname, basestring): 927 raise ValueError("DocTestFinder.find: __test__ keys " 928 "must be strings: %r" % 929 (type(valname),)) 930 if not (inspect.isfunction(val) or inspect.isclass(val) or 931 inspect.ismethod(val) or inspect.ismodule(val) or 932 isinstance(val, basestring)): 933 raise ValueError("DocTestFinder.find: __test__ values " 934 "must be strings, functions, methods, " 935 "classes, or modules: %r" % 936 (type(val),)) 937 valname = '%s.__test__.%s' % (name, valname) 938 self._find(tests, val, valname, module, source_lines, 939 globs, seen) 940 941 # Look for tests in a class's contained objects. 942 if inspect.isclass(obj) and self._recurse: 943 for valname, val in obj.__dict__.items(): 944 # Check if this contained object should be ignored. 945 if self._filter(val, name, valname): 946 continue 947 # Special handling for staticmethod/classmethod. 948 if isinstance(val, staticmethod): 949 val = getattr(obj, valname) 950 if isinstance(val, classmethod): 951 val = getattr(obj, valname).im_func 952 953 # Recurse to methods, properties, and nested classes. 954 if ((inspect.isfunction(val) or inspect.isclass(val) or 955 isinstance(val, property)) and 956 self._from_module(module, val)): 957 valname = '%s.%s' % (name, valname) 958 self._find(tests, val, valname, module, source_lines, 959 globs, seen) 960 961 def _get_test(self, obj, name, module, globs, source_lines): 962 """ 963 Return a DocTest for the given object, if it defines a docstring; 964 otherwise, return None. 965 """ 966 # Extract the object's docstring. If it doesn't have one, 967 # then return None (no test for this object). 968 if isinstance(obj, basestring): 969 docstring = obj 970 else: 971 try: 972 if obj.__doc__ is None: 973 docstring = '' 974 else: 975 docstring = obj.__doc__ 976 if not isinstance(docstring, basestring): 977 docstring = str(docstring) 978 except (TypeError, AttributeError): 979 docstring = '' 980 981 # Find the docstring's location in the file. 982 lineno = self._find_lineno(obj, source_lines) 983 984 # Don't bother if the docstring is empty. 985 if self._exclude_empty and not docstring: 986 return None 987 988 # Return a DocTest for this object. 989 if module is None: 990 filename = None 991 else: 992 filename = getattr(module, '__file__', module.__name__) 993 if filename[-4:] in (".pyc", ".pyo"): 994 filename = filename[:-1] 995 return self._parser.get_doctest(docstring, globs, name, 996 filename, lineno) 997 998 def _find_lineno(self, obj, source_lines): 999 """ 1000 Return a line number of the given object's docstring. Note: 1001 this method assumes that the object has a docstring. 1002 """ 1003 lineno = None 1004 1005 # Find the line number for modules. 1006 if inspect.ismodule(obj): 1007 lineno = 0 1008 1009 # Find the line number for classes. 1010 # Note: this could be fooled if a class is defined multiple 1011 # times in a single file. 1012 if inspect.isclass(obj): 1013 if source_lines is None: 1014 return None 1015 pat = re.compile(r'^\s*class\s*%s\b' % 1016 getattr(obj, '__name__', '-')) 1017 for i, line in enumerate(source_lines): 1018 if pat.match(line): 1019 lineno = i 1020 break 1021 1022 # Find the line number for functions & methods. 1023 if inspect.ismethod(obj): obj = obj.im_func 1024 if inspect.isfunction(obj): obj = obj.func_code 1025 if inspect.istraceback(obj): obj = obj.tb_frame 1026 if inspect.isframe(obj): obj = obj.f_code 1027 if inspect.iscode(obj): 1028 lineno = getattr(obj, 'co_firstlineno', None)-1 1029 1030 # Find the line number where the docstring starts. Assume 1031 # that it's the first line that begins with a quote mark. 1032 # Note: this could be fooled by a multiline function 1033 # signature, where a continuation line begins with a quote 1034 # mark. 1035 if lineno is not None: 1036 if source_lines is None: 1037 return lineno+1 1038 pat = re.compile('(^|.*:)\s*\w*("|\')') 1039 for lineno in range(lineno, len(source_lines)): 1040 if pat.match(source_lines[lineno]): 1041 return lineno 1042 1043 # We couldn't find the line number. 1044 return None 1045 1046###################################################################### 1047## 5. DocTest Runner 1048###################################################################### 1049 1050class DocTestRunner: 1051 """ 1052 A class used to run DocTest test cases, and accumulate statistics. 1053 The `run` method is used to process a single DocTest case. It 1054 returns a tuple `(f, t)`, where `t` is the number of test cases 1055 tried, and `f` is the number of test cases that failed. 1056 1057 >>> tests = DocTestFinder().find(_TestClass) 1058 >>> runner = DocTestRunner(verbose=False) 1059 >>> for test in tests: 1060 ... print runner.run(test) 1061 (0, 2) 1062 (0, 1) 1063 (0, 2) 1064 (0, 2) 1065 1066 The `summarize` method prints a summary of all the test cases that 1067 have been run by the runner, and returns an aggregated `(f, t)` 1068 tuple: 1069 1070 >>> runner.summarize(verbose=1) 1071 4 items passed all tests: 1072 2 tests in _TestClass 1073 2 tests in _TestClass.__init__ 1074 2 tests in _TestClass.get 1075 1 tests in _TestClass.square 1076 7 tests in 4 items. 1077 7 passed and 0 failed. 1078 Test passed. 1079 (0, 7) 1080 1081 The aggregated number of tried examples and failed examples is 1082 also available via the `tries` and `failures` attributes: 1083 1084 >>> runner.tries 1085 7 1086 >>> runner.failures 1087 0 1088 1089 The comparison between expected outputs and actual outputs is done 1090 by an `OutputChecker`. This comparison may be customized with a 1091 number of option flags; see the documentation for `testmod` for 1092 more information. If the option flags are insufficient, then the 1093 comparison may also be customized by passing a subclass of 1094 `OutputChecker` to the constructor. 1095 1096 The test runner's display output can be controlled in two ways. 1097 First, an output function (`out) can be passed to 1098 `TestRunner.run`; this function will be called with strings that 1099 should be displayed. It defaults to `sys.stdout.write`. If 1100 capturing the output is not sufficient, then the display output 1101 can be also customized by subclassing DocTestRunner, and 1102 overriding the methods `report_start`, `report_success`, 1103 `report_unexpected_exception`, and `report_failure`. 1104 """ 1105 # This divider string is used to separate failure messages, and to 1106 # separate sections of the summary. 1107 DIVIDER = "*" * 70 1108 1109 def __init__(self, checker=None, verbose=None, optionflags=0): 1110 """ 1111 Create a new test runner. 1112 1113 Optional keyword arg `checker` is the `OutputChecker` that 1114 should be used to compare the expected outputs and actual 1115 outputs of doctest examples. 1116 1117 Optional keyword arg 'verbose' prints lots of stuff if true, 1118 only failures if false; by default, it's true iff '-v' is in 1119 sys.argv. 1120 1121 Optional argument `optionflags` can be used to control how the 1122 test runner compares expected output to actual output, and how 1123 it displays failures. See the documentation for `testmod` for 1124 more information. 1125 """ 1126 self._checker = checker or OutputChecker() 1127 if verbose is None: 1128 verbose = '-v' in sys.argv 1129 self._verbose = verbose 1130 self.optionflags = optionflags 1131 self.original_optionflags = optionflags 1132 1133 # Keep track of the examples we've run. 1134 self.tries = 0 1135 self.failures = 0 1136 self._name2ft = {} 1137 1138 # Create a fake output target for capturing doctest output. 1139 self._fakeout = _SpoofOut() 1140 1141 #///////////////////////////////////////////////////////////////// 1142 # Reporting methods 1143 #///////////////////////////////////////////////////////////////// 1144 1145 def report_start(self, out, test, example): 1146 """ 1147 Report that the test runner is about to process the given 1148 example. (Only displays a message if verbose=True) 1149 """ 1150 if self._verbose: 1151 if example.want: 1152 out('Trying:\n' + _indent(example.source) + 1153 'Expecting:\n' + _indent(example.want)) 1154 else: 1155 out('Trying:\n' + _indent(example.source) + 1156 'Expecting nothing\n') 1157 1158 def report_success(self, out, test, example, got): 1159 """ 1160 Report that the given example ran successfully. (Only 1161 displays a message if verbose=True) 1162 """ 1163 if self._verbose: 1164 out("ok\n") 1165 1166 def report_failure(self, out, test, example, got): 1167 """ 1168 Report that the given example failed. 1169 """ 1170 out(self._failure_header(test, example) + 1171 self._checker.output_difference(example, got, self.optionflags)) 1172 1173 def report_unexpected_exception(self, out, test, example, exc_info): 1174 """ 1175 Report that the given example raised an unexpected exception. 1176 """ 1177 out(self._failure_header(test, example) + 1178 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) 1179 1180 def _failure_header(self, test, example): 1181 out = [self.DIVIDER] 1182 if test.filename: 1183 if test.lineno is not None and example.lineno is not None: 1184 lineno = test.lineno + example.lineno + 1 1185 else: 1186 lineno = '?' 1187 out.append('File "%s", line %s, in %s' % 1188 (test.filename, lineno, test.name)) 1189 else: 1190 out.append('Line %s, in %s' % (example.lineno+1, test.name)) 1191 out.append('Failed example:') 1192 source = example.source 1193 out.append(_indent(source)) 1194 return '\n'.join(out) 1195 1196 #///////////////////////////////////////////////////////////////// 1197 # DocTest Running 1198 #///////////////////////////////////////////////////////////////// 1199 1200 def __run(self, test, compileflags, out): 1201 """ 1202 Run the examples in `test`. Write the outcome of each example 1203 with one of the `DocTestRunner.report_*` methods, using the 1204 writer function `out`. `compileflags` is the set of compiler 1205 flags that should be used to execute examples. Return a tuple 1206 `(f, t)`, where `t` is the number of examples tried, and `f` 1207 is the number of examples that failed. The examples are run 1208 in the namespace `test.globs`. 1209 """ 1210 # Keep track of the number of failures and tries. 1211 failures = tries = 0 1212 1213 # Save the option flags (since option directives can be used 1214 # to modify them). 1215 original_optionflags = self.optionflags 1216 1217 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state 1218 1219 check = self._checker.check_output 1220 1221 # Process each example. 1222 for examplenum, example in enumerate(test.examples): 1223 1224 # If REPORT_ONLY_FIRST_FAILURE is set, then supress 1225 # reporting after the first failure. 1226 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and 1227 failures > 0) 1228 1229 # Merge in the example's options. 1230 self.optionflags = original_optionflags 1231 if example.options: 1232 for (optionflag, val) in example.options.items(): 1233 if val: 1234 self.optionflags |= optionflag 1235 else: 1236 self.optionflags &= ~optionflag 1237 1238 # If 'SKIP' is set, then skip this example. 1239 if self.optionflags & SKIP: 1240 continue 1241 1242 # Record that we started this example. 1243 tries += 1 1244 if not quiet: 1245 self.report_start(out, test, example) 1246 1247 # Use a special filename for compile(), so we can retrieve 1248 # the source code during interactive debugging (see 1249 # __patched_linecache_getlines). 1250 filename = '<doctest %s[%d]>' % (test.name, examplenum) 1251 1252 # Run the example in the given context (globs), and record 1253 # any exception that gets raised. (But don't intercept 1254 # keyboard interrupts.) 1255 try: 1256 # Don't blink! This is where the user's code gets run. 1257 exec compile(example.source, filename, "single", 1258 compileflags, 1) in test.globs 1259 self.debugger.set_continue() # ==== Example Finished ==== 1260 exception = None 1261 except KeyboardInterrupt: 1262 raise 1263 except: 1264 exception = sys.exc_info() 1265 self.debugger.set_continue() # ==== Example Finished ==== 1266 1267 got = self._fakeout.getvalue() # the actual output 1268 self._fakeout.truncate(0) 1269 outcome = FAILURE # guilty until proved innocent or insane 1270 1271 # If the example executed without raising any exceptions, 1272 # verify its output. 1273 if exception is None: 1274 if check(example.want, got, self.optionflags): 1275 outcome = SUCCESS 1276 1277 # The example raised an exception: check if it was expected. 1278 else: 1279 exc_info = sys.exc_info() 1280 exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] 1281 if not quiet: 1282 got += _exception_traceback(exc_info) 1283 1284 # If `example.exc_msg` is None, then we weren't expecting 1285 # an exception. 1286 if example.exc_msg is None: 1287 outcome = BOOM 1288 1289 # We expected an exception: see whether it matches. 1290 elif check(example.exc_msg, exc_msg, self.optionflags): 1291 outcome = SUCCESS 1292 1293 # Another chance if they didn't care about the detail. 1294 elif self.optionflags & IGNORE_EXCEPTION_DETAIL: 1295 m1 = re.match(r'[^:]*:', example.exc_msg) 1296 m2 = re.match(r'[^:]*:', exc_msg) 1297 if m1 and m2 and check(m1.group(0), m2.group(0), 1298 self.optionflags): 1299 outcome = SUCCESS 1300 1301 # Report the outcome. 1302 if outcome is SUCCESS: 1303 if not quiet: 1304 self.report_success(out, test, example, got) 1305 elif outcome is FAILURE: 1306 if not quiet: 1307 self.report_failure(out, test, example, got) 1308 failures += 1 1309 elif outcome is BOOM: 1310 if not quiet: 1311 self.report_unexpected_exception(out, test, example, 1312 exc_info) 1313 failures += 1 1314 else: 1315 assert False, ("unknown outcome", outcome) 1316 1317 # Restore the option flags (in case they were modified) 1318 self.optionflags = original_optionflags 1319 1320 # Record and return the number of failures and tries. 1321 self.__record_outcome(test, failures, tries) 1322 return failures, tries 1323 1324 def __record_outcome(self, test, f, t): 1325 """ 1326 Record the fact that the given DocTest (`test`) generated `f` 1327 failures out of `t` tried examples. 1328 """ 1329 f2, t2 = self._name2ft.get(test.name, (0,0)) 1330 self._name2ft[test.name] = (f+f2, t+t2) 1331 self.failures += f 1332 self.tries += t 1333 1334 __LINECACHE_FILENAME_RE = re.compile(r'<doctest ' 1335 r'(?P<name>[\w\.]+)' 1336 r'\[(?P<examplenum>\d+)\]>$') 1337 def __patched_linecache_getlines(self, filename, module_globals=None): 1338 m = self.__LINECACHE_FILENAME_RE.match(filename) 1339 if m and m.group('name') == self.test.name: 1340 example = self.test.examples[int(m.group('examplenum'))] 1341 return example.source.splitlines(True) 1342 else: 1343 return self.save_linecache_getlines(filename, module_globals) 1344 1345 def run(self, test, compileflags=None, out=None, clear_globs=True): 1346 """ 1347 Run the examples in `test`, and display the results using the 1348 writer function `out`. 1349 1350 The examples are run in the namespace `test.globs`. If 1351 `clear_globs` is true (the default), then this namespace will 1352 be cleared after the test runs, to help with garbage 1353 collection. If you would like to examine the namespace after 1354 the test completes, then use `clear_globs=False`. 1355 1356 `compileflags` gives the set of flags that should be used by 1357 the Python compiler when running the examples. If not 1358 specified, then it will default to the set of future-import 1359 flags that apply to `globs`. 1360 1361 The output of each example is checked using 1362 `DocTestRunner.check_output`, and the results are formatted by 1363 the `DocTestRunner.report_*` methods. 1364 """ 1365 self.test = test 1366 1367 if compileflags is None: 1368 compileflags = _extract_future_flags(test.globs) 1369 1370 save_stdout = sys.stdout 1371 if out is None: 1372 out = save_stdout.write 1373 sys.stdout = self._fakeout 1374 1375 # Patch pdb.set_trace to restore sys.stdout during interactive 1376 # debugging (so it's not still redirected to self._fakeout). 1377 # Note that the interactive output will go to *our* 1378 # save_stdout, even if that's not the real sys.stdout; this 1379 # allows us to write test cases for the set_trace behavior. 1380 save_set_trace = pdb.set_trace 1381 self.debugger = _OutputRedirectingPdb(save_stdout) 1382 self.debugger.reset() 1383 pdb.set_trace = self.debugger.set_trace 1384 1385 # Patch linecache.getlines, so we can see the example's source 1386 # when we're inside the debugger. 1387 self.save_linecache_getlines = linecache.getlines 1388 linecache.getlines = self.__patched_linecache_getlines 1389 1390 try: 1391 return self.__run(test, compileflags, out) 1392 finally: 1393 sys.stdout = save_stdout 1394 pdb.set_trace = save_set_trace 1395 linecache.getlines = self.save_linecache_getlines 1396 if clear_globs: 1397 test.globs.clear() 1398 1399 #///////////////////////////////////////////////////////////////// 1400 # Summarization 1401 #///////////////////////////////////////////////////////////////// 1402 def summarize(self, verbose=None): 1403 """ 1404 Print a summary of all the test cases that have been run by 1405 this DocTestRunner, and return a tuple `(f, t)`, where `f` is 1406 the total number of failed examples, and `t` is the total 1407 number of tried examples. 1408 1409 The optional `verbose` argument controls how detailed the 1410 summary is. If the verbosity is not specified, then the 1411 DocTestRunner's verbosity is used. 1412 """ 1413 if verbose is None: 1414 verbose = self._verbose 1415 notests = [] 1416 passed = [] 1417 failed = [] 1418 totalt = totalf = 0 1419 for x in self._name2ft.items(): 1420 name, (f, t) = x 1421 assert f <= t 1422 totalt += t 1423 totalf += f 1424 if t == 0: 1425 notests.append(name) 1426 elif f == 0: 1427 passed.append( (name, t) ) 1428 else: 1429 failed.append(x) 1430 if verbose: 1431 if notests: 1432 print len(notests), "items had no tests:" 1433 notests.sort() 1434 for thing in notests: 1435 print " ", thing 1436 if passed: 1437 print len(passed), "items passed all tests:" 1438 passed.sort() 1439 for thing, count in passed: 1440 print " %3d tests in %s" % (count, thing) 1441 if failed: 1442 print self.DIVIDER 1443 print len(failed), "items had failures:" 1444 failed.sort() 1445 for thing, (f, t) in failed: 1446 print " %3d of %3d in %s" % (f, t, thing) 1447 if verbose: 1448 print totalt, "tests in", len(self._name2ft), "items." 1449 print totalt - totalf, "passed and", totalf, "failed." 1450 if totalf: 1451 print "***Test Failed***", totalf, "failures." 1452 elif verbose: 1453 print "Test passed." 1454 return totalf, totalt 1455 1456 #///////////////////////////////////////////////////////////////// 1457 # Backward compatibility cruft to maintain doctest.master. 1458 #///////////////////////////////////////////////////////////////// 1459 def merge(self, other): 1460 d = self._name2ft 1461 for name, (f, t) in other._name2ft.items(): 1462 if name in d: 1463 print "*** DocTestRunner.merge: '" + name + "' in both" \ 1464 " testers; summing outcomes." 1465 f2, t2 = d[name] 1466 f = f + f2 1467 t = t + t2 1468 d[name] = f, t 1469 1470class OutputChecker: 1471 """ 1472 A class used to check the whether the actual output from a doctest 1473 example matches the expected output. `OutputChecker` defines two 1474 methods: `check_output`, which compares a given pair of outputs, 1475 and returns true if they match; and `output_difference`, which 1476 returns a string describing the differences between two outputs. 1477 """ 1478 def check_output(self, want, got, optionflags): 1479 """ 1480 Return True iff the actual output from an example (`got`) 1481 matches the expected output (`want`). These strings are 1482 always considered to match if they are identical; but 1483 depending on what option flags the test runner is using, 1484 several non-exact match types are also possible. See the 1485 documentation for `TestRunner` for more information about 1486 option flags. 1487 """ 1488 # Handle the common case first, for efficiency: 1489 # if they're string-identical, always return true. 1490 if got == want: 1491 return True 1492 1493 # The values True and False replaced 1 and 0 as the return 1494 # value for boolean comparisons in Python 2.3. 1495 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): 1496 if (got,want) == ("True\n", "1\n"): 1497 return True 1498 if (got,want) == ("False\n", "0\n"): 1499 return True 1500 1501 # <BLANKLINE> can be used as a special sequence to signify a 1502 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. 1503 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1504 # Replace <BLANKLINE> in want with a blank line. 1505 want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), 1506 '', want) 1507 # If a line in got contains only spaces, then remove the 1508 # spaces. 1509 got = re.sub('(?m)^\s*?$', '', got) 1510 if got == want: 1511 return True 1512 1513 # This flag causes doctest to ignore any differences in the 1514 # contents of whitespace strings. Note that this can be used 1515 # in conjunction with the ELLIPSIS flag. 1516 if optionflags & NORMALIZE_WHITESPACE: 1517 got = ' '.join(got.split()) 1518 want = ' '.join(want.split()) 1519 if got == want: 1520 return True 1521 1522 # The ELLIPSIS flag says to let the sequence "..." in `want` 1523 # match any substring in `got`. 1524 if optionflags & ELLIPSIS: 1525 if _ellipsis_match(want, got): 1526 return True 1527 1528 # We didn't find any match; return false. 1529 return False 1530 1531 # Should we do a fancy diff? 1532 def _do_a_fancy_diff(self, want, got, optionflags): 1533 # Not unless they asked for a fancy diff. 1534 if not optionflags & (REPORT_UDIFF | 1535 REPORT_CDIFF | 1536 REPORT_NDIFF): 1537 return False 1538 1539 # If expected output uses ellipsis, a meaningful fancy diff is 1540 # too hard ... or maybe not. In two real-life failures Tim saw, 1541 # a diff was a major help anyway, so this is commented out. 1542 # [todo] _ellipsis_match() knows which pieces do and don't match, 1543 # and could be the basis for a kick-ass diff in this case. 1544 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: 1545 ## return False 1546 1547 # ndiff does intraline difference marking, so can be useful even 1548 # for 1-line differences. 1549 if optionflags & REPORT_NDIFF: 1550 return True 1551 1552 # The other diff types need at least a few lines to be helpful. 1553 return want.count('\n') > 2 and got.count('\n') > 2 1554 1555 def output_difference(self, example, got, optionflags): 1556 """ 1557 Return a string describing the differences between the 1558 expected output for a given example (`example`) and the actual 1559 output (`got`). `optionflags` is the set of option flags used 1560 to compare `want` and `got`. 1561 """ 1562 want = example.want 1563 # If <BLANKLINE>s are being used, then replace blank lines 1564 # with <BLANKLINE> in the actual output string. 1565 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1566 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) 1567 1568 # Check if we should use diff. 1569 if self._do_a_fancy_diff(want, got, optionflags): 1570 # Split want & got into lines. 1571 want_lines = want.splitlines(True) # True == keep line ends 1572 got_lines = got.splitlines(True) 1573 # Use difflib to find their differences. 1574 if optionflags & REPORT_UDIFF: 1575 diff = difflib.unified_diff(want_lines, got_lines, n=2) 1576 diff = list(diff)[2:] # strip the diff header 1577 kind = 'unified diff with -expected +actual' 1578 elif optionflags & REPORT_CDIFF: 1579 diff = difflib.context_diff(want_lines, got_lines, n=2) 1580 diff = list(diff)[2:] # strip the diff header 1581 kind = 'context diff with expected followed by actual' 1582 elif optionflags & REPORT_NDIFF: 1583 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) 1584 diff = list(engine.compare(want_lines, got_lines)) 1585 kind = 'ndiff with -expected +actual' 1586 else: 1587 assert 0, 'Bad diff option' 1588 # Remove trailing whitespace on diff output. 1589 diff = [line.rstrip() + '\n' for line in diff] 1590 return 'Differences (%s):\n' % kind + _indent(''.join(diff)) 1591 1592 # If we're not using diff, then simply list the expected 1593 # output followed by the actual output. 1594 if want and got: 1595 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) 1596 elif want: 1597 return 'Expected:\n%sGot nothing\n' % _indent(want) 1598 elif got: 1599 return 'Expected nothing\nGot:\n%s' % _indent(got) 1600 else: 1601 return 'Expected nothing\nGot nothing\n' 1602 1603class DocTestFailure(Exception): 1604 """A DocTest example has failed in debugging mode. 1605 1606 The exception instance has variables: 1607 1608 - test: the DocTest object being run 1609 1610 - excample: the Example object that failed 1611 1612 - got: the actual output 1613 """ 1614 def __init__(self, test, example, got): 1615 self.test = test 1616 self.example = example 1617 self.got = got 1618 1619 def __str__(self): 1620 return str(self.test) 1621 1622class UnexpectedException(Exception): 1623 """A DocTest example has encountered an unexpected exception 1624 1625 The exception instance has variables: 1626 1627 - test: the DocTest object being run 1628 1629 - excample: the Example object that failed 1630 1631 - exc_info: the exception info 1632 """ 1633 def __init__(self, test, example, exc_info): 1634 self.test = test 1635 self.example = example 1636 self.exc_info = exc_info 1637 1638 def __str__(self): 1639 return str(self.test) 1640 1641class DebugRunner(DocTestRunner): 1642 r"""Run doc tests but raise an exception as soon as there is a failure. 1643 1644 If an unexpected exception occurs, an UnexpectedException is raised. 1645 It contains the test, the example, and the original exception: 1646 1647 >>> runner = DebugRunner(verbose=False) 1648 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 1649 ... {}, 'foo', 'foo.py', 0) 1650 >>> try: 1651 ... runner.run(test) 1652 ... except UnexpectedException, failure: 1653 ... pass 1654 1655 >>> failure.test is test 1656 True 1657 1658 >>> failure.example.want 1659 '42\n' 1660 1661 >>> exc_info = failure.exc_info 1662 >>> raise exc_info[0], exc_info[1], exc_info[2] 1663 Traceback (most recent call last): 1664 ... 1665 KeyError 1666 1667 We wrap the original exception to give the calling application 1668 access to the test and example information. 1669 1670 If the output doesn't match, then a DocTestFailure is raised: 1671 1672 >>> test = DocTestParser().get_doctest(''' 1673 ... >>> x = 1 1674 ... >>> x 1675 ... 2 1676 ... ''', {}, 'foo', 'foo.py', 0) 1677 1678 >>> try: 1679 ... runner.run(test) 1680 ... except DocTestFailure, failure: 1681 ... pass 1682 1683 DocTestFailure objects provide access to the test: 1684 1685 >>> failure.test is test 1686 True 1687 1688 As well as to the example: 1689 1690 >>> failure.example.want 1691 '2\n' 1692 1693 and the actual output: 1694 1695 >>> failure.got 1696 '1\n' 1697 1698 If a failure or error occurs, the globals are left intact: 1699 1700 >>> del test.globs['__builtins__'] 1701 >>> test.globs 1702 {'x': 1} 1703 1704 >>> test = DocTestParser().get_doctest(''' 1705 ... >>> x = 2 1706 ... >>> raise KeyError 1707 ... ''', {}, 'foo', 'foo.py', 0) 1708 1709 >>> runner.run(test) 1710 Traceback (most recent call last): 1711 ... 1712 UnexpectedException: <DocTest foo from foo.py:0 (2 examples)> 1713 1714 >>> del test.globs['__builtins__'] 1715 >>> test.globs 1716 {'x': 2} 1717 1718 But the globals are cleared if there is no error: 1719 1720 >>> test = DocTestParser().get_doctest(''' 1721 ... >>> x = 2 1722 ... ''', {}, 'foo', 'foo.py', 0) 1723 1724 >>> runner.run(test) 1725 (0, 1) 1726 1727 >>> test.globs 1728 {} 1729 1730 """ 1731 1732 def run(self, test, compileflags=None, out=None, clear_globs=True): 1733 r = DocTestRunner.run(self, test, compileflags, out, False) 1734 if clear_globs: 1735 test.globs.clear() 1736 return r 1737 1738 def report_unexpected_exception(self, out, test, example, exc_info): 1739 raise UnexpectedException(test, example, exc_info) 1740 1741 def report_failure(self, out, test, example, got): 1742 raise DocTestFailure(test, example, got) 1743 1744###################################################################### 1745## 6. Test Functions 1746###################################################################### 1747# These should be backwards compatible. 1748 1749# For backward compatibility, a global instance of a DocTestRunner 1750# class, updated by testmod. 1751master = None 1752 1753def testmod(m=None, name=None, globs=None, verbose=None, isprivate=None, 1754 report=True, optionflags=0, extraglobs=None, 1755 raise_on_error=False, exclude_empty=False): 1756 """m=None, name=None, globs=None, verbose=None, isprivate=None, 1757 report=True, optionflags=0, extraglobs=None, raise_on_error=False, 1758 exclude_empty=False 1759 1760 Test examples in docstrings in functions and classes reachable 1761 from module m (or the current module if m is not supplied), starting 1762 with m.__doc__. Unless isprivate is specified, private names 1763 are not skipped. 1764 1765 Also test examples reachable from dict m.__test__ if it exists and is 1766 not None. m.__test__ maps names to functions, classes and strings; 1767 function and class docstrings are tested even if the name is private; 1768 strings are tested directly, as if they were docstrings. 1769 1770 Return (#failures, #tests). 1771 1772 See doctest.__doc__ for an overview. 1773 1774 Optional keyword arg "name" gives the name of the module; by default 1775 use m.__name__. 1776 1777 Optional keyword arg "globs" gives a dict to be used as the globals 1778 when executing examples; by default, use m.__dict__. A copy of this 1779 dict is actually used for each docstring, so that each docstring's 1780 examples start with a clean slate. 1781 1782 Optional keyword arg "extraglobs" gives a dictionary that should be 1783 merged into the globals that are used to execute examples. By 1784 default, no extra globals are used. This is new in 2.4. 1785 1786 Optional keyword arg "verbose" prints lots of stuff if true, prints 1787 only failures if false; by default, it's true iff "-v" is in sys.argv. 1788 1789 Optional keyword arg "report" prints a summary at the end when true, 1790 else prints nothing at the end. In verbose mode, the summary is 1791 detailed, else very brief (in fact, empty if all tests passed). 1792 1793 Optional keyword arg "optionflags" or's together module constants, 1794 and defaults to 0. This is new in 2.3. Possible values (see the 1795 docs for details): 1796 1797 DONT_ACCEPT_TRUE_FOR_1 1798 DONT_ACCEPT_BLANKLINE 1799 NORMALIZE_WHITESPACE 1800 ELLIPSIS 1801 SKIP 1802 IGNORE_EXCEPTION_DETAIL 1803 REPORT_UDIFF 1804 REPORT_CDIFF 1805 REPORT_NDIFF 1806 REPORT_ONLY_FIRST_FAILURE 1807 1808 Optional keyword arg "raise_on_error" raises an exception on the 1809 first unexpected exception or failure. This allows failures to be 1810 post-mortem debugged. 1811 1812 Deprecated in Python 2.4: 1813 Optional keyword arg "isprivate" specifies a function used to 1814 determine whether a name is private. The default function is 1815 treat all functions as public. Optionally, "isprivate" can be 1816 set to doctest.is_private to skip over functions marked as private 1817 using the underscore naming convention; see its docs for details. 1818 1819 Advanced tomfoolery: testmod runs methods of a local instance of 1820 class doctest.Tester, then merges the results into (or creates) 1821 global Tester instance doctest.master. Methods of doctest.master 1822 can be called directly too, if you want to do something unusual. 1823 Passing report=0 to testmod is especially useful then, to delay 1824 displaying a summary. Invoke doctest.master.summarize(verbose) 1825 when you're done fiddling. 1826 """ 1827 global master 1828 1829 if isprivate is not None: 1830 warnings.warn("the isprivate argument is deprecated; " 1831 "examine DocTestFinder.find() lists instead", 1832 DeprecationWarning) 1833 1834 # If no module was given, then use __main__. 1835 if m is None: 1836 # DWA - m will still be None if this wasn't invoked from the command 1837 # line, in which case the following TypeError is about as good an error 1838 # as we should expect 1839 m = sys.modules.get('__main__') 1840 1841 # Check that we were actually given a module. 1842 if not inspect.ismodule(m): 1843 raise TypeError("testmod: module required; %r" % (m,)) 1844 1845 # If no name was given, then use the module's name. 1846 if name is None: 1847 name = m.__name__ 1848 1849 # Find, parse, and run all tests in the given module. 1850 finder = DocTestFinder(_namefilter=isprivate, exclude_empty=exclude_empty) 1851 1852 if raise_on_error: 1853 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 1854 else: 1855 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 1856 1857 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): 1858 runner.run(test) 1859 1860 if report: 1861 runner.summarize() 1862 1863 if master is None: 1864 master = runner 1865 else: 1866 master.merge(runner) 1867 1868 return runner.failures, runner.tries 1869 1870def testfile(filename, module_relative=True, name=None, package=None, 1871 globs=None, verbose=None, report=True, optionflags=0, 1872 extraglobs=None, raise_on_error=False, parser=DocTestParser()): 1873 """ 1874 Test examples in the given file. Return (#failures, #tests). 1875 1876 Optional keyword arg "module_relative" specifies how filenames 1877 should be interpreted: 1878 1879 - If "module_relative" is True (the default), then "filename" 1880 specifies a module-relative path. By default, this path is 1881 relative to the calling module's directory; but if the 1882 "package" argument is specified, then it is relative to that 1883 package. To ensure os-independence, "filename" should use 1884 "/" characters to separate path segments, and should not 1885 be an absolute path (i.e., it may not begin with "/"). 1886 1887 - If "module_relative" is False, then "filename" specifies an 1888 os-specific path. The path may be absolute or relative (to 1889 the current working directory). 1890 1891 Optional keyword arg "name" gives the name of the test; by default 1892 use the file's basename. 1893 1894 Optional keyword argument "package" is a Python package or the 1895 name of a Python package whose directory should be used as the 1896 base directory for a module relative filename. If no package is 1897 specified, then the calling module's directory is used as the base 1898 directory for module relative filenames. It is an error to 1899 specify "package" if "module_relative" is False. 1900 1901 Optional keyword arg "globs" gives a dict to be used as the globals 1902 when executing examples; by default, use {}. A copy of this dict 1903 is actually used for each docstring, so that each docstring's 1904 examples start with a clean slate. 1905 1906 Optional keyword arg "extraglobs" gives a dictionary that should be 1907 merged into the globals that are used to execute examples. By 1908 default, no extra globals are used. 1909 1910 Optional keyword arg "verbose" prints lots of stuff if true, prints 1911 only failures if false; by default, it's true iff "-v" is in sys.argv. 1912 1913 Optional keyword arg "report" prints a summary at the end when true, 1914 else prints nothing at the end. In verbose mode, the summary is 1915 detailed, else very brief (in fact, empty if all tests passed). 1916 1917 Optional keyword arg "optionflags" or's together module constants, 1918 and defaults to 0. Possible values (see the docs for details): 1919 1920 DONT_ACCEPT_TRUE_FOR_1 1921 DONT_ACCEPT_BLANKLINE 1922 NORMALIZE_WHITESPACE 1923 ELLIPSIS 1924 SKIP 1925 IGNORE_EXCEPTION_DETAIL 1926 REPORT_UDIFF 1927 REPORT_CDIFF 1928 REPORT_NDIFF 1929 REPORT_ONLY_FIRST_FAILURE 1930 1931 Optional keyword arg "raise_on_error" raises an exception on the 1932 first unexpected exception or failure. This allows failures to be 1933 post-mortem debugged. 1934 1935 Optional keyword arg "parser" specifies a DocTestParser (or 1936 subclass) that should be used to extract tests from the files. 1937 1938 Advanced tomfoolery: testmod runs methods of a local instance of 1939 class doctest.Tester, then merges the results into (or creates) 1940 global Tester instance doctest.master. Methods of doctest.master 1941 can be called directly too, if you want to do something unusual. 1942 Passing report=0 to testmod is especially useful then, to delay 1943 displaying a summary. Invoke doctest.master.summarize(verbose) 1944 when you're done fiddling. 1945 """ 1946 global master 1947 1948 if package and not module_relative: 1949 raise ValueError("Package may only be specified for module-" 1950 "relative paths.") 1951 1952 # Relativize the path 1953 text, filename = _load_testfile(filename, package, module_relative) 1954 1955 # If no name was given, then use the file's name. 1956 if name is None: 1957 name = os.path.basename(filename) 1958 1959 # Assemble the globals. 1960 if globs is None: 1961 globs = {} 1962 else: 1963 globs = globs.copy() 1964 if extraglobs is not None: 1965 globs.update(extraglobs) 1966 1967 if raise_on_error: 1968 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 1969 else: 1970 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 1971 1972 # Read the file, convert it to a test, and run it. 1973 test = parser.get_doctest(text, globs, name, filename, 0) 1974 runner.run(test) 1975 1976 if report: 1977 runner.summarize() 1978 1979 if master is None: 1980 master = runner 1981 else: 1982 master.merge(runner) 1983 1984 return runner.failures, runner.tries 1985 1986def run_docstring_examples(f, globs, verbose=False, name="NoName", 1987 compileflags=None, optionflags=0): 1988 """ 1989 Test examples in the given object's docstring (`f`), using `globs` 1990 as globals. Optional argument `name` is used in failure messages. 1991 If the optional argument `verbose` is true, then generate output 1992 even if there are no failures. 1993 1994 `compileflags` gives the set of flags that should be used by the 1995 Python compiler when running the examples. If not specified, then 1996 it will default to the set of future-import flags that apply to 1997 `globs`. 1998 1999 Optional keyword arg `optionflags` specifies options for the 2000 testing and output. See the documentation for `testmod` for more 2001 information. 2002 """ 2003 # Find, parse, and run all tests in the given module. 2004 finder = DocTestFinder(verbose=verbose, recurse=False) 2005 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2006 for test in finder.find(f, name, globs=globs): 2007 runner.run(test, compileflags=compileflags) 2008 2009###################################################################### 2010## 7. Tester 2011###################################################################### 2012# This is provided only for backwards compatibility. It's not 2013# actually used in any way. 2014 2015class Tester: 2016 def __init__(self, mod=None, globs=None, verbose=None, 2017 isprivate=None, optionflags=0): 2018 2019 warnings.warn("class Tester is deprecated; " 2020 "use class doctest.DocTestRunner instead", 2021 DeprecationWarning, stacklevel=2) 2022 if mod is None and globs is None: 2023 raise TypeError("Tester.__init__: must specify mod or globs") 2024 if mod is not None and not inspect.ismodule(mod): 2025 raise TypeError("Tester.__init__: mod must be a module; %r" % 2026 (mod,)) 2027 if globs is None: 2028 globs = mod.__dict__ 2029 self.globs = globs 2030 2031 self.verbose = verbose 2032 self.isprivate = isprivate 2033 self.optionflags = optionflags 2034 self.testfinder = DocTestFinder(_namefilter=isprivate) 2035 self.testrunner = DocTestRunner(verbose=verbose, 2036 optionflags=optionflags) 2037 2038 def runstring(self, s, name): 2039 test = DocTestParser().get_doctest(s, self.globs, name, None, None) 2040 if self.verbose: 2041 print "Running string", name 2042 (f,t) = self.testrunner.run(test) 2043 if self.verbose: 2044 print f, "of", t, "examples failed in string", name 2045 return (f,t) 2046 2047 def rundoc(self, object, name=None, module=None): 2048 f = t = 0 2049 tests = self.testfinder.find(object, name, module=module, 2050 globs=self.globs) 2051 for test in tests: 2052 (f2, t2) = self.testrunner.run(test) 2053 (f,t) = (f+f2, t+t2) 2054 return (f,t) 2055 2056 def rundict(self, d, name, module=None): 2057 import new 2058 m = new.module(name) 2059 m.__dict__.update(d) 2060 if module is None: 2061 module = False 2062 return self.rundoc(m, name, module) 2063 2064 def run__test__(self, d, name): 2065 import new 2066 m = new.module(name) 2067 m.__test__ = d 2068 return self.rundoc(m, name) 2069 2070 def summarize(self, verbose=None): 2071 return self.testrunner.summarize(verbose) 2072 2073 def merge(self, other): 2074 self.testrunner.merge(other.testrunner) 2075 2076###################################################################### 2077## 8. Unittest Support 2078###################################################################### 2079 2080_unittest_reportflags = 0 2081 2082def set_unittest_reportflags(flags): 2083 """Sets the unittest option flags. 2084 2085 The old flag is returned so that a runner could restore the old 2086 value if it wished to: 2087 2088 >>> import doctest 2089 >>> old = doctest._unittest_reportflags 2090 >>> doctest.set_unittest_reportflags(REPORT_NDIFF | 2091 ... REPORT_ONLY_FIRST_FAILURE) == old 2092 True 2093 2094 >>> doctest._unittest_reportflags == (REPORT_NDIFF | 2095 ... REPORT_ONLY_FIRST_FAILURE) 2096 True 2097 2098 Only reporting flags can be set: 2099 2100 >>> doctest.set_unittest_reportflags(ELLIPSIS) 2101 Traceback (most recent call last): 2102 ... 2103 ValueError: ('Only reporting flags allowed', 8) 2104 2105 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF | 2106 ... REPORT_ONLY_FIRST_FAILURE) 2107 True 2108 """ 2109 global _unittest_reportflags 2110 2111 if (flags & REPORTING_FLAGS) != flags: 2112 raise ValueError("Only reporting flags allowed", flags) 2113 old = _unittest_reportflags 2114 _unittest_reportflags = flags 2115 return old 2116 2117 2118class DocTestCase(unittest.TestCase): 2119 2120 def __init__(self, test, optionflags=0, setUp=None, tearDown=None, 2121 checker=None): 2122 2123 unittest.TestCase.__init__(self) 2124 self._dt_optionflags = optionflags 2125 self._dt_checker = checker 2126 self._dt_test = test 2127 self._dt_setUp = setUp 2128 self._dt_tearDown = tearDown 2129 2130 def setUp(self): 2131 test = self._dt_test 2132 2133 if self._dt_setUp is not None: 2134 self._dt_setUp(test) 2135 2136 def tearDown(self): 2137 test = self._dt_test 2138 2139 if self._dt_tearDown is not None: 2140 self._dt_tearDown(test) 2141 2142 test.globs.clear() 2143 2144 def runTest(self): 2145 test = self._dt_test 2146 old = sys.stdout 2147 new = StringIO() 2148 optionflags = self._dt_optionflags 2149 2150 if not (optionflags & REPORTING_FLAGS): 2151 # The option flags don't include any reporting flags, 2152 # so add the default reporting flags 2153 optionflags |= _unittest_reportflags 2154 2155 runner = DocTestRunner(optionflags=optionflags, 2156 checker=self._dt_checker, verbose=False) 2157 2158 try: 2159 runner.DIVIDER = "-"*70 2160 failures, tries = runner.run( 2161 test, out=new.write, clear_globs=False) 2162 finally: 2163 sys.stdout = old 2164 2165 if failures: 2166 raise self.failureException(self.format_failure(new.getvalue())) 2167 2168 def format_failure(self, err): 2169 test = self._dt_test 2170 if test.lineno is None: 2171 lineno = 'unknown line number' 2172 else: 2173 lineno = '%s' % test.lineno 2174 lname = '.'.join(test.name.split('.')[-1:]) 2175 return ('Failed doctest test for %s\n' 2176 ' File "%s", line %s, in %s\n\n%s' 2177 % (test.name, test.filename, lineno, lname, err) 2178 ) 2179 2180 def debug(self): 2181 r"""Run the test case without results and without catching exceptions 2182 2183 The unit test framework includes a debug method on test cases 2184 and test suites to support post-mortem debugging. The test code 2185 is run in such a way that errors are not caught. This way a 2186 caller can catch the errors and initiate post-mortem debugging. 2187 2188 The DocTestCase provides a debug method that raises 2189 UnexpectedException errors if there is an unexepcted 2190 exception: 2191 2192 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 2193 ... {}, 'foo', 'foo.py', 0) 2194 >>> case = DocTestCase(test) 2195 >>> try: 2196 ... case.debug() 2197 ... except UnexpectedException, failure: 2198 ... pass 2199 2200 The UnexpectedException contains the test, the example, and 2201 the original exception: 2202 2203 >>> failure.test is test 2204 True 2205 2206 >>> failure.example.want 2207 '42\n' 2208 2209 >>> exc_info = failure.exc_info 2210 >>> raise exc_info[0], exc_info[1], exc_info[2] 2211 Traceback (most recent call last): 2212 ... 2213 KeyError 2214 2215 If the output doesn't match, then a DocTestFailure is raised: 2216 2217 >>> test = DocTestParser().get_doctest(''' 2218 ... >>> x = 1 2219 ... >>> x 2220 ... 2 2221 ... ''', {}, 'foo', 'foo.py', 0) 2222 >>> case = DocTestCase(test) 2223 2224 >>> try: 2225 ... case.debug() 2226 ... except DocTestFailure, failure: 2227 ... pass 2228 2229 DocTestFailure objects provide access to the test: 2230 2231 >>> failure.test is test 2232 True 2233 2234 As well as to the example: 2235 2236 >>> failure.example.want 2237 '2\n' 2238 2239 and the actual output: 2240 2241 >>> failure.got 2242 '1\n' 2243 2244 """ 2245 2246 self.setUp() 2247 runner = DebugRunner(optionflags=self._dt_optionflags, 2248 checker=self._dt_checker, verbose=False) 2249 runner.run(self._dt_test) 2250 self.tearDown() 2251 2252 def id(self): 2253 return self._dt_test.name 2254 2255 def __repr__(self): 2256 name = self._dt_test.name.split('.') 2257 return "%s (%s)" % (name[-1], '.'.join(name[:-1])) 2258 2259 __str__ = __repr__ 2260 2261 def shortDescription(self): 2262 return "Doctest: " + self._dt_test.name 2263 2264def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, 2265 **options): 2266 """ 2267 Convert doctest tests for a module to a unittest test suite. 2268 2269 This converts each documentation string in a module that 2270 contains doctest tests to a unittest test case. If any of the 2271 tests in a doc string fail, then the test case fails. An exception 2272 is raised showing the name of the file containing the test and a 2273 (sometimes approximate) line number. 2274 2275 The `module` argument provides the module to be tested. The argument 2276 can be either a module or a module name. 2277 2278 If no argument is given, the calling module is used. 2279 2280 A number of options may be provided as keyword arguments: 2281 2282 setUp 2283 A set-up function. This is called before running the 2284 tests in each file. The setUp function will be passed a DocTest 2285 object. The setUp function can access the test globals as the 2286 globs attribute of the test passed. 2287 2288 tearDown 2289 A tear-down function. This is called after running the 2290 tests in each file. The tearDown function will be passed a DocTest 2291 object. The tearDown function can access the test globals as the 2292 globs attribute of the test passed. 2293 2294 globs 2295 A dictionary containing initial global variables for the tests. 2296 2297 optionflags 2298 A set of doctest option flags expressed as an integer. 2299 """ 2300 2301 if test_finder is None: 2302 test_finder = DocTestFinder() 2303 2304 module = _normalize_module(module) 2305 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) 2306 if globs is None: 2307 globs = module.__dict__ 2308 if not tests: 2309 # Why do we want to do this? Because it reveals a bug that might 2310 # otherwise be hidden. 2311 raise ValueError(module, "has no tests") 2312 2313 tests.sort() 2314 suite = unittest.TestSuite() 2315 for test in tests: 2316 if len(test.examples) == 0: 2317 continue 2318 if not test.filename: 2319 filename = module.__file__ 2320 if filename[-4:] in (".pyc", ".pyo"): 2321 filename = filename[:-1] 2322 test.filename = filename 2323 suite.addTest(DocTestCase(test, **options)) 2324 2325 return suite 2326 2327class DocFileCase(DocTestCase): 2328 2329 def id(self): 2330 return '_'.join(self._dt_test.name.split('.')) 2331 2332 def __repr__(self): 2333 return self._dt_test.filename 2334 __str__ = __repr__ 2335 2336 def format_failure(self, err): 2337 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' 2338 % (self._dt_test.name, self._dt_test.filename, err) 2339 ) 2340 2341def DocFileTest(path, module_relative=True, package=None, 2342 globs=None, parser=DocTestParser(), **options): 2343 if globs is None: 2344 globs = {} 2345 else: 2346 globs = globs.copy() 2347 2348 if package and not module_relative: 2349 raise ValueError("Package may only be specified for module-" 2350 "relative paths.") 2351 2352 # Relativize the path. 2353 doc, path = _load_testfile(path, package, module_relative) 2354 2355 if "__file__" not in globs: 2356 globs["__file__"] = path 2357 2358 # Find the file and read it. 2359 name = os.path.basename(path) 2360 2361 # Convert it to a test, and wrap it in a DocFileCase. 2362 test = parser.get_doctest(doc, globs, name, path, 0) 2363 return DocFileCase(test, **options) 2364 2365def DocFileSuite(*paths, **kw): 2366 """A unittest suite for one or more doctest files. 2367 2368 The path to each doctest file is given as a string; the 2369 interpretation of that string depends on the keyword argument 2370 "module_relative". 2371 2372 A number of options may be provided as keyword arguments: 2373 2374 module_relative 2375 If "module_relative" is True, then the given file paths are 2376 interpreted as os-independent module-relative paths. By 2377 default, these paths are relative to the calling module's 2378 directory; but if the "package" argument is specified, then 2379 they are relative to that package. To ensure os-independence, 2380 "filename" should use "/" characters to separate path 2381 segments, and may not be an absolute path (i.e., it may not 2382 begin with "/"). 2383 2384 If "module_relative" is False, then the given file paths are 2385 interpreted as os-specific paths. These paths may be absolute 2386 or relative (to the current working directory). 2387 2388 package 2389 A Python package or the name of a Python package whose directory 2390 should be used as the base directory for module relative paths. 2391 If "package" is not specified, then the calling module's 2392 directory is used as the base directory for module relative 2393 filenames. It is an error to specify "package" if 2394 "module_relative" is False. 2395 2396 setUp 2397 A set-up function. This is called before running the 2398 tests in each file. The setUp function will be passed a DocTest 2399 object. The setUp function can access the test globals as the 2400 globs attribute of the test passed. 2401 2402 tearDown 2403 A tear-down function. This is called after running the 2404 tests in each file. The tearDown function will be passed a DocTest 2405 object. The tearDown function can access the test globals as the 2406 globs attribute of the test passed. 2407 2408 globs 2409 A dictionary containing initial global variables for the tests. 2410 2411 optionflags 2412 A set of doctest option flags expressed as an integer. 2413 2414 parser 2415 A DocTestParser (or subclass) that should be used to extract 2416 tests from the files. 2417 """ 2418 suite = unittest.TestSuite() 2419 2420 # We do this here so that _normalize_module is called at the right 2421 # level. If it were called in DocFileTest, then this function 2422 # would be the caller and we might guess the package incorrectly. 2423 if kw.get('module_relative', True): 2424 kw['package'] = _normalize_module(kw.get('package')) 2425 2426 for path in paths: 2427 suite.addTest(DocFileTest(path, **kw)) 2428 2429 return suite 2430 2431###################################################################### 2432## 9. Debugging Support 2433###################################################################### 2434 2435def script_from_examples(s): 2436 r"""Extract script from text with examples. 2437 2438 Converts text with examples to a Python script. Example input is 2439 converted to regular code. Example output and all other words 2440 are converted to comments: 2441 2442 >>> text = ''' 2443 ... Here are examples of simple math. 2444 ... 2445 ... Python has super accurate integer addition 2446 ... 2447 ... >>> 2 + 2 2448 ... 5 2449 ... 2450 ... And very friendly error messages: 2451 ... 2452 ... >>> 1/0 2453 ... To Infinity 2454 ... And 2455 ... Beyond 2456 ... 2457 ... You can use logic if you want: 2458 ... 2459 ... >>> if 0: 2460 ... ... blah 2461 ... ... blah 2462 ... ... 2463 ... 2464 ... Ho hum 2465 ... ''' 2466 2467 >>> print script_from_examples(text) 2468 # Here are examples of simple math. 2469 # 2470 # Python has super accurate integer addition 2471 # 2472 2 + 2 2473 # Expected: 2474 ## 5 2475 # 2476 # And very friendly error messages: 2477 # 2478 1/0 2479 # Expected: 2480 ## To Infinity 2481 ## And 2482 ## Beyond 2483 # 2484 # You can use logic if you want: 2485 # 2486 if 0: 2487 blah 2488 blah 2489 # 2490 # Ho hum 2491 <BLANKLINE> 2492 """ 2493 output = [] 2494 for piece in DocTestParser().parse(s): 2495 if isinstance(piece, Example): 2496 # Add the example's source code (strip trailing NL) 2497 output.append(piece.source[:-1]) 2498 # Add the expected output: 2499 want = piece.want 2500 if want: 2501 output.append('# Expected:') 2502 output += ['## '+l for l in want.split('\n')[:-1]] 2503 else: 2504 # Add non-example text. 2505 output += [_comment_line(l) 2506 for l in piece.split('\n')[:-1]] 2507 2508 # Trim junk on both ends. 2509 while output and output[-1] == '#': 2510 output.pop() 2511 while output and output[0] == '#': 2512 output.pop(0) 2513 # Combine the output, and return it. 2514 # Add a courtesy newline to prevent exec from choking (see bug #1172785) 2515 return '\n'.join(output) + '\n' 2516 2517def testsource(module, name): 2518 """Extract the test sources from a doctest docstring as a script. 2519 2520 Provide the module (or dotted name of the module) containing the 2521 test to be debugged and the name (within the module) of the object 2522 with the doc string with tests to be debugged. 2523 """ 2524 module = _normalize_module(module) 2525 tests = DocTestFinder().find(module) 2526 test = [t for t in tests if t.name == name] 2527 if not test: 2528 raise ValueError(name, "not found in tests") 2529 test = test[0] 2530 testsrc = script_from_examples(test.docstring) 2531 return testsrc 2532 2533def debug_src(src, pm=False, globs=None): 2534 """Debug a single doctest docstring, in argument `src`'""" 2535 testsrc = script_from_examples(src) 2536 debug_script(testsrc, pm, globs) 2537 2538def debug_script(src, pm=False, globs=None): 2539 "Debug a test script. `src` is the script, as a string." 2540 import pdb 2541 2542 # Note that tempfile.NameTemporaryFile() cannot be used. As the 2543 # docs say, a file so created cannot be opened by name a second time 2544 # on modern Windows boxes, and execfile() needs to open it. 2545 srcfilename = tempfile.mktemp(".py", "doctestdebug") 2546 f = open(srcfilename, 'w') 2547 f.write(src) 2548 f.close() 2549 2550 try: 2551 if globs: 2552 globs = globs.copy() 2553 else: 2554 globs = {} 2555 2556 if pm: 2557 try: 2558 execfile(srcfilename, globs, globs) 2559 except: 2560 print sys.exc_info()[1] 2561 pdb.post_mortem(sys.exc_info()[2]) 2562 else: 2563 # Note that %r is vital here. '%s' instead can, e.g., cause 2564 # backslashes to get treated as metacharacters on Windows. 2565 pdb.run("execfile(%r)" % srcfilename, globs, globs) 2566 2567 finally: 2568 os.remove(srcfilename) 2569 2570def debug(module, name, pm=False): 2571 """Debug a single doctest docstring. 2572 2573 Provide the module (or dotted name of the module) containing the 2574 test to be debugged and the name (within the module) of the object 2575 with the docstring with tests to be debugged. 2576 """ 2577 module = _normalize_module(module) 2578 testsrc = testsource(module, name) 2579 debug_script(testsrc, pm, module.__dict__) 2580 2581###################################################################### 2582## 10. Example Usage 2583###################################################################### 2584class _TestClass: 2585 """ 2586 A pointless class, for sanity-checking of docstring testing. 2587 2588 Methods: 2589 square() 2590 get() 2591 2592 >>> _TestClass(13).get() + _TestClass(-12).get() 2593 1 2594 >>> hex(_TestClass(13).square().get()) 2595 '0xa9' 2596 """ 2597 2598 def __init__(self, val): 2599 """val -> _TestClass object with associated value val. 2600 2601 >>> t = _TestClass(123) 2602 >>> print t.get() 2603 123 2604 """ 2605 2606 self.val = val 2607 2608 def square(self): 2609 """square() -> square TestClass's associated value 2610 2611 >>> _TestClass(13).square().get() 2612 169 2613 """ 2614 2615 self.val = self.val ** 2 2616 return self 2617 2618 def get(self): 2619 """get() -> return TestClass's associated value. 2620 2621 >>> x = _TestClass(-42) 2622 >>> print x.get() 2623 -42 2624 """ 2625 2626 return self.val 2627 2628__test__ = {"_TestClass": _TestClass, 2629 "string": r""" 2630 Example of a string object, searched as-is. 2631 >>> x = 1; y = 2 2632 >>> x + y, x * y 2633 (3, 2) 2634 """, 2635 2636 "bool-int equivalence": r""" 2637 In 2.2, boolean expressions displayed 2638 0 or 1. By default, we still accept 2639 them. This can be disabled by passing 2640 DONT_ACCEPT_TRUE_FOR_1 to the new 2641 optionflags argument. 2642 >>> 4 == 4 2643 1 2644 >>> 4 == 4 2645 True 2646 >>> 4 > 4 2647 0 2648 >>> 4 > 4 2649 False 2650 """, 2651 2652 "blank lines": r""" 2653 Blank lines can be marked with <BLANKLINE>: 2654 >>> print 'foo\n\nbar\n' 2655 foo 2656 <BLANKLINE> 2657 bar 2658 <BLANKLINE> 2659 """, 2660 2661 "ellipsis": r""" 2662 If the ellipsis flag is used, then '...' can be used to 2663 elide substrings in the desired output: 2664 >>> print range(1000) #doctest: +ELLIPSIS 2665 [0, 1, 2, ..., 999] 2666 """, 2667 2668 "whitespace normalization": r""" 2669 If the whitespace normalization flag is used, then 2670 differences in whitespace are ignored. 2671 >>> print range(30) #doctest: +NORMALIZE_WHITESPACE 2672 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 2673 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2674 27, 28, 29] 2675 """, 2676 } 2677 2678def _test(): 2679 r = unittest.TextTestRunner() 2680 r.run(DocTestSuite()) 2681 2682if __name__ == "__main__": 2683 _test() 2684