1# -*- coding: utf-8 -*- 2 3"""unittest-xml-reporting is a PyUnit-based TestRunner that can export test 4results to XML files that can be consumed by a wide range of tools, such as 5build systems, IDEs and Continuous Integration servers. 6 7This module provides the XMLTestRunner class, which is heavily based on the 8default TextTestRunner. This makes the XMLTestRunner very simple to use. 9 10The script below, adapted from the unittest documentation, shows how to use 11XMLTestRunner in a very simple way. In fact, the only difference between this 12script and the original one is the last line: 13 14import random 15import unittest 16import xmlrunner 17 18class TestSequenceFunctions(unittest.TestCase): 19 def setUp(self): 20 self.seq = range(10) 21 22 def test_shuffle(self): 23 # make sure the shuffled sequence does not lose any elements 24 random.shuffle(self.seq) 25 self.seq.sort() 26 self.assertEqual(self.seq, range(10)) 27 28 def test_choice(self): 29 element = random.choice(self.seq) 30 self.assert_(element in self.seq) 31 32 def test_sample(self): 33 self.assertRaises(ValueError, random.sample, self.seq, 20) 34 for element in random.sample(self.seq, 5): 35 self.assert_(element in self.seq) 36 37if __name__ == '__main__': 38 unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports')) 39""" 40 41import os 42import sys 43import time 44from unittest import TestResult, _TextTestResult, TextTestRunner 45from cStringIO import StringIO 46import xml.dom.minidom 47 48 49class XMLDocument(xml.dom.minidom.Document): 50 def createCDATAOrText(self, data): 51 if ']]>' in data: 52 return self.createTextNode(data) 53 return self.createCDATASection(data) 54 55 56class _TestInfo(object): 57 """This class is used to keep useful information about the execution of a 58 test method. 59 """ 60 61 # Possible test outcomes 62 (SUCCESS, FAILURE, ERROR) = range(3) 63 64 def __init__(self, test_result, test_method, outcome=SUCCESS, err=None): 65 "Create a new instance of _TestInfo." 66 self.test_result = test_result 67 self.test_method = test_method 68 self.outcome = outcome 69 self.err = err 70 self.stdout = test_result.stdout and test_result.stdout.getvalue().strip() or '' 71 self.stderr = test_result.stdout and test_result.stderr.getvalue().strip() or '' 72 73 def get_elapsed_time(self): 74 """Return the time that shows how long the test method took to 75 execute. 76 """ 77 return self.test_result.stop_time - self.test_result.start_time 78 79 def get_description(self): 80 "Return a text representation of the test method." 81 return self.test_result.getDescription(self.test_method) 82 83 def get_error_info(self): 84 """Return a text representation of an exception thrown by a test 85 method. 86 """ 87 if not self.err: 88 return '' 89 if sys.version_info < (2,4): 90 return self.test_result._exc_info_to_string(self.err) 91 else: 92 return self.test_result._exc_info_to_string( 93 self.err, self.test_method) 94 95 96class _XMLTestResult(_TextTestResult): 97 """A test result class that can express test results in a XML report. 98 99 Used by XMLTestRunner. 100 """ 101 def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, \ 102 elapsed_times=True): 103 "Create a new instance of _XMLTestResult." 104 _TextTestResult.__init__(self, stream, descriptions, verbosity) 105 self.successes = [] 106 self.callback = None 107 self.elapsed_times = elapsed_times 108 self.output_patched = False 109 110 def _prepare_callback(self, test_info, target_list, verbose_str, 111 short_str): 112 """Append a _TestInfo to the given target list and sets a callback 113 method to be called by stopTest method. 114 """ 115 target_list.append(test_info) 116 def callback(): 117 """This callback prints the test method outcome to the stream, 118 as well as the elapsed time. 119 """ 120 121 # Ignore the elapsed times for a more reliable unit testing 122 if not self.elapsed_times: 123 self.start_time = self.stop_time = 0 124 125 if self.showAll: 126 self.stream.writeln('(%.3fs) %s' % \ 127 (test_info.get_elapsed_time(), verbose_str)) 128 elif self.dots: 129 self.stream.write(short_str) 130 self.callback = callback 131 132 def _patch_standard_output(self): 133 """Replace the stdout and stderr streams with string-based streams 134 in order to capture the tests' output. 135 """ 136 if not self.output_patched: 137 (self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr) 138 self.output_patched = True 139 (sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \ 140 (StringIO(), StringIO()) 141 142 def _restore_standard_output(self): 143 "Restore the stdout and stderr streams." 144 (sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr) 145 self.output_patched = False 146 147 def startTest(self, test): 148 "Called before execute each test method." 149 self._patch_standard_output() 150 self.start_time = time.time() 151 TestResult.startTest(self, test) 152 153 if self.showAll: 154 self.stream.write(' ' + self.getDescription(test)) 155 self.stream.write(" ... ") 156 157 def stopTest(self, test): 158 "Called after execute each test method." 159 self._restore_standard_output() 160 _TextTestResult.stopTest(self, test) 161 self.stop_time = time.time() 162 163 if self.callback and callable(self.callback): 164 self.callback() 165 self.callback = None 166 167 def addSuccess(self, test): 168 "Called when a test executes successfully." 169 self._prepare_callback(_TestInfo(self, test), 170 self.successes, 'OK', '.') 171 172 def addFailure(self, test, err): 173 "Called when a test method fails." 174 self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err), 175 self.failures, 'FAIL', 'F') 176 177 def addError(self, test, err): 178 "Called when a test method raises an error." 179 self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err), 180 self.errors, 'ERROR', 'E') 181 182 def printErrorList(self, flavour, errors): 183 "Write some information about the FAIL or ERROR to the stream." 184 for test_info in errors: 185 if isinstance(test_info, tuple): 186 test_info, exc_info = test_info 187 self.stream.writeln(self.separator1) 188 self.stream.writeln('%s [%.3fs]: %s' % ( 189 flavour, test_info.get_elapsed_time(), 190 test_info.get_description())) 191 self.stream.writeln(self.separator2) 192 self.stream.writeln('%s' % test_info.get_error_info()) 193 194 def _get_info_by_testcase(self): 195 """This method organizes test results by TestCase module. This 196 information is used during the report generation, where a XML report 197 will be generated for each TestCase. 198 """ 199 tests_by_testcase = {} 200 201 for tests in (self.successes, self.failures, self.errors): 202 for test_info in tests: 203 testcase = type(test_info.test_method) 204 205 # Ignore module name if it is '__main__' 206 module = testcase.__module__ + '.' 207 if module == '__main__.': 208 module = '' 209 testcase_name = module + testcase.__name__ 210 211 if testcase_name not in tests_by_testcase: 212 tests_by_testcase[testcase_name] = [] 213 tests_by_testcase[testcase_name].append(test_info) 214 215 return tests_by_testcase 216 217 def _report_testsuite(suite_name, tests, xml_document): 218 "Appends the testsuite section to the XML document." 219 testsuite = xml_document.createElement('testsuite') 220 xml_document.appendChild(testsuite) 221 222 testsuite.setAttribute('name', str(suite_name)) 223 testsuite.setAttribute('tests', str(len(tests))) 224 225 testsuite.setAttribute('time', '%.3f' % 226 sum([e.get_elapsed_time() for e in tests])) 227 228 failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE]) 229 testsuite.setAttribute('failures', str(failures)) 230 231 errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR]) 232 testsuite.setAttribute('errors', str(errors)) 233 234 return testsuite 235 236 _report_testsuite = staticmethod(_report_testsuite) 237 238 def _report_testcase(suite_name, test_result, xml_testsuite, xml_document): 239 "Appends a testcase section to the XML document." 240 testcase = xml_document.createElement('testcase') 241 xml_testsuite.appendChild(testcase) 242 243 testcase.setAttribute('classname', str(suite_name)) 244 testcase.setAttribute('name', test_result.test_method.shortDescription() 245 or getattr(test_result.test_method, '_testMethodName', 246 str(test_result.test_method))) 247 testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time()) 248 249 if (test_result.outcome != _TestInfo.SUCCESS): 250 elem_name = ('failure', 'error')[test_result.outcome-1] 251 failure = xml_document.createElement(elem_name) 252 testcase.appendChild(failure) 253 254 failure.setAttribute('type', str(test_result.err[0].__name__)) 255 failure.setAttribute('message', str(test_result.err[1])) 256 257 error_info = test_result.get_error_info() 258 failureText = xml_document.createCDATAOrText(error_info) 259 failure.appendChild(failureText) 260 261 _report_testcase = staticmethod(_report_testcase) 262 263 def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr): 264 "Appends the system-out and system-err sections to the XML document." 265 systemout = xml_document.createElement('system-out') 266 xml_testsuite.appendChild(systemout) 267 268 systemout_text = xml_document.createCDATAOrText(stdout) 269 systemout.appendChild(systemout_text) 270 271 systemerr = xml_document.createElement('system-err') 272 xml_testsuite.appendChild(systemerr) 273 274 systemerr_text = xml_document.createCDATAOrText(stderr) 275 systemerr.appendChild(systemerr_text) 276 277 _report_output = staticmethod(_report_output) 278 279 def generate_reports(self, test_runner): 280 "Generates the XML reports to a given XMLTestRunner object." 281 all_results = self._get_info_by_testcase() 282 283 if type(test_runner.output) == str and not \ 284 os.path.exists(test_runner.output): 285 os.makedirs(test_runner.output) 286 287 for suite, tests in all_results.items(): 288 doc = XMLDocument() 289 290 # Build the XML file 291 testsuite = _XMLTestResult._report_testsuite(suite, tests, doc) 292 stdout, stderr = [], [] 293 for test in tests: 294 _XMLTestResult._report_testcase(suite, test, testsuite, doc) 295 if test.stdout: 296 stdout.extend(['*****************', test.get_description(), test.stdout]) 297 if test.stderr: 298 stderr.extend(['*****************', test.get_description(), test.stderr]) 299 _XMLTestResult._report_output(test_runner, testsuite, doc, 300 '\n'.join(stdout), '\n'.join(stderr)) 301 xml_content = doc.toprettyxml(indent='\t') 302 303 if type(test_runner.output) is str: 304 report_file = open('%s%sTEST-%s.xml' % \ 305 (test_runner.output, os.sep, suite), 'w') 306 try: 307 report_file.write(xml_content) 308 finally: 309 report_file.close() 310 else: 311 # Assume that test_runner.output is a stream 312 test_runner.output.write(xml_content) 313 314 315class XMLTestRunner(TextTestRunner): 316 """A test runner class that outputs the results in JUnit like XML files. 317 """ 318 def __init__(self, output='.', stream=sys.stderr, descriptions=True, \ 319 verbose=False, elapsed_times=True): 320 "Create a new instance of XMLTestRunner." 321 verbosity = (1, 2)[verbose] 322 TextTestRunner.__init__(self, stream, descriptions, verbosity) 323 self.output = output 324 self.elapsed_times = elapsed_times 325 326 def _make_result(self): 327 """Create the TestResult object which will be used to store 328 information about the executed tests. 329 """ 330 return _XMLTestResult(self.stream, self.descriptions, \ 331 self.verbosity, self.elapsed_times) 332 333 def run(self, test): 334 "Run the given test case or test suite." 335 # Prepare the test execution 336 result = self._make_result() 337 338 # Print a nice header 339 self.stream.writeln() 340 self.stream.writeln('Running tests...') 341 self.stream.writeln(result.separator2) 342 343 # Execute tests 344 start_time = time.time() 345 test(result) 346 stop_time = time.time() 347 time_taken = stop_time - start_time 348 349 # Print results 350 result.printErrors() 351 self.stream.writeln(result.separator2) 352 run = result.testsRun 353 self.stream.writeln("Ran %d test%s in %.3fs" % 354 (run, run != 1 and "s" or "", time_taken)) 355 self.stream.writeln() 356 357 # Error traces 358 if not result.wasSuccessful(): 359 self.stream.write("FAILED (") 360 failed, errored = (len(result.failures), len(result.errors)) 361 if failed: 362 self.stream.write("failures=%d" % failed) 363 if errored: 364 if failed: 365 self.stream.write(", ") 366 self.stream.write("errors=%d" % errored) 367 self.stream.writeln(")") 368 else: 369 self.stream.writeln("OK") 370 371 # Generate reports 372 self.stream.writeln() 373 self.stream.writeln('Generating XML reports...') 374 result.generate_reports(self) 375 376 return result 377