1# -*- coding: utf-8 -*-
2
3"""unittest-xml-reporting is a PyUnit-based TestRunner that can export test
4results to XML files that can be consumed by a wide range of tools, such as
5build systems, IDEs and Continuous Integration servers.
6
7This module provides the XMLTestRunner class, which is heavily based on the
8default TextTestRunner. This makes the XMLTestRunner very simple to use.
9
10The script below, adapted from the unittest documentation, shows how to use
11XMLTestRunner in a very simple way. In fact, the only difference between this
12script and the original one is the last line:
13
14import random
15import unittest
16import xmlrunner
17
18class TestSequenceFunctions(unittest.TestCase):
19    def setUp(self):
20        self.seq = range(10)
21
22    def test_shuffle(self):
23        # make sure the shuffled sequence does not lose any elements
24        random.shuffle(self.seq)
25        self.seq.sort()
26        self.assertEqual(self.seq, range(10))
27
28    def test_choice(self):
29        element = random.choice(self.seq)
30        self.assert_(element in self.seq)
31
32    def test_sample(self):
33        self.assertRaises(ValueError, random.sample, self.seq, 20)
34        for element in random.sample(self.seq, 5):
35            self.assert_(element in self.seq)
36
37if __name__ == '__main__':
38    unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports'))
39"""
40
41import os
42import sys
43import time
44from unittest import TestResult, _TextTestResult, TextTestRunner
45from cStringIO import StringIO
46import xml.dom.minidom
47
48
49class XMLDocument(xml.dom.minidom.Document):
50    def createCDATAOrText(self, data):
51        if ']]>' in data:
52            return self.createTextNode(data)
53        return self.createCDATASection(data)
54
55
56class _TestInfo(object):
57    """This class is used to keep useful information about the execution of a
58    test method.
59    """
60
61    # Possible test outcomes
62    (SUCCESS, FAILURE, ERROR) = range(3)
63
64    def __init__(self, test_result, test_method, outcome=SUCCESS, err=None):
65        "Create a new instance of _TestInfo."
66        self.test_result = test_result
67        self.test_method = test_method
68        self.outcome = outcome
69        self.err = err
70        self.stdout = test_result.stdout and test_result.stdout.getvalue().strip() or ''
71        self.stderr = test_result.stdout and test_result.stderr.getvalue().strip() or ''
72
73    def get_elapsed_time(self):
74        """Return the time that shows how long the test method took to
75        execute.
76        """
77        return self.test_result.stop_time - self.test_result.start_time
78
79    def get_description(self):
80        "Return a text representation of the test method."
81        return self.test_result.getDescription(self.test_method)
82
83    def get_error_info(self):
84        """Return a text representation of an exception thrown by a test
85        method.
86        """
87        if not self.err:
88            return ''
89        if sys.version_info < (2,4):
90            return self.test_result._exc_info_to_string(self.err)
91        else:
92            return self.test_result._exc_info_to_string(
93                self.err, self.test_method)
94
95
96class _XMLTestResult(_TextTestResult):
97    """A test result class that can express test results in a XML report.
98
99    Used by XMLTestRunner.
100    """
101    def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, \
102        elapsed_times=True):
103        "Create a new instance of _XMLTestResult."
104        _TextTestResult.__init__(self, stream, descriptions, verbosity)
105        self.successes = []
106        self.callback = None
107        self.elapsed_times = elapsed_times
108        self.output_patched = False
109
110    def _prepare_callback(self, test_info, target_list, verbose_str,
111        short_str):
112        """Append a _TestInfo to the given target list and sets a callback
113        method to be called by stopTest method.
114        """
115        target_list.append(test_info)
116        def callback():
117            """This callback prints the test method outcome to the stream,
118            as well as the elapsed time.
119            """
120
121            # Ignore the elapsed times for a more reliable unit testing
122            if not self.elapsed_times:
123                self.start_time = self.stop_time = 0
124
125            if self.showAll:
126                self.stream.writeln('(%.3fs) %s' % \
127                    (test_info.get_elapsed_time(), verbose_str))
128            elif self.dots:
129                self.stream.write(short_str)
130        self.callback = callback
131
132    def _patch_standard_output(self):
133        """Replace the stdout and stderr streams with string-based streams
134        in order to capture the tests' output.
135        """
136        if not self.output_patched:
137            (self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr)
138            self.output_patched = True
139        (sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \
140            (StringIO(), StringIO())
141
142    def _restore_standard_output(self):
143        "Restore the stdout and stderr streams."
144        (sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr)
145        self.output_patched = False
146
147    def startTest(self, test):
148        "Called before execute each test method."
149        self._patch_standard_output()
150        self.start_time = time.time()
151        TestResult.startTest(self, test)
152
153        if self.showAll:
154            self.stream.write('  ' + self.getDescription(test))
155            self.stream.write(" ... ")
156
157    def stopTest(self, test):
158        "Called after execute each test method."
159        self._restore_standard_output()
160        _TextTestResult.stopTest(self, test)
161        self.stop_time = time.time()
162
163        if self.callback and callable(self.callback):
164            self.callback()
165            self.callback = None
166
167    def addSuccess(self, test):
168        "Called when a test executes successfully."
169        self._prepare_callback(_TestInfo(self, test),
170                               self.successes, 'OK', '.')
171
172    def addFailure(self, test, err):
173        "Called when a test method fails."
174        self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err),
175                               self.failures, 'FAIL', 'F')
176
177    def addError(self, test, err):
178        "Called when a test method raises an error."
179        self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err),
180                               self.errors, 'ERROR', 'E')
181
182    def printErrorList(self, flavour, errors):
183        "Write some information about the FAIL or ERROR to the stream."
184        for test_info in errors:
185            if isinstance(test_info, tuple):
186                test_info, exc_info = test_info
187            self.stream.writeln(self.separator1)
188            self.stream.writeln('%s [%.3fs]: %s' % (
189                flavour, test_info.get_elapsed_time(),
190                test_info.get_description()))
191            self.stream.writeln(self.separator2)
192            self.stream.writeln('%s' % test_info.get_error_info())
193
194    def _get_info_by_testcase(self):
195        """This method organizes test results by TestCase module. This
196        information is used during the report generation, where a XML report
197        will be generated for each TestCase.
198        """
199        tests_by_testcase = {}
200
201        for tests in (self.successes, self.failures, self.errors):
202            for test_info in tests:
203                testcase = type(test_info.test_method)
204
205                # Ignore module name if it is '__main__'
206                module = testcase.__module__ + '.'
207                if module == '__main__.':
208                    module = ''
209                testcase_name = module + testcase.__name__
210
211                if testcase_name not in tests_by_testcase:
212                    tests_by_testcase[testcase_name] = []
213                tests_by_testcase[testcase_name].append(test_info)
214
215        return tests_by_testcase
216
217    def _report_testsuite(suite_name, tests, xml_document):
218        "Appends the testsuite section to the XML document."
219        testsuite = xml_document.createElement('testsuite')
220        xml_document.appendChild(testsuite)
221
222        testsuite.setAttribute('name', str(suite_name))
223        testsuite.setAttribute('tests', str(len(tests)))
224
225        testsuite.setAttribute('time', '%.3f' %
226            sum([e.get_elapsed_time() for e in tests]))
227
228        failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE])
229        testsuite.setAttribute('failures', str(failures))
230
231        errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR])
232        testsuite.setAttribute('errors', str(errors))
233
234        return testsuite
235
236    _report_testsuite = staticmethod(_report_testsuite)
237
238    def _report_testcase(suite_name, test_result, xml_testsuite, xml_document):
239        "Appends a testcase section to the XML document."
240        testcase = xml_document.createElement('testcase')
241        xml_testsuite.appendChild(testcase)
242
243        testcase.setAttribute('classname', str(suite_name))
244        testcase.setAttribute('name', test_result.test_method.shortDescription()
245                              or getattr(test_result.test_method, '_testMethodName',
246                                         str(test_result.test_method)))
247        testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time())
248
249        if (test_result.outcome != _TestInfo.SUCCESS):
250            elem_name = ('failure', 'error')[test_result.outcome-1]
251            failure = xml_document.createElement(elem_name)
252            testcase.appendChild(failure)
253
254            failure.setAttribute('type', str(test_result.err[0].__name__))
255            failure.setAttribute('message', str(test_result.err[1]))
256
257            error_info = test_result.get_error_info()
258            failureText = xml_document.createCDATAOrText(error_info)
259            failure.appendChild(failureText)
260
261    _report_testcase = staticmethod(_report_testcase)
262
263    def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr):
264        "Appends the system-out and system-err sections to the XML document."
265        systemout = xml_document.createElement('system-out')
266        xml_testsuite.appendChild(systemout)
267
268        systemout_text = xml_document.createCDATAOrText(stdout)
269        systemout.appendChild(systemout_text)
270
271        systemerr = xml_document.createElement('system-err')
272        xml_testsuite.appendChild(systemerr)
273
274        systemerr_text = xml_document.createCDATAOrText(stderr)
275        systemerr.appendChild(systemerr_text)
276
277    _report_output = staticmethod(_report_output)
278
279    def generate_reports(self, test_runner):
280        "Generates the XML reports to a given XMLTestRunner object."
281        all_results = self._get_info_by_testcase()
282
283        if type(test_runner.output) == str and not \
284            os.path.exists(test_runner.output):
285            os.makedirs(test_runner.output)
286
287        for suite, tests in all_results.items():
288            doc = XMLDocument()
289
290            # Build the XML file
291            testsuite = _XMLTestResult._report_testsuite(suite, tests, doc)
292            stdout, stderr = [], []
293            for test in tests:
294                _XMLTestResult._report_testcase(suite, test, testsuite, doc)
295                if test.stdout:
296                    stdout.extend(['*****************', test.get_description(), test.stdout])
297                if test.stderr:
298                    stderr.extend(['*****************', test.get_description(), test.stderr])
299            _XMLTestResult._report_output(test_runner, testsuite, doc,
300                                          '\n'.join(stdout), '\n'.join(stderr))
301            xml_content = doc.toprettyxml(indent='\t')
302
303            if type(test_runner.output) is str:
304                report_file = open('%s%sTEST-%s.xml' % \
305                    (test_runner.output, os.sep, suite), 'w')
306                try:
307                    report_file.write(xml_content)
308                finally:
309                    report_file.close()
310            else:
311                # Assume that test_runner.output is a stream
312                test_runner.output.write(xml_content)
313
314
315class XMLTestRunner(TextTestRunner):
316    """A test runner class that outputs the results in JUnit like XML files.
317    """
318    def __init__(self, output='.', stream=sys.stderr, descriptions=True, \
319        verbose=False, elapsed_times=True):
320        "Create a new instance of XMLTestRunner."
321        verbosity = (1, 2)[verbose]
322        TextTestRunner.__init__(self, stream, descriptions, verbosity)
323        self.output = output
324        self.elapsed_times = elapsed_times
325
326    def _make_result(self):
327        """Create the TestResult object which will be used to store
328        information about the executed tests.
329        """
330        return _XMLTestResult(self.stream, self.descriptions, \
331            self.verbosity, self.elapsed_times)
332
333    def run(self, test):
334        "Run the given test case or test suite."
335        # Prepare the test execution
336        result = self._make_result()
337
338        # Print a nice header
339        self.stream.writeln()
340        self.stream.writeln('Running tests...')
341        self.stream.writeln(result.separator2)
342
343        # Execute tests
344        start_time = time.time()
345        test(result)
346        stop_time = time.time()
347        time_taken = stop_time - start_time
348
349        # Print results
350        result.printErrors()
351        self.stream.writeln(result.separator2)
352        run = result.testsRun
353        self.stream.writeln("Ran %d test%s in %.3fs" %
354            (run, run != 1 and "s" or "", time_taken))
355        self.stream.writeln()
356
357        # Error traces
358        if not result.wasSuccessful():
359            self.stream.write("FAILED (")
360            failed, errored = (len(result.failures), len(result.errors))
361            if failed:
362                self.stream.write("failures=%d" % failed)
363            if errored:
364                if failed:
365                    self.stream.write(", ")
366                self.stream.write("errors=%d" % errored)
367            self.stream.writeln(")")
368        else:
369            self.stream.writeln("OK")
370
371        # Generate reports
372        self.stream.writeln()
373        self.stream.writeln('Generating XML reports...')
374        result.generate_reports(self)
375
376        return result
377