xmlrunner.py revision 324c4644fee44b9898524c09511bd33c3f12e2df 
1"""
2XML Test Runner for PyUnit
3"""
4
5# Written by Sebastian Rittau <srittau@jroger.in-berlin.de> and placed in
6# the Public Domain. With contributions by Paolo Borelli.
7
8__revision__ = "$Id: /private/python/stdlib/xmlrunner.py 16654 2007-11-12T12:46:35.368945Z srittau  $"
9
10import os.path
11import re
12import sys
13import time
14import traceback
15import unittest
16from StringIO import StringIO
17from xml.sax.saxutils import escape
18
19from StringIO import StringIO
20
21
22class _TestInfo(object):
23
24    """Information about a particular test.
25
26    Used by _XMLTestResult.
27
28    """
29
30    def __init__(self, test, time):
31        (self._class, self._method) = test.id().rsplit(".", 1)
32        self._time = time
33        self._error = None
34        self._failure = None
35
36    @staticmethod
37    def create_success(test, time):
38        """Create a _TestInfo instance for a successful test."""
39        return _TestInfo(test, time)
40
41    @staticmethod
42    def create_failure(test, time, failure):
43        """Create a _TestInfo instance for a failed test."""
44        info = _TestInfo(test, time)
45        info._failure = failure
46        return info
47
48    @staticmethod
49    def create_error(test, time, error):
50        """Create a _TestInfo instance for an erroneous test."""
51        info = _TestInfo(test, time)
52        info._error = error
53        return info
54
55    def print_report(self, stream):
56        """Print information about this test case in XML format to the
57        supplied stream.
58
59        """
60        stream.write('  <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \
61            {
62                "class": self._class,
63                "method": self._method,
64                "time": self._time,
65            })
66        if self._failure != None:
67            self._print_error(stream, 'failure', self._failure)
68        if self._error != None:
69            self._print_error(stream, 'error', self._error)
70        stream.write('</testcase>\n')
71
72    def _print_error(self, stream, tagname, error):
73        """Print information from a failure or error to the supplied stream."""
74        text = escape(str(error[1]))
75        stream.write('\n')
76        stream.write('    <%s type="%s">%s\n' \
77            % (tagname, str(error[0]), text))
78        tb_stream = StringIO()
79        traceback.print_tb(error[2], None, tb_stream)
80        stream.write(escape(tb_stream.getvalue()))
81        stream.write('    </%s>\n' % tagname)
82        stream.write('  ')
83
84
85class _XMLTestResult(unittest.TestResult):
86
87    """A test result class that stores result as XML.
88
89    Used by XMLTestRunner.
90
91    """
92
93    def __init__(self, classname):
94        unittest.TestResult.__init__(self)
95        self._test_name = classname
96        self._start_time = None
97        self._tests = []
98        self._error = None
99        self._failure = None
100
101    def startTest(self, test):
102        unittest.TestResult.startTest(self, test)
103        self._error = None
104        self._failure = None
105        self._start_time = time.time()
106
107    def stopTest(self, test):
108        time_taken = time.time() - self._start_time
109        unittest.TestResult.stopTest(self, test)
110        if self._error:
111            info = _TestInfo.create_error(test, time_taken, self._error)
112        elif self._failure:
113            info = _TestInfo.create_failure(test, time_taken, self._failure)
114        else:
115            info = _TestInfo.create_success(test, time_taken)
116        self._tests.append(info)
117
118    def addError(self, test, err):
119        unittest.TestResult.addError(self, test, err)
120        self._error = err
121
122    def addFailure(self, test, err):
123        unittest.TestResult.addFailure(self, test, err)
124        self._failure = err
125
126    def print_report(self, stream, time_taken, out, err):
127        """Prints the XML report to the supplied stream.
128
129        The time the tests took to perform as well as the captured standard
130        output and standard error streams must be passed in.a
131
132        """
133        stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \
134            { "e": len(self.errors), "f": len(self.failures) })
135        stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \
136            {
137                "n": self._test_name,
138                "t": self.testsRun,
139                "time": time_taken,
140            })
141        for info in self._tests:
142            info.print_report(stream)
143        stream.write('  <system-out><![CDATA[%s]]></system-out>\n' % out)
144        stream.write('  <system-err><![CDATA[%s]]></system-err>\n' % err)
145        stream.write('</testsuite>\n')
146
147
148class XMLTestRunner(object):
149
150    """A test runner that stores results in XML format compatible with JUnit.
151
152    XMLTestRunner(stream=None) -> XML test runner
153
154    The XML file is written to the supplied stream. If stream is None, the
155    results are stored in a file called TEST-<module>.<class>.xml in the
156    current working directory (if not overridden with the path property),
157    where <module> and <class> are the module and class name of the test class.
158
159    """
160
161    def __init__(self, stream=None):
162        self._stream = stream
163        self._path = "."
164
165    def run(self, test):
166        """Run the given test case or test suite."""
167        class_ = test.__class__
168        classname = class_.__module__ + "." + class_.__name__
169        if self._stream == None:
170            filename = "TEST-%s.xml" % classname
171            stream = file(os.path.join(self._path, filename), "w")
172            stream.write('<?xml version="1.0" encoding="utf-8"?>\n')
173        else:
174            stream = self._stream
175
176        result = _XMLTestResult(classname)
177        start_time = time.time()
178
179        # TODO: Python 2.5: Use the with statement
180        old_stdout = sys.stdout
181        old_stderr = sys.stderr
182        sys.stdout = StringIO()
183        sys.stderr = StringIO()
184
185        try:
186            test(result)
187            try:
188                out_s = sys.stdout.getvalue()
189            except AttributeError:
190                out_s = ""
191            try:
192                err_s = sys.stderr.getvalue()
193            except AttributeError:
194                err_s = ""
195        finally:
196            sys.stdout = old_stdout
197            sys.stderr = old_stderr
198
199        time_taken = time.time() - start_time
200        result.print_report(stream, time_taken, out_s, err_s)
201        if self._stream == None:
202            stream.close()
203
204        return result
205
206    def _set_path(self, path):
207        self._path = path
208
209    path = property(lambda self: self._path, _set_path, None,
210            """The path where the XML files are stored.
211
212            This property is ignored when the XML file is written to a file
213            stream.""")
214
215
216class XMLTestRunnerTest(unittest.TestCase):
217    def setUp(self):
218        self._stream = StringIO()
219
220    def _try_test_run(self, test_class, expected):
221
222        """Run the test suite against the supplied test class and compare the
223        XML result against the expected XML string. Fail if the expected
224        string doesn't match the actual string. All time attribute in the
225        expected string should have the value "0.000". All error and failure
226        messages are reduced to "Foobar".
227
228        """
229
230        runner = XMLTestRunner(self._stream)
231        runner.run(unittest.makeSuite(test_class))
232
233        got = self._stream.getvalue()
234        # Replace all time="X.YYY" attributes by time="0.000" to enable a
235        # simple string comparison.
236        got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got)
237        # Likewise, replace all failure and error messages by a simple "Foobar"
238        # string.
239        got = re.sub(r'(?s)<failure (.*?)>.*?</failure>', r'<failure \1>Foobar</failure>', got)
240        got = re.sub(r'(?s)<error (.*?)>.*?</error>', r'<error \1>Foobar</error>', got)
241
242        self.assertEqual(expected, got)
243
244    def test_no_tests(self):
245        """Regression test: Check whether a test run without any tests
246        matches a previous run.
247
248        """
249        class TestTest(unittest.TestCase):
250            pass
251        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="0" time="0.000">
252  <system-out><![CDATA[]]></system-out>
253  <system-err><![CDATA[]]></system-err>
254</testsuite>
255""")
256
257    def test_success(self):
258        """Regression test: Check whether a test run with a successful test
259        matches a previous run.
260
261        """
262        class TestTest(unittest.TestCase):
263            def test_foo(self):
264                pass
265        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
266  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
267  <system-out><![CDATA[]]></system-out>
268  <system-err><![CDATA[]]></system-err>
269</testsuite>
270""")
271
272    def test_failure(self):
273        """Regression test: Check whether a test run with a failing test
274        matches a previous run.
275
276        """
277        class TestTest(unittest.TestCase):
278            def test_foo(self):
279                self.assert_(False)
280        self._try_test_run(TestTest, """<testsuite errors="0" failures="1" name="unittest.TestSuite" tests="1" time="0.000">
281  <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
282    <failure type="exceptions.AssertionError">Foobar</failure>
283  </testcase>
284  <system-out><![CDATA[]]></system-out>
285  <system-err><![CDATA[]]></system-err>
286</testsuite>
287""")
288
289    def test_error(self):
290        """Regression test: Check whether a test run with a erroneous test
291        matches a previous run.
292
293        """
294        class TestTest(unittest.TestCase):
295            def test_foo(self):
296                raise IndexError()
297        self._try_test_run(TestTest, """<testsuite errors="1" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
298  <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
299    <error type="exceptions.IndexError">Foobar</error>
300  </testcase>
301  <system-out><![CDATA[]]></system-out>
302  <system-err><![CDATA[]]></system-err>
303</testsuite>
304""")
305
306    def test_stdout_capture(self):
307        """Regression test: Check whether a test run with output to stdout
308        matches a previous run.
309
310        """
311        class TestTest(unittest.TestCase):
312            def test_foo(self):
313                print "Test"
314        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
315  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
316  <system-out><![CDATA[Test
317]]></system-out>
318  <system-err><![CDATA[]]></system-err>
319</testsuite>
320""")
321
322    def test_stderr_capture(self):
323        """Regression test: Check whether a test run with output to stderr
324        matches a previous run.
325
326        """
327        class TestTest(unittest.TestCase):
328            def test_foo(self):
329                print >>sys.stderr, "Test"
330        self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
331  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
332  <system-out><![CDATA[]]></system-out>
333  <system-err><![CDATA[Test
334]]></system-err>
335</testsuite>
336""")
337
338    class NullStream(object):
339        """A file-like object that discards everything written to it."""
340        def write(self, buffer):
341            pass
342
343    def test_unittests_changing_stdout(self):
344        """Check whether the XMLTestRunner recovers gracefully from unit tests
345        that change stdout, but don't change it back properly.
346
347        """
348        class TestTest(unittest.TestCase):
349            def test_foo(self):
350                sys.stdout = XMLTestRunnerTest.NullStream()
351
352        runner = XMLTestRunner(self._stream)
353        runner.run(unittest.makeSuite(TestTest))
354
355    def test_unittests_changing_stderr(self):
356        """Check whether the XMLTestRunner recovers gracefully from unit tests
357        that change stderr, but don't change it back properly.
358
359        """
360        class TestTest(unittest.TestCase):
361            def test_foo(self):
362                sys.stderr = XMLTestRunnerTest.NullStream()
363
364        runner = XMLTestRunner(self._stream)
365        runner.run(unittest.makeSuite(TestTest))
366
367
368class XMLTestProgram(unittest.TestProgram):
369    def runTests(self):
370        if self.testRunner is None:
371            self.testRunner = XMLTestRunner()
372        unittest.TestProgram.runTests(self)
373
374main = XMLTestProgram
375
376
377if __name__ == "__main__":
378    main(module=None)
379