layouttestresults.py revision 2daae5fd11344eaa88a0d92b0f6d65f8d2255c00 
1# Copyright (c) 2010, Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29# A module for parsing results.html files generated by old-run-webkit-tests
30# This class is one big hack and only needs to exist until we transition to new-run-webkit-tests.
31
32from webkitpy.common.system.deprecated_logging import log
33from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer
34from webkitpy.layout_tests.layout_package import test_results
35from webkitpy.layout_tests.layout_package import test_failures
36
37
38# FIXME: This should be unified with all the layout test results code in the layout_tests package
39# This doesn't belong in common.net, but we don't have a better place for it yet.
40def path_for_layout_test(test_name):
41    return "LayoutTests/%s" % test_name
42
43
44# FIXME: This should be unified with all the layout test results code in the layout_tests package
45# This doesn't belong in common.net, but we don't have a better place for it yet.
46class LayoutTestResults(object):
47    """This class knows how to parse old-run-webkit-tests results.html files."""
48
49    stderr_key = u'Tests that had stderr output:'
50    fail_key = u'Tests where results did not match expected results:'
51    timeout_key = u'Tests that timed out:'
52    crash_key = u'Tests that caused the DumpRenderTree tool to crash:'
53    missing_key = u'Tests that had no expected results (probably new):'
54    webprocess_crash_key = u'Tests that caused the Web process to crash:'
55
56    expected_keys = [
57        stderr_key,
58        fail_key,
59        crash_key,
60        webprocess_crash_key,
61        timeout_key,
62        missing_key,
63    ]
64
65    @classmethod
66    def _failures_from_fail_row(self, row):
67        # Look at all anchors in this row, and guess what type
68        # of new-run-webkit-test failures they equate to.
69        failures = set()
70        test_name = None
71        for anchor in row.findAll("a"):
72            anchor_text = unicode(anchor.string)
73            if not test_name:
74                test_name = anchor_text
75                continue
76            if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text:
77                failures.add(test_failures.FailureImageHashMismatch())
78            elif anchor_text in ["expected", "actual", "diff", "pretty diff"]:
79                failures.add(test_failures.FailureTextMismatch())
80            else:
81                log("Unhandled link text in results.html parsing: %s.  Please file a bug against webkitpy." % anchor_text)
82        # FIXME: Its possible the row contained no links due to ORWT brokeness.
83        # We should probably assume some type of failure anyway.
84        return failures
85
86    @classmethod
87    def _failures_from_row(cls, row, table_title):
88        if table_title == cls.fail_key:
89            return cls._failures_from_fail_row(row)
90        if table_title == cls.crash_key:
91            return [test_failures.FailureCrash()]
92        if table_title == cls.webprocess_crash_key:
93            return [test_failures.FailureCrash()]
94        if table_title == cls.timeout_key:
95            return [test_failures.FailureTimeout()]
96        if table_title == cls.missing_key:
97            return [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()]
98        return None
99
100    @classmethod
101    def _test_result_from_row(cls, row, table_title):
102        test_name = unicode(row.find("a").string)
103        failures = cls._failures_from_row(row, table_title)
104        # TestResult is a class designed to work with new-run-webkit-tests.
105        # old-run-webkit-tests does not save quite enough information in results.html for us to parse.
106        # FIXME: It's unclear if test_name should include LayoutTests or not.
107        return test_results.TestResult(test_name, failures)
108
109    @classmethod
110    def _parse_results_table(cls, table):
111        table_title = unicode(table.findPreviousSibling("p").string)
112        if table_title not in cls.expected_keys:
113            # This Exception should only ever be hit if run-webkit-tests changes its results.html format.
114            raise Exception("Unhandled title: %s" % table_title)
115        # Ignore stderr failures.  Everyone ignores them anyway.
116        if table_title == cls.stderr_key:
117            return []
118        # FIXME: We might end with two TestResults object for the same test if it appears in more than one row.
119        return [cls._test_result_from_row(row, table_title) for row in table.findAll("tr")]
120
121    @classmethod
122    def _parse_results_html(cls, page):
123        tables = BeautifulSoup(page).findAll("table")
124        return sum([cls._parse_results_table(table) for table in tables], [])
125
126    @classmethod
127    def results_from_string(cls, string):
128        if not string:
129            return None
130        test_results = cls._parse_results_html(string)
131        if not test_results:
132            return None
133        return cls(test_results)
134
135    def __init__(self, test_results):
136        self._test_results = test_results
137        self._failure_limit_count = None
138
139    # FIXME: run-webkit-tests should store the --exit-after-N-failures value
140    # (or some indication of early exit) somewhere in the results.html/results.json
141    # file.  Until it does, callers should set the limit to
142    # --exit-after-N-failures value used in that run.  Consumers of LayoutTestResults
143    # may use that value to know if absence from the failure list means PASS.
144    # https://bugs.webkit.org/show_bug.cgi?id=58481
145    def set_failure_limit_count(self, limit):
146        self._failure_limit_count = limit
147
148    def failure_limit_count(self):
149        return self._failure_limit_count
150
151    def test_results(self):
152        return self._test_results
153
154    def results_matching_failure_types(self, failure_types):
155        return [result for result in self._test_results if result.has_failure_matching_types(failure_types)]
156
157    def tests_matching_failure_types(self, failure_types):
158        return [result.filename for result in self.results_matching_failure_types(failure_types)]
159
160    def failing_test_results(self):
161        # These should match the "fail", "crash", and "timeout" keys.
162        failure_types = [test_failures.FailureTextMismatch, test_failures.FailureImageHashMismatch, test_failures.FailureCrash, test_failures.FailureTimeout]
163        return self.results_matching_failure_types(failure_types)
164
165    def failing_tests(self):
166        return [result.filename for result in self.failing_test_results()]
167