1#!/usr/bin/env python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30import logging
31import os
32import simplejson
33
34from layout_package import json_results_generator
35from layout_package import test_expectations
36from layout_package import test_failures
37
38
39class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator):
40    """A JSON results generator for layout tests."""
41
42    LAYOUT_TESTS_PATH = "LayoutTests"
43
44    # Additional JSON fields.
45    WONTFIX = "wontfixCounts"
46    DEFERRED = "deferredCounts"
47
48    def __init__(self, port, builder_name, build_name, build_number,
49        results_file_base_path, builder_base_url,
50        test_timings, expectations, result_summary, all_tests):
51        """Modifies the results.json file. Grabs it off the archive directory
52        if it is not found locally.
53
54        Args:
55          result_summary: ResultsSummary object storing the summary of the test
56              results.
57          (see the comment of JSONResultsGenerator.__init__ for other Args)
58        """
59        self._port = port
60        self._builder_name = builder_name
61        self._build_name = build_name
62        self._build_number = build_number
63        self._builder_base_url = builder_base_url
64        self._results_file_path = os.path.join(results_file_base_path,
65            self.RESULTS_FILENAME)
66        self._expectations = expectations
67
68        # We don't use self._skipped_tests and self._passed_tests as we
69        # override _InsertFailureSummaries.
70
71        # We want relative paths to LayoutTest root for JSON output.
72        path_to_name = self._get_path_relative_to_layout_test_root
73        self._result_summary = result_summary
74        self._failures = dict(
75            (path_to_name(test), test_failures.determine_result_type(failures))
76            for (test, failures) in result_summary.failures.iteritems())
77        self._all_tests = [path_to_name(test) for test in all_tests]
78        self._test_timings = dict(
79            (path_to_name(test_tuple.filename), test_tuple.test_run_time)
80            for test_tuple in test_timings)
81
82        self._generate_json_output()
83
84    def _get_path_relative_to_layout_test_root(self, test):
85        """Returns the path of the test relative to the layout test root.
86        For example, for:
87          src/third_party/WebKit/LayoutTests/fast/forms/foo.html
88        We would return
89          fast/forms/foo.html
90        """
91        index = test.find(self.LAYOUT_TESTS_PATH)
92        if index is not -1:
93            index += len(self.LAYOUT_TESTS_PATH)
94
95        if index is -1:
96            # Already a relative path.
97            relativePath = test
98        else:
99            relativePath = test[index + 1:]
100
101        # Make sure all paths are unix-style.
102        return relativePath.replace('\\', '/')
103
104    # override
105    def _convert_json_to_current_version(self, results_json):
106        archive_version = None
107        if self.VERSION_KEY in results_json:
108            archive_version = results_json[self.VERSION_KEY]
109
110        super(JSONLayoutResultsGenerator,
111              self)._convert_json_to_current_version(results_json)
112
113        # version 2->3
114        if archive_version == 2:
115            for results_for_builder in results_json.itervalues():
116                try:
117                    test_results = results_for_builder[self.TESTS]
118                except:
119                    continue
120
121            for test in test_results:
122                # Make sure all paths are relative
123                test_path = self._get_path_relative_to_layout_test_root(test)
124                if test_path != test:
125                    test_results[test_path] = test_results[test]
126                    del test_results[test]
127
128    # override
129    def _insert_failure_summaries(self, results_for_builder):
130        summary = self._result_summary
131
132        self._insert_item_into_raw_list(results_for_builder,
133            len((set(summary.failures.keys()) |
134                summary.tests_by_expectation[test_expectations.SKIP]) &
135                summary.tests_by_timeline[test_expectations.NOW]),
136            self.FIXABLE_COUNT)
137        self._insert_item_into_raw_list(results_for_builder,
138            self._get_failure_summary_entry(test_expectations.NOW),
139            self.FIXABLE)
140        self._insert_item_into_raw_list(results_for_builder,
141            len(self._expectations.get_tests_with_timeline(
142                test_expectations.NOW)), self.ALL_FIXABLE_COUNT)
143        self._insert_item_into_raw_list(results_for_builder,
144            self._get_failure_summary_entry(test_expectations.DEFER),
145            self.DEFERRED)
146        self._insert_item_into_raw_list(results_for_builder,
147            self._get_failure_summary_entry(test_expectations.WONTFIX),
148            self.WONTFIX)
149
150    # override
151    def _normalize_results_json(self, test, test_name, tests):
152        super(JSONLayoutResultsGenerator, self)._normalize_results_json(
153            test, test_name, tests)
154
155        # Remove tests that don't exist anymore.
156        full_path = os.path.join(self._port.layout_tests_dir(), test_name)
157        full_path = os.path.normpath(full_path)
158        if not os.path.exists(full_path):
159            del tests[test_name]
160
161    def _get_failure_summary_entry(self, timeline):
162        """Creates a summary object to insert into the JSON.
163
164        Args:
165          summary   ResultSummary object with test results
166          timeline  current test_expectations timeline to build entry for
167                    (e.g., test_expectations.NOW, etc.)
168        """
169        entry = {}
170        summary = self._result_summary
171        timeline_tests = summary.tests_by_timeline[timeline]
172        entry[self.SKIP_RESULT] = len(
173            summary.tests_by_expectation[test_expectations.SKIP] &
174            timeline_tests)
175        entry[self.PASS_RESULT] = len(
176            summary.tests_by_expectation[test_expectations.PASS] &
177            timeline_tests)
178        for failure_type in summary.tests_by_expectation.keys():
179            if failure_type not in self.FAILURE_TO_CHAR:
180                continue
181            count = len(summary.tests_by_expectation[failure_type] &
182                        timeline_tests)
183            entry[self.FAILURE_TO_CHAR[failure_type]] = count
184        return entry
185