1#!/usr/bin/env python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30import logging 31import os 32import simplejson 33 34from layout_package import json_results_generator 35from layout_package import test_expectations 36from layout_package import test_failures 37 38 39class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): 40 """A JSON results generator for layout tests.""" 41 42 LAYOUT_TESTS_PATH = "LayoutTests" 43 44 # Additional JSON fields. 45 WONTFIX = "wontfixCounts" 46 DEFERRED = "deferredCounts" 47 48 def __init__(self, port, builder_name, build_name, build_number, 49 results_file_base_path, builder_base_url, 50 test_timings, expectations, result_summary, all_tests): 51 """Modifies the results.json file. Grabs it off the archive directory 52 if it is not found locally. 53 54 Args: 55 result_summary: ResultsSummary object storing the summary of the test 56 results. 57 (see the comment of JSONResultsGenerator.__init__ for other Args) 58 """ 59 self._port = port 60 self._builder_name = builder_name 61 self._build_name = build_name 62 self._build_number = build_number 63 self._builder_base_url = builder_base_url 64 self._results_file_path = os.path.join(results_file_base_path, 65 self.RESULTS_FILENAME) 66 self._expectations = expectations 67 68 # We don't use self._skipped_tests and self._passed_tests as we 69 # override _InsertFailureSummaries. 70 71 # We want relative paths to LayoutTest root for JSON output. 72 path_to_name = self._get_path_relative_to_layout_test_root 73 self._result_summary = result_summary 74 self._failures = dict( 75 (path_to_name(test), test_failures.determine_result_type(failures)) 76 for (test, failures) in result_summary.failures.iteritems()) 77 self._all_tests = [path_to_name(test) for test in all_tests] 78 self._test_timings = dict( 79 (path_to_name(test_tuple.filename), test_tuple.test_run_time) 80 for test_tuple in test_timings) 81 82 self._generate_json_output() 83 84 def _get_path_relative_to_layout_test_root(self, test): 85 """Returns the path of the test relative to the layout test root. 86 For example, for: 87 src/third_party/WebKit/LayoutTests/fast/forms/foo.html 88 We would return 89 fast/forms/foo.html 90 """ 91 index = test.find(self.LAYOUT_TESTS_PATH) 92 if index is not -1: 93 index += len(self.LAYOUT_TESTS_PATH) 94 95 if index is -1: 96 # Already a relative path. 97 relativePath = test 98 else: 99 relativePath = test[index + 1:] 100 101 # Make sure all paths are unix-style. 102 return relativePath.replace('\\', '/') 103 104 # override 105 def _convert_json_to_current_version(self, results_json): 106 archive_version = None 107 if self.VERSION_KEY in results_json: 108 archive_version = results_json[self.VERSION_KEY] 109 110 super(JSONLayoutResultsGenerator, 111 self)._convert_json_to_current_version(results_json) 112 113 # version 2->3 114 if archive_version == 2: 115 for results_for_builder in results_json.itervalues(): 116 try: 117 test_results = results_for_builder[self.TESTS] 118 except: 119 continue 120 121 for test in test_results: 122 # Make sure all paths are relative 123 test_path = self._get_path_relative_to_layout_test_root(test) 124 if test_path != test: 125 test_results[test_path] = test_results[test] 126 del test_results[test] 127 128 # override 129 def _insert_failure_summaries(self, results_for_builder): 130 summary = self._result_summary 131 132 self._insert_item_into_raw_list(results_for_builder, 133 len((set(summary.failures.keys()) | 134 summary.tests_by_expectation[test_expectations.SKIP]) & 135 summary.tests_by_timeline[test_expectations.NOW]), 136 self.FIXABLE_COUNT) 137 self._insert_item_into_raw_list(results_for_builder, 138 self._get_failure_summary_entry(test_expectations.NOW), 139 self.FIXABLE) 140 self._insert_item_into_raw_list(results_for_builder, 141 len(self._expectations.get_tests_with_timeline( 142 test_expectations.NOW)), self.ALL_FIXABLE_COUNT) 143 self._insert_item_into_raw_list(results_for_builder, 144 self._get_failure_summary_entry(test_expectations.DEFER), 145 self.DEFERRED) 146 self._insert_item_into_raw_list(results_for_builder, 147 self._get_failure_summary_entry(test_expectations.WONTFIX), 148 self.WONTFIX) 149 150 # override 151 def _normalize_results_json(self, test, test_name, tests): 152 super(JSONLayoutResultsGenerator, self)._normalize_results_json( 153 test, test_name, tests) 154 155 # Remove tests that don't exist anymore. 156 full_path = os.path.join(self._port.layout_tests_dir(), test_name) 157 full_path = os.path.normpath(full_path) 158 if not os.path.exists(full_path): 159 del tests[test_name] 160 161 def _get_failure_summary_entry(self, timeline): 162 """Creates a summary object to insert into the JSON. 163 164 Args: 165 summary ResultSummary object with test results 166 timeline current test_expectations timeline to build entry for 167 (e.g., test_expectations.NOW, etc.) 168 """ 169 entry = {} 170 summary = self._result_summary 171 timeline_tests = summary.tests_by_timeline[timeline] 172 entry[self.SKIP_RESULT] = len( 173 summary.tests_by_expectation[test_expectations.SKIP] & 174 timeline_tests) 175 entry[self.PASS_RESULT] = len( 176 summary.tests_by_expectation[test_expectations.PASS] & 177 timeline_tests) 178 for failure_type in summary.tests_by_expectation.keys(): 179 if failure_type not in self.FAILURE_TO_CHAR: 180 continue 181 count = len(summary.tests_by_expectation[failure_type] & 182 timeline_tests) 183 entry[self.FAILURE_TO_CHAR[failure_type]] = count 184 return entry 185