1926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# Copyright (C) 2010 Google Inc. All rights reserved. 2926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 3926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 4926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# Redistribution and use in source and binary forms, with or without 5926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# modification, are permitted provided that the following conditions are 6926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# met: 7926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 8926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# * Redistributions of source code must retain the above copyright 9926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# notice, this list of conditions and the following disclaimer. 10926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# * Redistributions in binary form must reproduce the above 11926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 12926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# in the documentation and/or other materials provided with the 13926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# distribution. 14926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 15926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# contributors may be used to endorse or promote products derived from 16926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# this software without specific prior written permission. 17926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 18926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 30926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)import logging 317242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucciimport re 329bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles)import signal 3381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles)import time 34926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 35926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models import test_expectations 36926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models import test_failures 37926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 38926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 39926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)_log = logging.getLogger(__name__) 40926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 41f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)OK_EXIT_STATUS = 0 42f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) 43bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)# This matches what the shell does on POSIX. 449bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles)INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128 45926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 46bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)# POSIX limits status codes to 0-255. Normally run-webkit-tests returns the number 47bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)# of tests that failed. These indicate exceptional conditions triggered by the 48bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)# script itself, so we count backwards from 255 (aka -1) to enumerate them. 4910f88d5669dbd969c059d61ba09fa37dd72ac559Ben Murdoch# 5010f88d5669dbd969c059d61ba09fa37dd72ac559Ben Murdoch# FIXME: crbug.com/357866. We really shouldn't return the number of failures 5110f88d5669dbd969c059d61ba09fa37dd72ac559Ben Murdoch# in the exit code at all. 5210f88d5669dbd969c059d61ba09fa37dd72ac559Ben MurdochEARLY_EXIT_STATUS = 251 53f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)SYS_DEPS_EXIT_STATUS = 252 54bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)NO_TESTS_EXIT_STATUS = 253 55f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)NO_DEVICES_EXIT_STATUS = 254 56f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)UNEXPECTED_ERROR_EXIT_STATUS = 255 57bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) 58f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)ERROR_CODES = ( 59f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) INTERRUPTED_EXIT_STATUS, 6010f88d5669dbd969c059d61ba09fa37dd72ac559Ben Murdoch EARLY_EXIT_STATUS, 61f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) SYS_DEPS_EXIT_STATUS, 62f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) NO_TESTS_EXIT_STATUS, 63f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) NO_DEVICES_EXIT_STATUS, 64f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) UNEXPECTED_ERROR_EXIT_STATUS, 65f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)) 66bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) 67d5428f32f5d1719f774f62e19147104ca245a3abTorne (Richard Coles)# In order to avoid colliding with the above codes, we put a ceiling on 68d5428f32f5d1719f774f62e19147104ca245a3abTorne (Richard Coles)# the value returned by num_regressions 69d5428f32f5d1719f774f62e19147104ca245a3abTorne (Richard Coles)MAX_FAILURES_EXIT_STATUS = 101 70bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) 71bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)class TestRunException(Exception): 72bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) def __init__(self, code, msg): 73bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) self.code = code 74bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) self.msg = msg 75bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) 76bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) 77926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class TestRunResults(object): 78926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def __init__(self, expectations, num_tests): 79926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.total = num_tests 80926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.remaining = self.total 81926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.expectations = expectations 82926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.expected = 0 83591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch self.expected_failures = 0 84926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected = 0 85926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_failures = 0 86926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_crashes = 0 87926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_timeouts = 0 88926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.tests_by_expectation = {} 89926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.tests_by_timeline = {} 90926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.results_by_name = {} # Map of test name to the last result for the test. 91926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.all_results = [] # All results from a run, including every iteration of every test. 92926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_results_by_name = {} 93926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.failures_by_name = {} 94926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.total_failures = 0 95926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.expected_skips = 0 96926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for expectation in test_expectations.TestExpectations.EXPECTATIONS.values(): 97926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.tests_by_expectation[expectation] = set() 98926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for timeline in test_expectations.TestExpectations.TIMELINES.values(): 99926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.tests_by_timeline[timeline] = expectations.get_tests_with_timeline(timeline) 100926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.slow_tests = set() 101926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.interrupted = False 1029bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self.keyboard_interrupted = False 103591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch self.run_time = 0 # The wall clock time spent running the tests (layout_test_runner.run()). 104926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 105926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def add(self, test_result, expected, test_is_slow): 106f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) result_type_for_stats = test_result.type 107f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) if test_expectations.WONTFIX in self.expectations.model().get_expectations(test_result.test_name): 108f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) result_type_for_stats = test_expectations.WONTFIX 109f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) self.tests_by_expectation[result_type_for_stats].add(test_result.test_name) 110f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) 111926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.results_by_name[test_result.test_name] = test_result 112926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_result.type != test_expectations.SKIP: 113926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.all_results.append(test_result) 114926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.remaining -= 1 115926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if len(test_result.failures): 116926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.total_failures += 1 117926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.failures_by_name[test_result.test_name] = test_result.failures 118926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if expected: 119926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.expected += 1 120926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_result.type == test_expectations.SKIP: 121926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.expected_skips += 1 122591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch elif test_result.type != test_expectations.PASS: 123591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch self.expected_failures += 1 124926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else: 125926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_results_by_name[test_result.test_name] = test_result 126926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected += 1 127926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if len(test_result.failures): 128926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_failures += 1 129926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_result.type == test_expectations.CRASH: 130926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_crashes += 1 131926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) elif test_result.type == test_expectations.TIMEOUT: 132926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.unexpected_timeouts += 1 133926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_is_slow: 134926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.slow_tests.add(test_result.test_name) 135926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 136926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 137926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class RunDetails(object): 13893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def __init__(self, exit_code, summarized_full_results=None, summarized_failing_results=None, initial_results=None, retry_results=None, enabled_pixel_tests_in_retry=False): 139926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.exit_code = exit_code 14093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.summarized_full_results = summarized_full_results 14193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.summarized_failing_results = summarized_failing_results 142926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.initial_results = initial_results 143926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.retry_results = retry_results 144926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.enabled_pixel_tests_in_retry = enabled_pixel_tests_in_retry 145926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 146926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 147926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)def _interpret_test_failures(failures): 148926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict = {} 149926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) failure_types = [type(failure) for failure in failures] 150926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # FIXME: get rid of all this is_* values once there is a 1:1 map between 151926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # TestFailure type and test_expectations.EXPECTATION. 152926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_failures.FailureMissingAudio in failure_types: 153926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict['is_missing_audio'] = True 154926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 155926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_failures.FailureMissingResult in failure_types: 156926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict['is_missing_text'] = True 157926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 158926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types: 159926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict['is_missing_image'] = True 160926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 16109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) if test_failures.FailureTestHarnessAssertion in failure_types: 16209380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) test_dict['is_testharness_test'] = True 16309380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) 164926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return test_dict 165926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 166926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1677242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tuccidef _chromium_commit_position(scm, path): 1687242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci log = scm.most_recent_log_matching('Cr-Commit-Position:', path) 1697242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci match = re.search('^\s*Cr-Commit-Position:.*@\{#(?P<commit_position>\d+)\}', log, re.MULTILINE) 1707242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci if not match: 1717242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci return "" 1727242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci return str(match.group('commit_position')) 1737242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci 1747242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci 17593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)def summarize_results(port_obj, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, only_include_failing=False): 176926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) """Returns a dictionary containing a summary of the test runs, with the following fields: 177926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'version': a version indicator 178926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'fixable': The number of fixable tests (NOW - PASS) 179926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'skipped': The number of skipped tests (NOW & SKIPPED) 180926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'num_regressions': The number of non-flaky failures 181926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'num_flaky': The number of flaky failures 182926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'num_passes': The number of unexpected passes 183926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} 184926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) """ 185926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results = {} 186926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['version'] = 3 187926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 188926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) tbe = initial_results.tests_by_expectation 189926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) tbt = initial_results.tests_by_timeline 190926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['fixable'] = len(tbt[test_expectations.NOW] - tbe[test_expectations.PASS]) 19181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) # FIXME: Remove this. It is redundant with results['num_failures_by_type']. 192926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['skipped'] = len(tbt[test_expectations.NOW] & tbe[test_expectations.SKIP]) 193926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 194926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) num_passes = 0 195926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) num_flaky = 0 196926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) num_regressions = 0 197926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) keywords = {} 198926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for expecation_string, expectation_enum in test_expectations.TestExpectations.EXPECTATIONS.iteritems(): 199926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) keywords[expectation_enum] = expecation_string.upper() 200926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 20181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) num_failures_by_type = {} 20281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) for expectation in initial_results.tests_by_expectation: 203f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) tests = initial_results.tests_by_expectation[expectation] 204f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) if expectation != test_expectations.WONTFIX: 205f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) tests &= tbt[test_expectations.NOW] 206f5e4ad553afbc08dd2e729bb77e937a9a94d5827Torne (Richard Coles) num_failures_by_type[keywords[expectation]] = len(tests) 20781a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) # The number of failures by type. 20881a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) results['num_failures_by_type'] = num_failures_by_type 20981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 210926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) tests = {} 211926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 212926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for test_name, result in initial_results.results_by_name.iteritems(): 213926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) expected = expectations.get_expectations_string(test_name) 214926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) result_type = result.type 215926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) actual = [keywords[result_type]] 216926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 21793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if only_include_failing and result.type == test_expectations.SKIP: 21893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) continue 219926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 220926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if result_type == test_expectations.PASS: 221926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) num_passes += 1 22293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if not result.has_stderr and only_include_failing: 223e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) continue 22481a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) elif result_type != test_expectations.SKIP and test_name in initial_results.unexpected_results_by_name: 22509380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) if retry_results: 22609380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) if test_name not in retry_results.unexpected_results_by_name: 22709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) # The test failed unexpectedly at first, but ran as expected the second time -> flaky. 22809380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) actual.extend(expectations.get_expectations_string(test_name).split(" ")) 22909380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) num_flaky += 1 23009380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) else: 23109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) retry_result_type = retry_results.unexpected_results_by_name[test_name].type 23209380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) if retry_result_type == test_expectations.PASS: 23309380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) # The test failed unexpectedly at first, then passed unexpectedly -> unexpected pass. 23409380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) num_passes += 1 23509380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) if not result.has_stderr and only_include_failing: 23609380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) continue 23709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) else: 23809380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) # The test failed unexpectedly both times -> regression. 23909380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) num_regressions += 1 24009380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) if not keywords[retry_result_type] in actual: 24109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) actual.append(keywords[retry_result_type]) 242926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else: 24309380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) # The test failed unexpectedly, but we didn't do any retries -> regression. 244926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) num_regressions += 1 245926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 24693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_dict = {} 24793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 24893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) rounded_run_time = round(result.test_run_time, 1) 24993ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if rounded_run_time: 25093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_dict['time'] = rounded_run_time 25193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 25293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if result.has_stderr: 25393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_dict['has_stderr'] = True 25493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 255fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch bugs = expectations.model().get_expectation_line(test_name).bugs 25693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if bugs: 25793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_dict['bugs'] = bugs 25893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 25993ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if result.reftest_type: 26093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_dict.update(reftest_type=list(result.reftest_type)) 26193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 262926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict['expected'] = expected 263926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict['actual'] = " ".join(actual) 264926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 26581a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) def is_expected(actual_result): 266f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) return expectations.matches_an_expected_result(test_name, result_type, 267f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) port_obj.get_option('pixel_tests') or result.reftest_type, 268f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) port_obj.get_option('enable_sanitizer')) 26981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 27081a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) # To avoid bloating the output results json too much, only add an entry for whether the failure is unexpected. 27181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) if not all(is_expected(actual_result) for actual_result in actual): 27281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) test_dict['is_unexpected'] = True 27381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 274926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict.update(_interpret_test_failures(result.failures)) 275926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 276926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if retry_results: 277926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) retry_result = retry_results.unexpected_results_by_name.get(test_name) 278926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if retry_result: 279926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict.update(_interpret_test_failures(retry_result.failures)) 280926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 281f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) if (result.has_repaint_overlay): 282f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) test_dict['has_repaint_overlay'] = True 283f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 284926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # Store test hierarchically by directory. e.g. 285926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # foo/bar/baz.html: test_dict 286926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # foo/bar/baz1.html: test_dict 287926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # 288926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # becomes 289926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # foo: { 290926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # bar: { 291926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # baz.html: test_dict, 292926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # baz1.html: test_dict 293926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # } 294926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) # } 295926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) parts = test_name.split('/') 296926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) current_map = tests 297926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) for i, part in enumerate(parts): 298926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if i == (len(parts) - 1): 299926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) current_map[part] = test_dict 300926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) break 301926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if part not in current_map: 302926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) current_map[part] = {} 303926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) current_map = current_map[part] 304926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 305926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['tests'] = tests 30681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) # FIXME: Remove this. It is redundant with results['num_failures_by_type']. 307926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['num_passes'] = num_passes 308926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['num_flaky'] = num_flaky 30981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) # FIXME: Remove this. It is redundant with results['num_failures_by_type']. 310926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['num_regressions'] = num_regressions 311926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['interrupted'] = initial_results.interrupted # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?) 312926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['layout_tests_dir'] = port_obj.layout_tests_dir() 313926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['has_wdiff'] = port_obj.wdiff_available() 314926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['has_pretty_patch'] = port_obj.pretty_patch_available() 315926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) results['pixel_tests_enabled'] = port_obj.get_option('pixel_tests') 31681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) results['seconds_since_epoch'] = int(time.time()) 31781a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) results['build_number'] = port_obj.get_option('build_number') 31881a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) results['builder_name'] = port_obj.get_option('builder_name') 319926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 32076c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) # Don't do this by default since it takes >100ms. 32176c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) # It's only used for uploading data to the flakiness dashboard. 32276c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) results['chromium_revision'] = '' 32376c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) results['blink_revision'] = '' 32476c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) if port_obj.get_option('builder_name'): 32576c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) for (name, path) in port_obj.repository_paths(): 32676c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) scm = port_obj.host.scm_for_path(path) 32776c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) if scm: 3287242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci if name.lower() == 'chromium': 3297242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci rev = _chromium_commit_position(scm, path) 3307242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci else: 3317242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci rev = scm.svn_revision(path) 33276c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) if rev: 33376c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) results[name.lower() + '_revision'] = rev 33476c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) else: 33576c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) _log.warn('Failed to determine svn revision for %s, ' 33676c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) 'leaving "%s_revision" key blank in full_results.json.' 33776c265b59aa821ccbf8c75ab2bb0d036e97d2956Torne (Richard Coles) % (path, name)) 338926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 339926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return results 340