1d0825bca7fe65beaee391d30da42e937db621564Steve Block#!/usr/bin/env python 25e2bc6953fe6923165b8a5d7679939693a1d58d6Steve Block# Copyright (C) 2010 Google Inc. All rights reserved. 3a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 4d0825bca7fe65beaee391d30da42e937db621564Steve Block# 5d0825bca7fe65beaee391d30da42e937db621564Steve Block# Redistribution and use in source and binary forms, with or without 6d0825bca7fe65beaee391d30da42e937db621564Steve Block# modification, are permitted provided that the following conditions are 7d0825bca7fe65beaee391d30da42e937db621564Steve Block# met: 8d0825bca7fe65beaee391d30da42e937db621564Steve Block# 9d0825bca7fe65beaee391d30da42e937db621564Steve Block# * Redistributions of source code must retain the above copyright 10d0825bca7fe65beaee391d30da42e937db621564Steve Block# notice, this list of conditions and the following disclaimer. 11d0825bca7fe65beaee391d30da42e937db621564Steve Block# * Redistributions in binary form must reproduce the above 12d0825bca7fe65beaee391d30da42e937db621564Steve Block# copyright notice, this list of conditions and the following disclaimer 13d0825bca7fe65beaee391d30da42e937db621564Steve Block# in the documentation and/or other materials provided with the 14d0825bca7fe65beaee391d30da42e937db621564Steve Block# distribution. 155e2bc6953fe6923165b8a5d7679939693a1d58d6Steve Block# * Neither the name of Google Inc. nor the names of its 16d0825bca7fe65beaee391d30da42e937db621564Steve Block# contributors may be used to endorse or promote products derived from 17d0825bca7fe65beaee391d30da42e937db621564Steve Block# this software without specific prior written permission. 18d0825bca7fe65beaee391d30da42e937db621564Steve Block# 19d0825bca7fe65beaee391d30da42e937db621564Steve Block# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20d0825bca7fe65beaee391d30da42e937db621564Steve Block# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21d0825bca7fe65beaee391d30da42e937db621564Steve Block# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22d0825bca7fe65beaee391d30da42e937db621564Steve Block# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23d0825bca7fe65beaee391d30da42e937db621564Steve Block# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24d0825bca7fe65beaee391d30da42e937db621564Steve Block# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25d0825bca7fe65beaee391d30da42e937db621564Steve Block# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26d0825bca7fe65beaee391d30da42e937db621564Steve Block# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27d0825bca7fe65beaee391d30da42e937db621564Steve Block# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28d0825bca7fe65beaee391d30da42e937db621564Steve Block# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29d0825bca7fe65beaee391d30da42e937db621564Steve Block# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30d0825bca7fe65beaee391d30da42e937db621564Steve Block 31f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch""" 32d0825bca7fe65beaee391d30da42e937db621564Steve BlockThe TestRunner class runs a series of tests (TestType interface) against a set 33d0825bca7fe65beaee391d30da42e937db621564Steve Blockof test files. If a test file fails a TestType, it returns a list TestFailure 34d0825bca7fe65beaee391d30da42e937db621564Steve Blockobjects to the TestRunner. The TestRunner then aggregates the TestFailures to 35d0825bca7fe65beaee391d30da42e937db621564Steve Blockcreate a final report. 36d0825bca7fe65beaee391d30da42e937db621564Steve Block""" 37d0825bca7fe65beaee391d30da42e937db621564Steve Block 38dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockfrom __future__ import with_statement 39dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 4081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochimport copy 41d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport errno 42d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport logging 43d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport math 44d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport Queue 45d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport random 46d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport sys 47d0825bca7fe65beaee391d30da42e937db621564Steve Blockimport time 48f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 4981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import json_layout_results_generator 5081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import json_results_generator 5181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import printing 5281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import test_expectations 5381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import test_failures 5481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import test_results 5581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package import test_results_uploader 5681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package.result_summary import ResultSummary 5781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochfrom webkitpy.layout_tests.layout_package.test_input import TestInput 58f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 59dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockfrom webkitpy.thirdparty import simplejson 604576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wangfrom webkitpy.tool import grammar 61dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 62dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block_log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests") 63dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 64d0825bca7fe65beaee391d30da42e937db621564Steve Block# Builder base URL where we have the archived test results. 65d0825bca7fe65beaee391d30da42e937db621564Steve BlockBUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" 66d0825bca7fe65beaee391d30da42e937db621564Steve Block 67d0825bca7fe65beaee391d30da42e937db621564Steve BlockTestExpectationsFile = test_expectations.TestExpectationsFile 68d0825bca7fe65beaee391d30da42e937db621564Steve Block 69d0825bca7fe65beaee391d30da42e937db621564Steve Block 7081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochdef summarize_results(port_obj, expectations, result_summary, retry_summary, test_timings, only_unexpected): 7121939df44de1705786c545cd1bf519d47250322dBen Murdoch """Summarize any unexpected results as a dict. 7221939df44de1705786c545cd1bf519d47250322dBen Murdoch 7321939df44de1705786c545cd1bf519d47250322dBen Murdoch FIXME: split this data structure into a separate class? 7421939df44de1705786c545cd1bf519d47250322dBen Murdoch 7521939df44de1705786c545cd1bf519d47250322dBen Murdoch Args: 7621939df44de1705786c545cd1bf519d47250322dBen Murdoch port_obj: interface to port-specific hooks 7721939df44de1705786c545cd1bf519d47250322dBen Murdoch expectations: test_expectations.TestExpectations object 7821939df44de1705786c545cd1bf519d47250322dBen Murdoch result_summary: summary object from initial test runs 7921939df44de1705786c545cd1bf519d47250322dBen Murdoch retry_summary: summary object from final test run of retried tests 8081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch test_timings: a list of TestResult objects which contain test runtimes in seconds 8181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch only_unexpected: whether to return a summary only for the unexpected results 8221939df44de1705786c545cd1bf519d47250322dBen Murdoch Returns: 8321939df44de1705786c545cd1bf519d47250322dBen Murdoch A dictionary containing a summary of the unexpected results from the 8421939df44de1705786c545cd1bf519d47250322dBen Murdoch run, with the following fields: 8521939df44de1705786c545cd1bf519d47250322dBen Murdoch 'version': a version indicator (1 in this version) 8621939df44de1705786c545cd1bf519d47250322dBen Murdoch 'fixable': # of fixable tests (NOW - PASS) 8721939df44de1705786c545cd1bf519d47250322dBen Murdoch 'skipped': # of skipped tests (NOW & SKIPPED) 8821939df44de1705786c545cd1bf519d47250322dBen Murdoch 'num_regressions': # of non-flaky failures 8921939df44de1705786c545cd1bf519d47250322dBen Murdoch 'num_flaky': # of flaky failures 9021939df44de1705786c545cd1bf519d47250322dBen Murdoch 'num_passes': # of unexpected passes 9181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 'tests': a dict of tests -> {'expected': '...', 'actual': '...', 'time_ms': ...} 9221939df44de1705786c545cd1bf519d47250322dBen Murdoch """ 9321939df44de1705786c545cd1bf519d47250322dBen Murdoch results = {} 9421939df44de1705786c545cd1bf519d47250322dBen Murdoch results['version'] = 1 9521939df44de1705786c545cd1bf519d47250322dBen Murdoch 9681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch test_timings_map = dict((test_result.filename, test_result.test_run_time) for test_result in test_timings) 9781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 9821939df44de1705786c545cd1bf519d47250322dBen Murdoch tbe = result_summary.tests_by_expectation 9921939df44de1705786c545cd1bf519d47250322dBen Murdoch tbt = result_summary.tests_by_timeline 10021939df44de1705786c545cd1bf519d47250322dBen Murdoch results['fixable'] = len(tbt[test_expectations.NOW] - 10121939df44de1705786c545cd1bf519d47250322dBen Murdoch tbe[test_expectations.PASS]) 10221939df44de1705786c545cd1bf519d47250322dBen Murdoch results['skipped'] = len(tbt[test_expectations.NOW] & 10321939df44de1705786c545cd1bf519d47250322dBen Murdoch tbe[test_expectations.SKIP]) 10421939df44de1705786c545cd1bf519d47250322dBen Murdoch 10521939df44de1705786c545cd1bf519d47250322dBen Murdoch num_passes = 0 10621939df44de1705786c545cd1bf519d47250322dBen Murdoch num_flaky = 0 10721939df44de1705786c545cd1bf519d47250322dBen Murdoch num_regressions = 0 10821939df44de1705786c545cd1bf519d47250322dBen Murdoch keywords = {} 10981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch for expecation_string, expectation_enum in TestExpectationsFile.EXPECTATIONS.iteritems(): 11081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch keywords[expectation_enum] = expecation_string.upper() 11181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 11281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch for modifier_string, modifier_enum in TestExpectationsFile.MODIFIERS.iteritems(): 11381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch keywords[modifier_enum] = modifier_string.upper() 11421939df44de1705786c545cd1bf519d47250322dBen Murdoch 11521939df44de1705786c545cd1bf519d47250322dBen Murdoch tests = {} 11681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch original_results = result_summary.unexpected_results if only_unexpected else result_summary.results 11781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 11881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch for filename, result in original_results.iteritems(): 11921939df44de1705786c545cd1bf519d47250322dBen Murdoch # Note that if a test crashed in the original run, we ignore 12021939df44de1705786c545cd1bf519d47250322dBen Murdoch # whether or not it crashed when we retried it (if we retried it), 12121939df44de1705786c545cd1bf519d47250322dBen Murdoch # and always consider the result not flaky. 12221939df44de1705786c545cd1bf519d47250322dBen Murdoch test = port_obj.relative_test_filename(filename) 12321939df44de1705786c545cd1bf519d47250322dBen Murdoch expected = expectations.get_expectations_string(filename) 12481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch result_type = result.type 12581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch actual = [keywords[result_type]] 12621939df44de1705786c545cd1bf519d47250322dBen Murdoch 12781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if result_type == test_expectations.PASS: 12821939df44de1705786c545cd1bf519d47250322dBen Murdoch num_passes += 1 12981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch elif result_type == test_expectations.CRASH: 13021939df44de1705786c545cd1bf519d47250322dBen Murdoch num_regressions += 1 13181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch elif filename in result_summary.unexpected_results: 13221939df44de1705786c545cd1bf519d47250322dBen Murdoch if filename not in retry_summary.unexpected_results: 13381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch actual.extend(expectations.get_expectations_string(filename).split(" ")) 13421939df44de1705786c545cd1bf519d47250322dBen Murdoch num_flaky += 1 13521939df44de1705786c545cd1bf519d47250322dBen Murdoch else: 13681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch retry_result_type = retry_summary.unexpected_results[filename].type 13781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if result_type != retry_result_type: 13881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch actual.append(keywords[retry_result_type]) 13921939df44de1705786c545cd1bf519d47250322dBen Murdoch num_flaky += 1 14021939df44de1705786c545cd1bf519d47250322dBen Murdoch else: 14121939df44de1705786c545cd1bf519d47250322dBen Murdoch num_regressions += 1 14221939df44de1705786c545cd1bf519d47250322dBen Murdoch 14321939df44de1705786c545cd1bf519d47250322dBen Murdoch tests[test] = {} 14421939df44de1705786c545cd1bf519d47250322dBen Murdoch tests[test]['expected'] = expected 14521939df44de1705786c545cd1bf519d47250322dBen Murdoch tests[test]['actual'] = " ".join(actual) 1462daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch # FIXME: Set this correctly once https://webkit.org/b/37739 is fixed 1472daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch # and only set it if there actually is stderr data. 1482daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['has_stderr'] = False 1492daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 1502daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch failure_types = [type(f) for f in result.failures] 1512daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if test_failures.FailureMissingAudio in failure_types: 1522daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['is_missing_audio'] = True 1532daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 1542daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if test_failures.FailureReftestMismatch in failure_types: 1552daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['is_reftest'] = True 1562daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 1572daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch for f in result.failures: 1582daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if 'is_reftest' in result.failures: 1592daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['is_reftest'] = True 1602daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 1612daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if test_failures.FailureReftestMismatchDidNotOccur in failure_types: 1622daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['is_mismatch_reftest'] = True 1632daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 1642daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if test_failures.FailureMissingResult in failure_types: 1652daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['is_missing_text'] = True 1662daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 1672daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types: 1682daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests[test]['is_missing_image'] = True 16921939df44de1705786c545cd1bf519d47250322dBen Murdoch 17081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if filename in test_timings_map: 17181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch time_seconds = test_timings_map[filename] 17281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch tests[test]['time_ms'] = int(1000 * time_seconds) 17381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 17421939df44de1705786c545cd1bf519d47250322dBen Murdoch results['tests'] = tests 17521939df44de1705786c545cd1bf519d47250322dBen Murdoch results['num_passes'] = num_passes 17621939df44de1705786c545cd1bf519d47250322dBen Murdoch results['num_flaky'] = num_flaky 17721939df44de1705786c545cd1bf519d47250322dBen Murdoch results['num_regressions'] = num_regressions 1782daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch # FIXME: If non-chromium ports start using an expectations file, 1792daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch # we should make this check more robust. 1802daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch results['uses_expectations_file'] = port_obj.name().find('chromium') != -1 1812daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch results['layout_tests_dir'] = port_obj.layout_tests_dir() 1822daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch results['has_wdiff'] = port_obj.wdiff_available() 1832daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch results['has_pretty_patch'] = port_obj.pretty_patch_available() 18421939df44de1705786c545cd1bf519d47250322dBen Murdoch 18521939df44de1705786c545cd1bf519d47250322dBen Murdoch return results 18621939df44de1705786c545cd1bf519d47250322dBen Murdoch 18721939df44de1705786c545cd1bf519d47250322dBen Murdoch 188f05b935882198ccf7d81675736e3aeb089c5113aBen Murdochclass TestRunInterruptedException(Exception): 189f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch """Raised when a test run should be stopped immediately.""" 190f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch def __init__(self, reason): 191f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.reason = reason 192f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 19381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def __reduce__(self): 19481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch return self.__class__, (self.reason,) 19581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 196f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 197d0825bca7fe65beaee391d30da42e937db621564Steve Blockclass TestRunner: 198d0825bca7fe65beaee391d30da42e937db621564Steve Block """A class for managing running a series of tests on a series of layout 199d0825bca7fe65beaee391d30da42e937db621564Steve Block test files.""" 200d0825bca7fe65beaee391d30da42e937db621564Steve Block 201d0825bca7fe65beaee391d30da42e937db621564Steve Block 202d0825bca7fe65beaee391d30da42e937db621564Steve Block # The per-test timeout in milliseconds, if no --time-out-ms option was 203d0825bca7fe65beaee391d30da42e937db621564Steve Block # given to run_webkit_tests. This should correspond to the default timeout 204dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block # in DumpRenderTree. 205d0825bca7fe65beaee391d30da42e937db621564Steve Block DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 206d0825bca7fe65beaee391d30da42e937db621564Steve Block 207f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch def __init__(self, port, options, printer): 208d0825bca7fe65beaee391d30da42e937db621564Steve Block """Initialize test runner data structures. 209d0825bca7fe65beaee391d30da42e937db621564Steve Block 210d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 2118a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block port: an object implementing port-specific 212d0825bca7fe65beaee391d30da42e937db621564Steve Block options: a dictionary of command line options 21321939df44de1705786c545cd1bf519d47250322dBen Murdoch printer: a Printer object to record updates to. 214d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 2158a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block self._port = port 216ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self._fs = port._filesystem 217d0825bca7fe65beaee391d30da42e937db621564Steve Block self._options = options 21821939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer = printer 219f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self._message_broker = None 220d0825bca7fe65beaee391d30da42e937db621564Steve Block 221ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.HTTP_SUBDIR = self._fs.join('', 'http', '') 222ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.WEBSOCKET_SUBDIR = self._fs.join('', 'websocket', '') 223ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.LAYOUT_TESTS_DIRECTORY = "LayoutTests" + self._fs.sep 224ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 225ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 226d0825bca7fe65beaee391d30da42e937db621564Steve Block # disable wss server. need to install pyOpenSSL on buildbots. 227d0825bca7fe65beaee391d30da42e937db621564Steve Block # self._websocket_secure_server = websocket_server.PyWebSocket( 228d0825bca7fe65beaee391d30da42e937db621564Steve Block # options.results_directory, use_tls=True, port=9323) 229d0825bca7fe65beaee391d30da42e937db621564Steve Block 230d0825bca7fe65beaee391d30da42e937db621564Steve Block # a set of test files, and the same tests as a list 231d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files = set() 232d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files_list = None 233d0825bca7fe65beaee391d30da42e937db621564Steve Block self._result_queue = Queue.Queue() 23421939df44de1705786c545cd1bf519d47250322dBen Murdoch self._retrying = False 2352daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._results_directory = self._port.results_directory() 236d0825bca7fe65beaee391d30da42e937db621564Steve Block 2375ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def collect_tests(self, args, last_unexpected_results): 238d0825bca7fe65beaee391d30da42e937db621564Steve Block """Find all the files to test. 239d0825bca7fe65beaee391d30da42e937db621564Steve Block 240d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 2415ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen args: list of test arguments from the command line 2425ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen last_unexpected_results: list of unexpected results to retest, if any 2435ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2445ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen """ 245ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch paths = self._strip_test_dir_prefixes(args) 2465ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen paths += last_unexpected_results 2475ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if self._options.test_list: 248ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch paths += self._strip_test_dir_prefixes(read_test_files(self._fs, self._options.test_list)) 249bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self._test_files = self._port.tests(paths) 250d0825bca7fe65beaee391d30da42e937db621564Steve Block 251ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def _strip_test_dir_prefixes(self, paths): 252ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch return [self._strip_test_dir_prefix(path) for path in paths if path] 253ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 25428040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu def _strip_test_dir_prefix(self, path): 255ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch if path.startswith(self.LAYOUT_TESTS_DIRECTORY): 256ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch return path[len(self.LAYOUT_TESTS_DIRECTORY):] 25728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu return path 25828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 2595ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def lint(self): 260f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch lint_failed = False 2612fc2651226baac27029e38c9d6ef883fa32084dbSteve Block for test_configuration in self._port.all_test_configurations(): 262f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch try: 2632fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.lint_expectations(test_configuration) 264f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch except test_expectations.ParseError: 265f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch lint_failed = True 2662fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._printer.write("") 267f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 268f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch if lint_failed: 269f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch _log.error("Lint failed.") 270f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch return -1 271f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 272f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch _log.info("Lint succeeded.") 2735ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen return 0 2745ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2752fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def lint_expectations(self, config): 2762fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port = self._port 2772fc2651226baac27029e38c9d6ef883fa32084dbSteve Block test_expectations.TestExpectations( 2782fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port, 2792fc2651226baac27029e38c9d6ef883fa32084dbSteve Block None, 2802fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port.test_expectations(), 2812fc2651226baac27029e38c9d6ef883fa32084dbSteve Block config, 2822fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._options.lint_test_files, 2832fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port.test_expectations_overrides()) 2842fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 2852fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def parse_expectations(self): 286d0825bca7fe65beaee391d30da42e937db621564Steve Block """Parse the expectations from the test_list files and return a data 287d0825bca7fe65beaee391d30da42e937db621564Steve Block structure holding them. Throws an error if the test_list files have 288d0825bca7fe65beaee391d30da42e937db621564Steve Block invalid syntax.""" 2892fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port = self._port 290f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self._expectations = test_expectations.TestExpectations( 2912fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port, 2922fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._test_files, 2932fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port.test_expectations(), 2942fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port.test_configuration(), 2952fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._options.lint_test_files, 2962fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port.test_expectations_overrides()) 297d0825bca7fe65beaee391d30da42e937db621564Steve Block 298cad810f21b803229eb11403f9209855525a25d57Steve Block # FIXME: This method is way too long and needs to be broken into pieces. 29921939df44de1705786c545cd1bf519d47250322dBen Murdoch def prepare_lists_and_print_output(self): 300d0825bca7fe65beaee391d30da42e937db621564Steve Block """Create appropriate subsets of test lists and returns a 301d0825bca7fe65beaee391d30da42e937db621564Steve Block ResultSummary object. Also prints expected test counts. 302d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 303d0825bca7fe65beaee391d30da42e937db621564Steve Block 304d0825bca7fe65beaee391d30da42e937db621564Steve Block # Remove skipped - both fixable and ignored - files from the 305d0825bca7fe65beaee391d30da42e937db621564Steve Block # top-level list of files to test. 306d0825bca7fe65beaee391d30da42e937db621564Steve Block num_all_test_files = len(self._test_files) 30721939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_expected("Found: %d tests" % 30821939df44de1705786c545cd1bf519d47250322dBen Murdoch (len(self._test_files))) 30921939df44de1705786c545cd1bf519d47250322dBen Murdoch if not num_all_test_files: 3105ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen _log.critical('No tests to run.') 3115ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen return None 31221939df44de1705786c545cd1bf519d47250322dBen Murdoch 313d0825bca7fe65beaee391d30da42e937db621564Steve Block skipped = set() 314d0825bca7fe65beaee391d30da42e937db621564Steve Block if num_all_test_files > 1 and not self._options.force: 315d0825bca7fe65beaee391d30da42e937db621564Steve Block skipped = self._expectations.get_tests_with_result_type( 316d0825bca7fe65beaee391d30da42e937db621564Steve Block test_expectations.SKIP) 317d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files -= skipped 318d0825bca7fe65beaee391d30da42e937db621564Steve Block 319d0825bca7fe65beaee391d30da42e937db621564Steve Block # Create a sorted list of test files so the subset chunk, 320d0825bca7fe65beaee391d30da42e937db621564Steve Block # if used, contains alphabetically consecutive tests. 321d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files_list = list(self._test_files) 322d0825bca7fe65beaee391d30da42e937db621564Steve Block if self._options.randomize_order: 323d0825bca7fe65beaee391d30da42e937db621564Steve Block random.shuffle(self._test_files_list) 324d0825bca7fe65beaee391d30da42e937db621564Steve Block else: 325d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files_list.sort() 326d0825bca7fe65beaee391d30da42e937db621564Steve Block 327d0825bca7fe65beaee391d30da42e937db621564Steve Block # If the user specifies they just want to run a subset of the tests, 328d0825bca7fe65beaee391d30da42e937db621564Steve Block # just grab a subset of the non-skipped tests. 329d0825bca7fe65beaee391d30da42e937db621564Steve Block if self._options.run_chunk or self._options.run_part: 330d0825bca7fe65beaee391d30da42e937db621564Steve Block chunk_value = self._options.run_chunk or self._options.run_part 331d0825bca7fe65beaee391d30da42e937db621564Steve Block test_files = self._test_files_list 332d0825bca7fe65beaee391d30da42e937db621564Steve Block try: 333d0825bca7fe65beaee391d30da42e937db621564Steve Block (chunk_num, chunk_len) = chunk_value.split(":") 334d0825bca7fe65beaee391d30da42e937db621564Steve Block chunk_num = int(chunk_num) 335d0825bca7fe65beaee391d30da42e937db621564Steve Block assert(chunk_num >= 0) 336d0825bca7fe65beaee391d30da42e937db621564Steve Block test_size = int(chunk_len) 337d0825bca7fe65beaee391d30da42e937db621564Steve Block assert(test_size > 0) 338d0825bca7fe65beaee391d30da42e937db621564Steve Block except: 339dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block _log.critical("invalid chunk '%s'" % chunk_value) 3405abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick return None 341d0825bca7fe65beaee391d30da42e937db621564Steve Block 342d0825bca7fe65beaee391d30da42e937db621564Steve Block # Get the number of tests 343d0825bca7fe65beaee391d30da42e937db621564Steve Block num_tests = len(test_files) 344d0825bca7fe65beaee391d30da42e937db621564Steve Block 345d0825bca7fe65beaee391d30da42e937db621564Steve Block # Get the start offset of the slice. 346d0825bca7fe65beaee391d30da42e937db621564Steve Block if self._options.run_chunk: 347d0825bca7fe65beaee391d30da42e937db621564Steve Block chunk_len = test_size 348d0825bca7fe65beaee391d30da42e937db621564Steve Block # In this case chunk_num can be really large. We need 349d0825bca7fe65beaee391d30da42e937db621564Steve Block # to make the slave fit in the current number of tests. 350d0825bca7fe65beaee391d30da42e937db621564Steve Block slice_start = (chunk_num * chunk_len) % num_tests 351d0825bca7fe65beaee391d30da42e937db621564Steve Block else: 352d0825bca7fe65beaee391d30da42e937db621564Steve Block # Validate the data. 353d0825bca7fe65beaee391d30da42e937db621564Steve Block assert(test_size <= num_tests) 354d0825bca7fe65beaee391d30da42e937db621564Steve Block assert(chunk_num <= test_size) 355d0825bca7fe65beaee391d30da42e937db621564Steve Block 356d0825bca7fe65beaee391d30da42e937db621564Steve Block # To count the chunk_len, and make sure we don't skip 357d0825bca7fe65beaee391d30da42e937db621564Steve Block # some tests, we round to the next value that fits exactly 358d0825bca7fe65beaee391d30da42e937db621564Steve Block # all the parts. 359d0825bca7fe65beaee391d30da42e937db621564Steve Block rounded_tests = num_tests 360d0825bca7fe65beaee391d30da42e937db621564Steve Block if rounded_tests % test_size != 0: 361d0825bca7fe65beaee391d30da42e937db621564Steve Block rounded_tests = (num_tests + test_size - 362d0825bca7fe65beaee391d30da42e937db621564Steve Block (num_tests % test_size)) 363d0825bca7fe65beaee391d30da42e937db621564Steve Block 364d0825bca7fe65beaee391d30da42e937db621564Steve Block chunk_len = rounded_tests / test_size 365d0825bca7fe65beaee391d30da42e937db621564Steve Block slice_start = chunk_len * (chunk_num - 1) 366d0825bca7fe65beaee391d30da42e937db621564Steve Block # It does not mind if we go over test_size. 367d0825bca7fe65beaee391d30da42e937db621564Steve Block 368d0825bca7fe65beaee391d30da42e937db621564Steve Block # Get the end offset of the slice. 369d0825bca7fe65beaee391d30da42e937db621564Steve Block slice_end = min(num_tests, slice_start + chunk_len) 370d0825bca7fe65beaee391d30da42e937db621564Steve Block 371d0825bca7fe65beaee391d30da42e937db621564Steve Block files = test_files[slice_start:slice_end] 372d0825bca7fe65beaee391d30da42e937db621564Steve Block 373d0825bca7fe65beaee391d30da42e937db621564Steve Block tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % ( 374d0825bca7fe65beaee391d30da42e937db621564Steve Block (slice_end - slice_start), slice_start, slice_end, num_tests) 37521939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_expected(tests_run_msg) 376d0825bca7fe65beaee391d30da42e937db621564Steve Block 377d0825bca7fe65beaee391d30da42e937db621564Steve Block # If we reached the end and we don't have enough tests, we run some 378d0825bca7fe65beaee391d30da42e937db621564Steve Block # from the beginning. 37928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu if slice_end - slice_start < chunk_len: 38028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu extra = chunk_len - (slice_end - slice_start) 381d0825bca7fe65beaee391d30da42e937db621564Steve Block extra_msg = (' last chunk is partial, appending [0:%d]' % 382d0825bca7fe65beaee391d30da42e937db621564Steve Block extra) 38321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_expected(extra_msg) 384d0825bca7fe65beaee391d30da42e937db621564Steve Block tests_run_msg += "\n" + extra_msg 385d0825bca7fe65beaee391d30da42e937db621564Steve Block files.extend(test_files[0:extra]) 3862daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests_run_filename = self._fs.join(self._results_directory, "tests_run.txt") 387ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self._fs.write_text_file(tests_run_filename, tests_run_msg) 388d0825bca7fe65beaee391d30da42e937db621564Steve Block 389d0825bca7fe65beaee391d30da42e937db621564Steve Block len_skip_chunk = int(len(files) * len(skipped) / 390d0825bca7fe65beaee391d30da42e937db621564Steve Block float(len(self._test_files))) 391d0825bca7fe65beaee391d30da42e937db621564Steve Block skip_chunk_list = list(skipped)[0:len_skip_chunk] 392d0825bca7fe65beaee391d30da42e937db621564Steve Block skip_chunk = set(skip_chunk_list) 393d0825bca7fe65beaee391d30da42e937db621564Steve Block 394d0825bca7fe65beaee391d30da42e937db621564Steve Block # Update expectations so that the stats are calculated correctly. 395d0825bca7fe65beaee391d30da42e937db621564Steve Block # We need to pass a list that includes the right # of skipped files 396d0825bca7fe65beaee391d30da42e937db621564Steve Block # to ParseExpectations so that ResultSummary() will get the correct 397d0825bca7fe65beaee391d30da42e937db621564Steve Block # stats. So, we add in the subset of skipped files, and then 398d0825bca7fe65beaee391d30da42e937db621564Steve Block # subtract them back out. 399d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files_list = files + skip_chunk_list 400d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files = set(self._test_files_list) 401d0825bca7fe65beaee391d30da42e937db621564Steve Block 4022fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.parse_expectations() 403d0825bca7fe65beaee391d30da42e937db621564Steve Block 404d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files = set(files) 405d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files_list = files 406d0825bca7fe65beaee391d30da42e937db621564Steve Block else: 407d0825bca7fe65beaee391d30da42e937db621564Steve Block skip_chunk = skipped 408d0825bca7fe65beaee391d30da42e937db621564Steve Block 409d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary = ResultSummary(self._expectations, 410d0825bca7fe65beaee391d30da42e937db621564Steve Block self._test_files | skip_chunk) 41121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_expected_results_of_type(result_summary, 412d0825bca7fe65beaee391d30da42e937db621564Steve Block test_expectations.PASS, "passes") 41321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_expected_results_of_type(result_summary, 414d0825bca7fe65beaee391d30da42e937db621564Steve Block test_expectations.FAIL, "failures") 41521939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_expected_results_of_type(result_summary, 416d0825bca7fe65beaee391d30da42e937db621564Steve Block test_expectations.FLAKY, "flaky") 41721939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_expected_results_of_type(result_summary, 418d0825bca7fe65beaee391d30da42e937db621564Steve Block test_expectations.SKIP, "skipped") 419d0825bca7fe65beaee391d30da42e937db621564Steve Block 420d0825bca7fe65beaee391d30da42e937db621564Steve Block if self._options.force: 42121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_expected('Running all tests, including ' 42221939df44de1705786c545cd1bf519d47250322dBen Murdoch 'skips (--force)') 423d0825bca7fe65beaee391d30da42e937db621564Steve Block else: 424d0825bca7fe65beaee391d30da42e937db621564Steve Block # Note that we don't actually run the skipped tests (they were 425d0825bca7fe65beaee391d30da42e937db621564Steve Block # subtracted out of self._test_files, above), but we stub out the 426d0825bca7fe65beaee391d30da42e937db621564Steve Block # results here so the statistics can remain accurate. 427d0825bca7fe65beaee391d30da42e937db621564Steve Block for test in skip_chunk: 428cad810f21b803229eb11403f9209855525a25d57Steve Block result = test_results.TestResult(test) 429dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block result.type = test_expectations.SKIP 430dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block result_summary.add(result, expected=True) 43121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_expected('') 432d0825bca7fe65beaee391d30da42e937db621564Steve Block 433ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch # Check to make sure we didn't filter out all of the tests. 434ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch if not len(self._test_files): 435ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch _log.info("All tests are being skipped") 436ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch return None 437ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 438d0825bca7fe65beaee391d30da42e937db621564Steve Block return result_summary 439d0825bca7fe65beaee391d30da42e937db621564Steve Block 440d0825bca7fe65beaee391d30da42e937db621564Steve Block def _get_dir_for_test_file(self, test_file): 441d0825bca7fe65beaee391d30da42e937db621564Steve Block """Returns the highest-level directory by which to shard the given 442d0825bca7fe65beaee391d30da42e937db621564Steve Block test file.""" 443ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch index = test_file.rfind(self._fs.sep + self.LAYOUT_TESTS_DIRECTORY) 444d0825bca7fe65beaee391d30da42e937db621564Steve Block 445ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch test_file = test_file[index + len(self.LAYOUT_TESTS_DIRECTORY):] 446ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch test_file_parts = test_file.split(self._fs.sep, 1) 447d0825bca7fe65beaee391d30da42e937db621564Steve Block directory = test_file_parts[0] 448d0825bca7fe65beaee391d30da42e937db621564Steve Block test_file = test_file_parts[1] 449d0825bca7fe65beaee391d30da42e937db621564Steve Block 450d0825bca7fe65beaee391d30da42e937db621564Steve Block # The http tests are very stable on mac/linux. 451d0825bca7fe65beaee391d30da42e937db621564Steve Block # TODO(ojan): Make the http server on Windows be apache so we can 452d0825bca7fe65beaee391d30da42e937db621564Steve Block # turn shard the http tests there as well. Switching to apache is 453d0825bca7fe65beaee391d30da42e937db621564Steve Block # what made them stable on linux/mac. 454d0825bca7fe65beaee391d30da42e937db621564Steve Block return_value = directory 455d0825bca7fe65beaee391d30da42e937db621564Steve Block while ((directory != 'http' or sys.platform in ('darwin', 'linux2')) 456ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch and test_file.find(self._fs.sep) >= 0): 457ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch test_file_parts = test_file.split(self._fs.sep, 1) 458d0825bca7fe65beaee391d30da42e937db621564Steve Block directory = test_file_parts[0] 459ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch return_value = self._fs.join(return_value, directory) 460d0825bca7fe65beaee391d30da42e937db621564Steve Block test_file = test_file_parts[1] 461d0825bca7fe65beaee391d30da42e937db621564Steve Block 462d0825bca7fe65beaee391d30da42e937db621564Steve Block return return_value 463d0825bca7fe65beaee391d30da42e937db621564Steve Block 4646b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner def _get_test_input_for_file(self, test_file): 4656b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner """Returns the appropriate TestInput object for the file. Mostly this 466d0825bca7fe65beaee391d30da42e937db621564Steve Block is used for looking up the timeout value (in ms) to use for the given 467d0825bca7fe65beaee391d30da42e937db621564Steve Block test.""" 4684576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang if self._test_is_slow(test_file): 4696b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner return TestInput(test_file, self._options.slow_time_out_ms) 4706b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner return TestInput(test_file, self._options.time_out_ms) 471d0825bca7fe65beaee391d30da42e937db621564Steve Block 472a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch def _test_requires_lock(self, test_file): 473a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch """Return True if the test needs to be locked when 474a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch running multiple copies of NRWTs.""" 475ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch split_path = test_file.split(self._port._filesystem.sep) 476a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch return 'http' in split_path or 'websocket' in split_path 477a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 4784576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def _test_is_slow(self, test_file): 4794576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang return self._expectations.has_modifier(test_file, 4804576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang test_expectations.SLOW) 481d0825bca7fe65beaee391d30da42e937db621564Steve Block 4824576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def _shard_tests(self, test_files, use_real_shards): 4834576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang """Groups tests into batches. 4844576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang This helps ensure that tests that depend on each other (aka bad tests!) 4854576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang continue to run together as most cross-tests dependencies tend to 486f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch occur within the same directory. If use_real_shards is False, we 4874576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang put each (non-HTTP/websocket) test into its own shard for maximum 4884576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang concurrency instead of trying to do any sort of real sharding. 489d0825bca7fe65beaee391d30da42e937db621564Steve Block 490d0825bca7fe65beaee391d30da42e937db621564Steve Block Return: 4914576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang A list of lists of TestInput objects. 492d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 4934576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang # FIXME: when we added http locking, we changed how this works such 4944576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang # that we always lump all of the HTTP threads into a single shard. 4954576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang # That will slow down experimental-fully-parallel, but it's unclear 4964576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang # what the best alternative is completely revamping how we track 4974576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang # when to grab the lock. 498d0825bca7fe65beaee391d30da42e937db621564Steve Block 499a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch test_lists = [] 500a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch tests_to_http_lock = [] 5014576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang if not use_real_shards: 502d0825bca7fe65beaee391d30da42e937db621564Steve Block for test_file in test_files: 5036b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner test_input = self._get_test_input_for_file(test_file) 504a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch if self._test_requires_lock(test_file): 5056b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner tests_to_http_lock.append(test_input) 506a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch else: 5076b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner test_lists.append((".", [test_input])) 508a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch else: 509a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch tests_by_dir = {} 510a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch for test_file in test_files: 511a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch directory = self._get_dir_for_test_file(test_file) 5126b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner test_input = self._get_test_input_for_file(test_file) 513a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch if self._test_requires_lock(test_file): 5146b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner tests_to_http_lock.append(test_input) 515a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch else: 516a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch tests_by_dir.setdefault(directory, []) 5176b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner tests_by_dir[directory].append(test_input) 518a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # Sort by the number of tests in the dir so that the ones with the 519a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # most tests get run first in order to maximize parallelization. 520a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # Number of tests is a good enough, but not perfect, approximation 521a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # of how long that set of tests will take to run. We can't just use 522a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # a PriorityQueue until we move to Python 2.6. 523a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch for directory in tests_by_dir: 524a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch test_list = tests_by_dir[directory] 525a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch test_list_tuple = (directory, test_list) 526d0825bca7fe65beaee391d30da42e937db621564Steve Block test_lists.append(test_list_tuple) 527a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1]))) 528d0825bca7fe65beaee391d30da42e937db621564Steve Block 529d0825bca7fe65beaee391d30da42e937db621564Steve Block # Put the http tests first. There are only a couple hundred of them, 530d0825bca7fe65beaee391d30da42e937db621564Steve Block # but each http test takes a very long time to run, so sorting by the 531d0825bca7fe65beaee391d30da42e937db621564Steve Block # number of tests doesn't accurately capture how long they take to run. 532a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch if tests_to_http_lock: 533a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock)) 534d0825bca7fe65beaee391d30da42e937db621564Steve Block 5354576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang return test_lists 536d0825bca7fe65beaee391d30da42e937db621564Steve Block 537d0825bca7fe65beaee391d30da42e937db621564Steve Block def _contains_tests(self, subdir): 538dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block for test_file in self._test_files: 539d0825bca7fe65beaee391d30da42e937db621564Steve Block if test_file.find(subdir) >= 0: 540d0825bca7fe65beaee391d30da42e937db621564Steve Block return True 541d0825bca7fe65beaee391d30da42e937db621564Steve Block return False 542d0825bca7fe65beaee391d30da42e937db621564Steve Block 5432daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def _num_workers(self, num_shards): 5442daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch num_workers = min(int(self._options.child_processes), num_shards) 5452daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch driver_name = self._port.driver_name() 5462daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if num_workers == 1: 5472daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._printer.print_config("Running 1 %s over %s" % 5482daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch (driver_name, grammar.pluralize('shard', num_shards))) 5492daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch else: 5502daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._printer.print_config("Running %d %ss in parallel over %d shards" % 5512daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch (num_workers, driver_name, num_shards)) 5522daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch return num_workers 553d0825bca7fe65beaee391d30da42e937db621564Steve Block 5548a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block def _run_tests(self, file_list, result_summary): 555d0825bca7fe65beaee391d30da42e937db621564Steve Block """Runs the tests in the file_list. 556d0825bca7fe65beaee391d30da42e937db621564Steve Block 557f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch Return: A tuple (interrupted, keyboard_interrupted, thread_timings, 558f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch test_timings, individual_test_timings) 559f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch interrupted is whether the run was interrupted 560f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch keyboard_interrupted is whether the interruption was because someone 561f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch typed Ctrl^C 562d0825bca7fe65beaee391d30da42e937db621564Steve Block thread_timings is a list of dicts with the total runtime 563d0825bca7fe65beaee391d30da42e937db621564Steve Block of each thread with 'name', 'num_tests', 'total_time' properties 564d0825bca7fe65beaee391d30da42e937db621564Steve Block test_timings is a list of timings for each sharded subdirectory 565d0825bca7fe65beaee391d30da42e937db621564Steve Block of the form [time, directory_name, num_tests] 566d0825bca7fe65beaee391d30da42e937db621564Steve Block individual_test_timings is a list of run times for each test 567d0825bca7fe65beaee391d30da42e937db621564Steve Block in the form {filename:filename, test_run_time:test_run_time} 568d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: summary object to populate with the results 569d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 5702daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch raise NotImplementedError() 571f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick 5724576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def update(self): 5734576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang self.update_summary(self._current_result_summary) 574f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick 575f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick def _collect_timing_info(self, threads): 576f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick test_timings = {} 577f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick individual_test_timings = [] 578f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick thread_timings = [] 5795af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke 580d0825bca7fe65beaee391d30da42e937db621564Steve Block for thread in threads: 5815af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke thread_timings.append({'name': thread.getName(), 5825af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke 'num_tests': thread.get_num_tests(), 5835af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke 'total_time': thread.get_total_time()}) 584a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch test_timings.update(thread.get_test_group_timing_stats()) 5855af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke individual_test_timings.extend(thread.get_test_results()) 586f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick 587f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick return (thread_timings, test_timings, individual_test_timings) 588d0825bca7fe65beaee391d30da42e937db621564Steve Block 589dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block def needs_http(self): 590dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block """Returns whether the test runner needs an HTTP server.""" 591dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block return self._contains_tests(self.HTTP_SUBDIR) 592dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 593a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch def needs_websocket(self): 594a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch """Returns whether the test runner needs a WEBSOCKET server.""" 595a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch return self._contains_tests(self.WEBSOCKET_SUBDIR) 596a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 5975ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def set_up_run(self): 5985ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen """Configures the system to be ready to run tests. 5995ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6005ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen Returns a ResultSummary object if we should continue to run tests, 6015ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen or None if we should abort. 6025ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6035ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen """ 6045ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # This must be started before we check the system dependencies, 6055ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # since the helper may do things to make the setup correct. 6065ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._printer.print_update("Starting helper ...") 6075ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._port.start_helper() 6085ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6095ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # Check that the system dependencies (themes, fonts, ...) are correct. 6105ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if not self._options.nocheck_sys_deps: 6115ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._printer.print_update("Checking system dependencies ...") 6125ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if not self._port.check_sys_deps(self.needs_http()): 6135ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._port.stop_helper() 6145ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen return None 6155ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6165ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if self._options.clobber_old_results: 6175ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._clobber_old_results() 6185ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6195ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # Create the output directory if it doesn't already exist. 6202daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._port.maybe_make_directory(self._results_directory) 6215ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6225ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._port.setup_test_run() 6235ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6245ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._printer.print_update("Preparing tests ...") 6255ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen result_summary = self.prepare_lists_and_print_output() 6265ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if not result_summary: 6275ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen return None 6285ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 6295ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen return result_summary 6305ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 63121939df44de1705786c545cd1bf519d47250322dBen Murdoch def run(self, result_summary): 632d0825bca7fe65beaee391d30da42e937db621564Steve Block """Run all our tests on all our test files. 633d0825bca7fe65beaee391d30da42e937db621564Steve Block 634d0825bca7fe65beaee391d30da42e937db621564Steve Block For each test file, we run each test type. If there are any failures, 635d0825bca7fe65beaee391d30da42e937db621564Steve Block we collect them for reporting. 636d0825bca7fe65beaee391d30da42e937db621564Steve Block 637d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 638d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: a summary object tracking the test results. 639d0825bca7fe65beaee391d30da42e937db621564Steve Block 640d0825bca7fe65beaee391d30da42e937db621564Steve Block Return: 641dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block The number of unexpected results (0 == success) 642d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 6435ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # gather_test_files() must have been called first to initialize us. 6445ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # If we didn't find any files to test, we've errored out already in 6455ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # prepare_lists_and_print_output(). 6465ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen assert(len(self._test_files)) 647d0825bca7fe65beaee391d30da42e937db621564Steve Block 6485ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen start_time = time.time() 649d0825bca7fe65beaee391d30da42e937db621564Steve Block 650f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch interrupted, keyboard_interrupted, thread_timings, test_timings, \ 6515af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke individual_test_timings = ( 6528a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block self._run_tests(self._test_files_list, result_summary)) 653d0825bca7fe65beaee391d30da42e937db621564Steve Block 654d0825bca7fe65beaee391d30da42e937db621564Steve Block # We exclude the crashes from the list of results to retry, because 655d0825bca7fe65beaee391d30da42e937db621564Steve Block # we want to treat even a potentially flaky crash as an error. 656d0825bca7fe65beaee391d30da42e937db621564Steve Block failures = self._get_failures(result_summary, include_crashes=False) 657d0825bca7fe65beaee391d30da42e937db621564Steve Block retry_summary = result_summary 65821939df44de1705786c545cd1bf519d47250322dBen Murdoch while (len(failures) and self._options.retry_failures and 659f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch not self._retrying and not interrupted): 660dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block _log.info('') 66121939df44de1705786c545cd1bf519d47250322dBen Murdoch _log.info("Retrying %d unexpected failure(s) ..." % len(failures)) 662dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block _log.info('') 66321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._retrying = True 664d0825bca7fe65beaee391d30da42e937db621564Steve Block retry_summary = ResultSummary(self._expectations, failures.keys()) 6655af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke # Note that we intentionally ignore the return value here. 6668a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block self._run_tests(failures.keys(), retry_summary) 667d0825bca7fe65beaee391d30da42e937db621564Steve Block failures = self._get_failures(retry_summary, include_crashes=True) 668d0825bca7fe65beaee391d30da42e937db621564Steve Block 669d0825bca7fe65beaee391d30da42e937db621564Steve Block end_time = time.time() 670d0825bca7fe65beaee391d30da42e937db621564Steve Block 67121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_timing_statistics(end_time - start_time, 67221939df44de1705786c545cd1bf519d47250322dBen Murdoch thread_timings, test_timings, 67321939df44de1705786c545cd1bf519d47250322dBen Murdoch individual_test_timings, 67421939df44de1705786c545cd1bf519d47250322dBen Murdoch result_summary) 675d0825bca7fe65beaee391d30da42e937db621564Steve Block 67621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_result_summary(result_summary) 677d0825bca7fe65beaee391d30da42e937db621564Steve Block 678d0825bca7fe65beaee391d30da42e937db621564Steve Block sys.stdout.flush() 679d0825bca7fe65beaee391d30da42e937db621564Steve Block sys.stderr.flush() 680d0825bca7fe65beaee391d30da42e937db621564Steve Block 68121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_one_line_summary(result_summary.total, 6825af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke result_summary.expected, 6835af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke result_summary.unexpected) 684d0825bca7fe65beaee391d30da42e937db621564Steve Block 68581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch unexpected_results = summarize_results(self._port, 68681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=True) 68721939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_unexpected_results(unexpected_results) 688d0825bca7fe65beaee391d30da42e937db621564Steve Block 6892fc2651226baac27029e38c9d6ef883fa32084dbSteve Block # FIXME: remove record_results. It's just used for testing. There's no need 6902fc2651226baac27029e38c9d6ef883fa32084dbSteve Block # for it to be a commandline argument. 691f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch if (self._options.record_results and not self._options.dry_run and 69281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch not keyboard_interrupted): 6936b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # Write the same data to log files and upload generated JSON files 6946b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # to appengine server. 69581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch summarized_results = summarize_results(self._port, 69681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=False) 69781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self._upload_json_files(unexpected_results, summarized_results, result_summary, 6986b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner individual_test_timings) 6996c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen 700dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block # Write the summary to disk (results.html) and display it if requested. 701f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch if not self._options.dry_run: 7022daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._copy_results_html_file() 7032daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if self._options.show_results: 7042daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._show_results_html_file(result_summary) 705d0825bca7fe65beaee391d30da42e937db621564Steve Block 7065af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke # Now that we've completed all the processing we can, we re-raise 7075af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke # a KeyboardInterrupt if necessary so the caller can handle it. 7085af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke if keyboard_interrupted: 7095af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke raise KeyboardInterrupt 7105af96e2c7b73ebc627c6894727826a7576d31758Leon Clarke 711d0825bca7fe65beaee391d30da42e937db621564Steve Block # Ignore flaky failures and unexpected passes so we don't turn the 712d0825bca7fe65beaee391d30da42e937db621564Steve Block # bot red for those. 713d0825bca7fe65beaee391d30da42e937db621564Steve Block return unexpected_results['num_regressions'] 714d0825bca7fe65beaee391d30da42e937db621564Steve Block 7155ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def clean_up_run(self): 7165ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen """Restores the system after we're done running tests.""" 7175ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 7185ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen _log.debug("flushing stdout") 7195ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen sys.stdout.flush() 7205ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen _log.debug("flushing stderr") 7215ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen sys.stderr.flush() 7225ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen _log.debug("stopping helper") 7235ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._port.stop_helper() 7245ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 725d0825bca7fe65beaee391d30da42e937db621564Steve Block def update_summary(self, result_summary): 726dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block """Update the summary and print results with any completed tests.""" 727d0825bca7fe65beaee391d30da42e937db621564Steve Block while True: 728d0825bca7fe65beaee391d30da42e937db621564Steve Block try: 729e14391e94c850b8bd03680c23b38978db68687a8John Reck result = test_results.TestResult.loads(self._result_queue.get_nowait()) 730d0825bca7fe65beaee391d30da42e937db621564Steve Block except Queue.Empty: 731d0825bca7fe65beaee391d30da42e937db621564Steve Block return 73221939df44de1705786c545cd1bf519d47250322dBen Murdoch 7332fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._update_summary_with_result(result_summary, result) 7342fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 7352fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def _update_summary_with_result(self, result_summary, result): 7362fc2651226baac27029e38c9d6ef883fa32084dbSteve Block expected = self._expectations.matches_an_expected_result( 7372fc2651226baac27029e38c9d6ef883fa32084dbSteve Block result.filename, result.type, self._options.pixel_tests) 7382fc2651226baac27029e38c9d6ef883fa32084dbSteve Block result_summary.add(result, expected) 7392fc2651226baac27029e38c9d6ef883fa32084dbSteve Block exp_str = self._expectations.get_expectations_string( 7402fc2651226baac27029e38c9d6ef883fa32084dbSteve Block result.filename) 7412fc2651226baac27029e38c9d6ef883fa32084dbSteve Block got_str = self._expectations.expectation_to_string(result.type) 7422fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._printer.print_test_result(result, expected, exp_str, got_str) 7432fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._printer.print_progress(result_summary, self._retrying, 7442fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._test_files_list) 7452fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 7462fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def interrupt_if_at_failure_limit(limit, count, message): 7472fc2651226baac27029e38c9d6ef883fa32084dbSteve Block if limit and count >= limit: 7482fc2651226baac27029e38c9d6ef883fa32084dbSteve Block raise TestRunInterruptedException(message % count) 7492fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 7502fc2651226baac27029e38c9d6ef883fa32084dbSteve Block interrupt_if_at_failure_limit( 7512fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._options.exit_after_n_failures, 7522fc2651226baac27029e38c9d6ef883fa32084dbSteve Block result_summary.unexpected_failures, 7532fc2651226baac27029e38c9d6ef883fa32084dbSteve Block "Aborting run since %d failures were reached") 7542fc2651226baac27029e38c9d6ef883fa32084dbSteve Block interrupt_if_at_failure_limit( 7552fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self._options.exit_after_n_crashes_or_timeouts, 7562fc2651226baac27029e38c9d6ef883fa32084dbSteve Block result_summary.unexpected_crashes_or_timeouts, 7572fc2651226baac27029e38c9d6ef883fa32084dbSteve Block "Aborting run since %d crashes or timeouts were reached") 758f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 7595ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def _clobber_old_results(self): 7605ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # Just clobber the actual test results directories since the other 7615ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # files in the results directory are explicitly used for cross-run 7625ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # tracking. 7635ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._printer.print_update("Clobbering old results in %s" % 7642daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._results_directory) 7655ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen layout_tests_dir = self._port.layout_tests_dir() 766bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen possible_dirs = self._port.test_dirs() 7675ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen for dirname in possible_dirs: 768ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch if self._fs.isdir(self._fs.join(layout_tests_dir, dirname)): 7692daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._fs.rmtree(self._fs.join(self._results_directory, dirname)) 7705ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 771d0825bca7fe65beaee391d30da42e937db621564Steve Block def _get_failures(self, result_summary, include_crashes): 772d0825bca7fe65beaee391d30da42e937db621564Steve Block """Filters a dict of results and returns only the failures. 773d0825bca7fe65beaee391d30da42e937db621564Steve Block 774d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 775d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: the results of the test run 776d0825bca7fe65beaee391d30da42e937db621564Steve Block include_crashes: whether crashes are included in the output. 777d0825bca7fe65beaee391d30da42e937db621564Steve Block We use False when finding the list of failures to retry 778d0825bca7fe65beaee391d30da42e937db621564Steve Block to see if the results were flaky. Although the crashes may also be 779d0825bca7fe65beaee391d30da42e937db621564Steve Block flaky, we treat them as if they aren't so that they're not ignored. 780d0825bca7fe65beaee391d30da42e937db621564Steve Block Returns: 781d0825bca7fe65beaee391d30da42e937db621564Steve Block a dict of files -> results 782d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 783d0825bca7fe65beaee391d30da42e937db621564Steve Block failed_results = {} 784d0825bca7fe65beaee391d30da42e937db621564Steve Block for test, result in result_summary.unexpected_results.iteritems(): 78581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if (result.type == test_expectations.PASS or 78681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch result.type == test_expectations.CRASH and not include_crashes): 787d0825bca7fe65beaee391d30da42e937db621564Steve Block continue 78881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch failed_results[test] = result.type 789d0825bca7fe65beaee391d30da42e937db621564Steve Block 790d0825bca7fe65beaee391d30da42e937db621564Steve Block return failed_results 791d0825bca7fe65beaee391d30da42e937db621564Steve Block 79281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def _char_for_result(self, result): 79381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch result = result.lower() 79481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if result in TestExpectationsFile.EXPECTATIONS: 79581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch result_enum_value = TestExpectationsFile.EXPECTATIONS[result] 79681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch else: 79781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch result_enum_value = TestExpectationsFile.MODIFIERS[result] 79881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch return json_layout_results_generator.JSONLayoutResultsGenerator.FAILURE_TO_CHAR[result_enum_value] 79981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 80081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def _upload_json_files(self, unexpected_results, summarized_results, result_summary, 8012fc2651226baac27029e38c9d6ef883fa32084dbSteve Block individual_test_timings): 802d0825bca7fe65beaee391d30da42e937db621564Steve Block """Writes the results of the test run as JSON files into the results 8036b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner dir and upload the files to the appengine server. 804d0825bca7fe65beaee391d30da42e937db621564Steve Block 805d0825bca7fe65beaee391d30da42e937db621564Steve Block There are three different files written into the results dir: 806d0825bca7fe65beaee391d30da42e937db621564Steve Block unexpected_results.json: A short list of any unexpected results. 807d0825bca7fe65beaee391d30da42e937db621564Steve Block This is used by the buildbots to display results. 808d0825bca7fe65beaee391d30da42e937db621564Steve Block expectations.json: This is used by the flakiness dashboard. 809d0825bca7fe65beaee391d30da42e937db621564Steve Block results.json: A full list of the results - used by the flakiness 810d0825bca7fe65beaee391d30da42e937db621564Steve Block dashboard and the aggregate results dashboard. 811d0825bca7fe65beaee391d30da42e937db621564Steve Block 812d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 813d0825bca7fe65beaee391d30da42e937db621564Steve Block unexpected_results: dict of unexpected results 81481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch summarized_results: dict of results 815d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: full summary object 816d0825bca7fe65beaee391d30da42e937db621564Steve Block individual_test_timings: list of test times (used by the flakiness 817d0825bca7fe65beaee391d30da42e937db621564Steve Block dashboard). 818d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 8192daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch _log.debug("Writing JSON files in %s." % self._results_directory) 82081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 8212daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch unexpected_json_path = self._fs.join(self._results_directory, "unexpected_results.json") 82281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch json_results_generator.write_json(self._fs, unexpected_results, unexpected_json_path) 82381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 8242daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch full_results_path = self._fs.join(self._results_directory, "full_results.json") 82581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch json_results_generator.write_json(self._fs, summarized_results, full_results_path) 826d0825bca7fe65beaee391d30da42e937db621564Steve Block 827d0825bca7fe65beaee391d30da42e937db621564Steve Block # Write a json file of the test_expectations.txt file for the layout 828d0825bca7fe65beaee391d30da42e937db621564Steve Block # tests dashboard. 8292daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch expectations_path = self._fs.join(self._results_directory, "expectations.json") 830d0825bca7fe65beaee391d30da42e937db621564Steve Block expectations_json = \ 831d0825bca7fe65beaee391d30da42e937db621564Steve Block self._expectations.get_expectations_json_for_all_platforms() 832ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self._fs.write_text_file(expectations_path, 833ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch u"ADD_EXPECTATIONS(%s);" % expectations_json) 834d0825bca7fe65beaee391d30da42e937db621564Steve Block 8356b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner generator = json_layout_results_generator.JSONLayoutResultsGenerator( 8368a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block self._port, self._options.builder_name, self._options.build_name, 8372daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._options.build_number, self._results_directory, 838d0825bca7fe65beaee391d30da42e937db621564Steve Block BUILDER_BASE_URL, individual_test_timings, 839dd8bb3de4f353a81954234999f1fea748aee2ea9Ben Murdoch self._expectations, result_summary, self._test_files_list, 8406b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner self._options.test_results_server, 8416b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner "layout-tests", 8426b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner self._options.master_name) 843d0825bca7fe65beaee391d30da42e937db621564Steve Block 844dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block _log.debug("Finished writing JSON files.") 845d0825bca7fe65beaee391d30da42e937db621564Steve Block 84681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch json_files = ["expectations.json", "incremental_results.json", "full_results.json"] 8476c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen 8486b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner generator.upload_json_files(json_files) 8496c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen 8505ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def _print_config(self): 8515ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen """Prints the configuration for the test run.""" 8525ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p = self._printer 8535ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("Using port '%s'" % self._port.name()) 8542fc2651226baac27029e38c9d6ef883fa32084dbSteve Block p.print_config("Test configuration: %s" % self._port.test_configuration()) 8552daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch p.print_config("Placing test results in %s" % self._results_directory) 8565ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if self._options.new_baseline: 8575ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("Placing new baselines in %s" % 8585ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._port.baseline_path()) 8595ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("Using %s build" % self._options.configuration) 8605ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if self._options.pixel_tests: 8615ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("Pixel tests enabled") 8625ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen else: 8635ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("Pixel tests disabled") 8645ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 8655ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("Regular timeout: %s, slow test timeout: %s" % 8665ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen (self._options.time_out_ms, 8675ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self._options.slow_time_out_ms)) 8685ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 8694576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang p.print_config('Command line: ' + 8704576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang ' '.join(self._port.driver_cmd_line())) 8714576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang p.print_config("Worker model: %s" % self._options.worker_model) 8725ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen p.print_config("") 8735ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 87421939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_expected_results_of_type(self, result_summary, 875d0825bca7fe65beaee391d30da42e937db621564Steve Block result_type, result_type_str): 876d0825bca7fe65beaee391d30da42e937db621564Steve Block """Print the number of the tests in a given result class. 877d0825bca7fe65beaee391d30da42e937db621564Steve Block 878d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 879d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary - the object containing all the results to report on 880d0825bca7fe65beaee391d30da42e937db621564Steve Block result_type - the particular result type to report in the summary. 881d0825bca7fe65beaee391d30da42e937db621564Steve Block result_type_str - a string description of the result_type. 882d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 883d0825bca7fe65beaee391d30da42e937db621564Steve Block tests = self._expectations.get_tests_with_result_type(result_type) 884d0825bca7fe65beaee391d30da42e937db621564Steve Block now = result_summary.tests_by_timeline[test_expectations.NOW] 885d0825bca7fe65beaee391d30da42e937db621564Steve Block wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX] 886d0825bca7fe65beaee391d30da42e937db621564Steve Block 887d0825bca7fe65beaee391d30da42e937db621564Steve Block # We use a fancy format string in order to print the data out in a 888d0825bca7fe65beaee391d30da42e937db621564Steve Block # nicely-aligned table. 889e14391e94c850b8bd03680c23b38978db68687a8John Reck fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)" 890e14391e94c850b8bd03680c23b38978db68687a8John Reck % (self._num_digits(now), self._num_digits(wontfix))) 89121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_expected(fmtstr % 892e14391e94c850b8bd03680c23b38978db68687a8John Reck (len(tests), result_type_str, len(tests & now), len(tests & wontfix))) 893d0825bca7fe65beaee391d30da42e937db621564Steve Block 894d0825bca7fe65beaee391d30da42e937db621564Steve Block def _num_digits(self, num): 895d0825bca7fe65beaee391d30da42e937db621564Steve Block """Returns the number of digits needed to represent the length of a 896d0825bca7fe65beaee391d30da42e937db621564Steve Block sequence.""" 897d0825bca7fe65beaee391d30da42e937db621564Steve Block ndigits = 1 898d0825bca7fe65beaee391d30da42e937db621564Steve Block if len(num): 899d0825bca7fe65beaee391d30da42e937db621564Steve Block ndigits = int(math.log10(len(num))) + 1 900d0825bca7fe65beaee391d30da42e937db621564Steve Block return ndigits 901d0825bca7fe65beaee391d30da42e937db621564Steve Block 90221939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_timing_statistics(self, total_time, thread_timings, 903d0825bca7fe65beaee391d30da42e937db621564Steve Block directory_test_timings, individual_test_timings, 904d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary): 905d0825bca7fe65beaee391d30da42e937db621564Steve Block """Record timing-specific information for the test run. 906d0825bca7fe65beaee391d30da42e937db621564Steve Block 907d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 908d0825bca7fe65beaee391d30da42e937db621564Steve Block total_time: total elapsed time (in seconds) for the test run 909d0825bca7fe65beaee391d30da42e937db621564Steve Block thread_timings: wall clock time each thread ran for 910d0825bca7fe65beaee391d30da42e937db621564Steve Block directory_test_timings: timing by directory 911d0825bca7fe65beaee391d30da42e937db621564Steve Block individual_test_timings: timing by file 912d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: summary object for the test run 913d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 91421939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("Test timing:") 91521939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" %6.2f total testing time" % total_time) 91621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 91721939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("Thread timing:") 918d0825bca7fe65beaee391d30da42e937db621564Steve Block cuml_time = 0 919d0825bca7fe65beaee391d30da42e937db621564Steve Block for t in thread_timings: 92021939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" %10s: %5d tests, %6.2f secs" % 921d0825bca7fe65beaee391d30da42e937db621564Steve Block (t['name'], t['num_tests'], t['total_time'])) 922d0825bca7fe65beaee391d30da42e937db621564Steve Block cuml_time += t['total_time'] 92321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" %6.2f cumulative, %6.2f optimal" % 924dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block (cuml_time, cuml_time / int(self._options.child_processes))) 92521939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 926d0825bca7fe65beaee391d30da42e937db621564Steve Block 92721939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_aggregate_test_statistics(individual_test_timings) 92821939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_individual_test_times(individual_test_timings, 929d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary) 93021939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_directory_timings(directory_test_timings) 931d0825bca7fe65beaee391d30da42e937db621564Steve Block 93221939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_aggregate_test_statistics(self, individual_test_timings): 933d0825bca7fe65beaee391d30da42e937db621564Steve Block """Prints aggregate statistics (e.g. median, mean, etc.) for all tests. 934d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 935f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch individual_test_timings: List of TestResults for all tests. 936d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 93781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch times_for_dump_render_tree = [test_stats.test_run_time for test_stats in individual_test_timings] 93881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self._print_statistics_for_test_timings("PER TEST TIME IN TESTSHELL (seconds):", 93981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch times_for_dump_render_tree) 940d0825bca7fe65beaee391d30da42e937db621564Steve Block 94121939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_individual_test_times(self, individual_test_timings, 942d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary): 943d0825bca7fe65beaee391d30da42e937db621564Steve Block """Prints the run times for slow, timeout and crash tests. 944d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 945f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch individual_test_timings: List of TestStats for all tests. 946d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: summary object for test run 947d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 948dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block # Reverse-sort by the time spent in DumpRenderTree. 949d0825bca7fe65beaee391d30da42e937db621564Steve Block individual_test_timings.sort(lambda a, b: 950d0825bca7fe65beaee391d30da42e937db621564Steve Block cmp(b.test_run_time, a.test_run_time)) 951d0825bca7fe65beaee391d30da42e937db621564Steve Block 952d0825bca7fe65beaee391d30da42e937db621564Steve Block num_printed = 0 953d0825bca7fe65beaee391d30da42e937db621564Steve Block slow_tests = [] 954d0825bca7fe65beaee391d30da42e937db621564Steve Block timeout_or_crash_tests = [] 955d0825bca7fe65beaee391d30da42e937db621564Steve Block unexpected_slow_tests = [] 956d0825bca7fe65beaee391d30da42e937db621564Steve Block for test_tuple in individual_test_timings: 957d0825bca7fe65beaee391d30da42e937db621564Steve Block filename = test_tuple.filename 958d0825bca7fe65beaee391d30da42e937db621564Steve Block is_timeout_crash_or_slow = False 9594576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang if self._test_is_slow(filename): 960d0825bca7fe65beaee391d30da42e937db621564Steve Block is_timeout_crash_or_slow = True 961d0825bca7fe65beaee391d30da42e937db621564Steve Block slow_tests.append(test_tuple) 962d0825bca7fe65beaee391d30da42e937db621564Steve Block 963d0825bca7fe65beaee391d30da42e937db621564Steve Block if filename in result_summary.failures: 96421939df44de1705786c545cd1bf519d47250322dBen Murdoch result = result_summary.results[filename].type 965d0825bca7fe65beaee391d30da42e937db621564Steve Block if (result == test_expectations.TIMEOUT or 966d0825bca7fe65beaee391d30da42e937db621564Steve Block result == test_expectations.CRASH): 967d0825bca7fe65beaee391d30da42e937db621564Steve Block is_timeout_crash_or_slow = True 968d0825bca7fe65beaee391d30da42e937db621564Steve Block timeout_or_crash_tests.append(test_tuple) 969d0825bca7fe65beaee391d30da42e937db621564Steve Block 970d0825bca7fe65beaee391d30da42e937db621564Steve Block if (not is_timeout_crash_or_slow and 97121939df44de1705786c545cd1bf519d47250322dBen Murdoch num_printed < printing.NUM_SLOW_TESTS_TO_LOG): 972d0825bca7fe65beaee391d30da42e937db621564Steve Block num_printed = num_printed + 1 973d0825bca7fe65beaee391d30da42e937db621564Steve Block unexpected_slow_tests.append(test_tuple) 974d0825bca7fe65beaee391d30da42e937db621564Steve Block 97521939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 97621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_test_list_timing("%s slowest tests that are not " 977d0825bca7fe65beaee391d30da42e937db621564Steve Block "marked as SLOW and did not timeout/crash:" % 97821939df44de1705786c545cd1bf519d47250322dBen Murdoch printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests) 97921939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 98021939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_test_list_timing("Tests marked as SLOW:", slow_tests) 98121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 98221939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_test_list_timing("Tests that timed out or crashed:", 983d0825bca7fe65beaee391d30da42e937db621564Steve Block timeout_or_crash_tests) 98421939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 985d0825bca7fe65beaee391d30da42e937db621564Steve Block 98621939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_test_list_timing(self, title, test_list): 987d0825bca7fe65beaee391d30da42e937db621564Steve Block """Print timing info for each test. 988d0825bca7fe65beaee391d30da42e937db621564Steve Block 989d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 990d0825bca7fe65beaee391d30da42e937db621564Steve Block title: section heading 991d0825bca7fe65beaee391d30da42e937db621564Steve Block test_list: tests that fall in this section 992d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 99321939df44de1705786c545cd1bf519d47250322dBen Murdoch if self._printer.disabled('slowest'): 99421939df44de1705786c545cd1bf519d47250322dBen Murdoch return 99521939df44de1705786c545cd1bf519d47250322dBen Murdoch 99621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(title) 997d0825bca7fe65beaee391d30da42e937db621564Steve Block for test_tuple in test_list: 998d0825bca7fe65beaee391d30da42e937db621564Steve Block filename = test_tuple.filename[len( 9998a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block self._port.layout_tests_dir()) + 1:] 1000d0825bca7fe65beaee391d30da42e937db621564Steve Block filename = filename.replace('\\', '/') 1001d0825bca7fe65beaee391d30da42e937db621564Steve Block test_run_time = round(test_tuple.test_run_time, 1) 100221939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" %s took %s seconds" % 100321939df44de1705786c545cd1bf519d47250322dBen Murdoch (filename, test_run_time)) 1004d0825bca7fe65beaee391d30da42e937db621564Steve Block 100521939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_directory_timings(self, directory_test_timings): 1006d0825bca7fe65beaee391d30da42e937db621564Steve Block """Print timing info by directory for any directories that 1007d0825bca7fe65beaee391d30da42e937db621564Steve Block take > 10 seconds to run. 1008d0825bca7fe65beaee391d30da42e937db621564Steve Block 1009d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 1010d0825bca7fe65beaee391d30da42e937db621564Steve Block directory_test_timing: time info for each directory 1011d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 1012d0825bca7fe65beaee391d30da42e937db621564Steve Block timings = [] 1013d0825bca7fe65beaee391d30da42e937db621564Steve Block for directory in directory_test_timings: 1014d0825bca7fe65beaee391d30da42e937db621564Steve Block num_tests, time_for_directory = directory_test_timings[directory] 1015d0825bca7fe65beaee391d30da42e937db621564Steve Block timings.append((round(time_for_directory, 1), directory, 1016d0825bca7fe65beaee391d30da42e937db621564Steve Block num_tests)) 1017d0825bca7fe65beaee391d30da42e937db621564Steve Block timings.sort() 1018d0825bca7fe65beaee391d30da42e937db621564Steve Block 101921939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("Time to process slowest subdirectories:") 1020d0825bca7fe65beaee391d30da42e937db621564Steve Block min_seconds_to_print = 10 1021d0825bca7fe65beaee391d30da42e937db621564Steve Block for timing in timings: 1022d0825bca7fe65beaee391d30da42e937db621564Steve Block if timing[0] > min_seconds_to_print: 102321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing( 102421939df44de1705786c545cd1bf519d47250322dBen Murdoch " %s took %s seconds to run %s tests." % (timing[1], 102521939df44de1705786c545cd1bf519d47250322dBen Murdoch timing[0], timing[2])) 102621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 1027d0825bca7fe65beaee391d30da42e937db621564Steve Block 102821939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_statistics_for_test_timings(self, title, timings): 1029d0825bca7fe65beaee391d30da42e937db621564Steve Block """Prints the median, mean and standard deviation of the values in 1030d0825bca7fe65beaee391d30da42e937db621564Steve Block timings. 1031d0825bca7fe65beaee391d30da42e937db621564Steve Block 1032d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 1033d0825bca7fe65beaee391d30da42e937db621564Steve Block title: Title for these timings. 1034d0825bca7fe65beaee391d30da42e937db621564Steve Block timings: A list of floats representing times. 1035d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 103621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(title) 1037d0825bca7fe65beaee391d30da42e937db621564Steve Block timings.sort() 1038d0825bca7fe65beaee391d30da42e937db621564Steve Block 1039d0825bca7fe65beaee391d30da42e937db621564Steve Block num_tests = len(timings) 1040dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block if not num_tests: 1041dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block return 1042d0825bca7fe65beaee391d30da42e937db621564Steve Block percentile90 = timings[int(.9 * num_tests)] 1043d0825bca7fe65beaee391d30da42e937db621564Steve Block percentile99 = timings[int(.99 * num_tests)] 1044d0825bca7fe65beaee391d30da42e937db621564Steve Block 1045d0825bca7fe65beaee391d30da42e937db621564Steve Block if num_tests % 2 == 1: 1046d0825bca7fe65beaee391d30da42e937db621564Steve Block median = timings[((num_tests - 1) / 2) - 1] 1047d0825bca7fe65beaee391d30da42e937db621564Steve Block else: 1048d0825bca7fe65beaee391d30da42e937db621564Steve Block lower = timings[num_tests / 2 - 1] 1049d0825bca7fe65beaee391d30da42e937db621564Steve Block upper = timings[num_tests / 2] 1050d0825bca7fe65beaee391d30da42e937db621564Steve Block median = (float(lower + upper)) / 2 1051d0825bca7fe65beaee391d30da42e937db621564Steve Block 1052d0825bca7fe65beaee391d30da42e937db621564Steve Block mean = sum(timings) / num_tests 1053d0825bca7fe65beaee391d30da42e937db621564Steve Block 1054d0825bca7fe65beaee391d30da42e937db621564Steve Block for time in timings: 1055d0825bca7fe65beaee391d30da42e937db621564Steve Block sum_of_deviations = math.pow(time - mean, 2) 1056d0825bca7fe65beaee391d30da42e937db621564Steve Block 1057d0825bca7fe65beaee391d30da42e937db621564Steve Block std_deviation = math.sqrt(sum_of_deviations / num_tests) 105821939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" Median: %6.3f" % median) 105921939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" Mean: %6.3f" % mean) 106021939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" 90th percentile: %6.3f" % percentile90) 106121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" 99th percentile: %6.3f" % percentile99) 106221939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing(" Standard dev: %6.3f" % std_deviation) 106321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_timing("") 106421939df44de1705786c545cd1bf519d47250322dBen Murdoch 106521939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_result_summary(self, result_summary): 1066d0825bca7fe65beaee391d30da42e937db621564Steve Block """Print a short summary about how many tests passed. 1067d0825bca7fe65beaee391d30da42e937db621564Steve Block 1068d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 1069d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: information to log 1070d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 1071d0825bca7fe65beaee391d30da42e937db621564Steve Block failed = len(result_summary.failures) 1072d0825bca7fe65beaee391d30da42e937db621564Steve Block skipped = len( 1073d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary.tests_by_expectation[test_expectations.SKIP]) 1074d0825bca7fe65beaee391d30da42e937db621564Steve Block total = result_summary.total 1075d0825bca7fe65beaee391d30da42e937db621564Steve Block passed = total - failed - skipped 1076d0825bca7fe65beaee391d30da42e937db621564Steve Block pct_passed = 0.0 1077d0825bca7fe65beaee391d30da42e937db621564Steve Block if total > 0: 1078d0825bca7fe65beaee391d30da42e937db621564Steve Block pct_passed = float(passed) * 100 / total 1079d0825bca7fe65beaee391d30da42e937db621564Steve Block 108021939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual("") 108121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" % 1082d0825bca7fe65beaee391d30da42e937db621564Steve Block (passed, total, pct_passed)) 108321939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual("") 108421939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_result_summary_entry(result_summary, 1085e14391e94c850b8bd03680c23b38978db68687a8John Reck test_expectations.NOW, "Tests to be fixed") 1086d0825bca7fe65beaee391d30da42e937db621564Steve Block 108721939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual("") 108821939df44de1705786c545cd1bf519d47250322dBen Murdoch self._print_result_summary_entry(result_summary, 1089d0825bca7fe65beaee391d30da42e937db621564Steve Block test_expectations.WONTFIX, 1090d0825bca7fe65beaee391d30da42e937db621564Steve Block "Tests that will only be fixed if they crash (WONTFIX)") 109121939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual("") 1092d0825bca7fe65beaee391d30da42e937db621564Steve Block 109321939df44de1705786c545cd1bf519d47250322dBen Murdoch def _print_result_summary_entry(self, result_summary, timeline, 1094d0825bca7fe65beaee391d30da42e937db621564Steve Block heading): 1095d0825bca7fe65beaee391d30da42e937db621564Steve Block """Print a summary block of results for a particular timeline of test. 1096d0825bca7fe65beaee391d30da42e937db621564Steve Block 1097d0825bca7fe65beaee391d30da42e937db621564Steve Block Args: 1098d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary: summary to print results for 1099d0825bca7fe65beaee391d30da42e937db621564Steve Block timeline: the timeline to print results for (NOT, WONTFIX, etc.) 1100d0825bca7fe65beaee391d30da42e937db621564Steve Block heading: a textual description of the timeline 1101d0825bca7fe65beaee391d30da42e937db621564Steve Block """ 1102d0825bca7fe65beaee391d30da42e937db621564Steve Block total = len(result_summary.tests_by_timeline[timeline]) 1103d0825bca7fe65beaee391d30da42e937db621564Steve Block not_passing = (total - 1104d0825bca7fe65beaee391d30da42e937db621564Steve Block len(result_summary.tests_by_expectation[test_expectations.PASS] & 1105d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary.tests_by_timeline[timeline])) 110621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual("=> %s (%d):" % (heading, not_passing)) 1107d0825bca7fe65beaee391d30da42e937db621564Steve Block 1108d0825bca7fe65beaee391d30da42e937db621564Steve Block for result in TestExpectationsFile.EXPECTATION_ORDER: 1109d0825bca7fe65beaee391d30da42e937db621564Steve Block if result == test_expectations.PASS: 1110d0825bca7fe65beaee391d30da42e937db621564Steve Block continue 1111d0825bca7fe65beaee391d30da42e937db621564Steve Block results = (result_summary.tests_by_expectation[result] & 1112d0825bca7fe65beaee391d30da42e937db621564Steve Block result_summary.tests_by_timeline[timeline]) 1113d0825bca7fe65beaee391d30da42e937db621564Steve Block desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result] 1114d0825bca7fe65beaee391d30da42e937db621564Steve Block if not_passing and len(results): 1115d0825bca7fe65beaee391d30da42e937db621564Steve Block pct = len(results) * 100.0 / not_passing 111621939df44de1705786c545cd1bf519d47250322dBen Murdoch self._printer.print_actual(" %5d %-24s (%4.1f%%)" % 111721939df44de1705786c545cd1bf519d47250322dBen Murdoch (len(results), desc[len(results) != 1], pct)) 1118d0825bca7fe65beaee391d30da42e937db621564Steve Block 11192daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def _copy_results_html_file(self): 11202daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch base_dir = self._port.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'layout_tests', 'layout_package') 11212daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch results_file = self._fs.join(base_dir, 'json_results.html') 11222daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch # FIXME: What should we do if this doesn't exist (e.g., in unit tests)? 11232daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if self._fs.exists(results_file): 11242daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self._fs.copyfile(results_file, self._fs.join(self._results_directory, "results.html")) 1125d0825bca7fe65beaee391d30da42e937db621564Steve Block 11262daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def _show_results_html_file(self, result_summary): 11272daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch """Shows the results.html page.""" 1128d0825bca7fe65beaee391d30da42e937db621564Steve Block if self._options.full_results_html: 1129d0825bca7fe65beaee391d30da42e937db621564Steve Block test_files = result_summary.failures.keys() 1130d0825bca7fe65beaee391d30da42e937db621564Steve Block else: 11312daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch unexpected_failures = self._get_failures(result_summary, include_crashes=True) 1132d0825bca7fe65beaee391d30da42e937db621564Steve Block test_files = unexpected_failures.keys() 1133d0825bca7fe65beaee391d30da42e937db621564Steve Block 11342daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if not len(test_files): 11352daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch return 1136d0825bca7fe65beaee391d30da42e937db621564Steve Block 11372daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch results_filename = self._fs.join(self._results_directory, "results.html") 11388a0914b749bbe7da7768e07a7db5c6d4bb09472bSteve Block self._port.show_results_html_file(results_filename) 1139d0825bca7fe65beaee391d30da42e937db621564Steve Block 1140d0825bca7fe65beaee391d30da42e937db621564Steve Block 1141ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdochdef read_test_files(fs, files): 1142d0825bca7fe65beaee391d30da42e937db621564Steve Block tests = [] 1143d0825bca7fe65beaee391d30da42e937db621564Steve Block for file in files: 11445abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick try: 1145ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch file_contents = fs.read_text_file(file).split('\n') 1146ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch for line in file_contents: 1147ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch line = test_expectations.strip_comments(line) 1148ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch if line: 1149ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests.append(line) 11505abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick except IOError, e: 11515abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick if e.errno == errno.ENOENT: 11525abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick _log.critical('') 11535abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick _log.critical('--test-list file "%s" not found' % file) 11545abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick raise 1155d0825bca7fe65beaee391d30da42e937db621564Steve Block return tests 1156