15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2012 Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 30e38fbeeb576b5094e34e038ab88d9d6a5c5c2214Torne (Richard Coles)import unittest 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.host_mock import MockHost 33926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.common.system.systemhost_mock import MockSystemHost 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests import run_webkit_tests 35926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models import test_expectations 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models import test_failures 38926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models.test_run_results import TestRunResults 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models.test_input import TestInput 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models.test_results import TestResult 41926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.port.test import TestPort 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)TestExpectations = test_expectations.TestExpectations 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FakePrinter(object): 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) num_completed = 0 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) num_tests = 0 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 51926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def print_expected(self, run_results, get_tests_with_result_type): 52926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) pass 53926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def print_workers_and_shards(self, num_workers, num_shards, num_locked_shards): 555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def print_started_test(self, test_name): 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def print_finished_test(self, result, expected, exp_str, got_str): 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def write(self, msg): 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def write_update(self, msg): 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def flush(self): 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class LockCheckingRunner(LayoutTestRunner): 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, port, options, printer, tester, http_lock): 75926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False) 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._finished_list_called = False 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._tester = tester 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._should_have_http_lock = http_lock 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def handle_finished_list(self, source, list_name, num_tests, elapsed_time): 815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not self._finished_list_called: 82926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self._tester.assertEqual(list_name, 'locked_tests') 835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._tester.assertTrue(self._remaining_locked_shards) 845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._tester.assertTrue(self._has_http_lock is self._should_have_http_lock) 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(LockCheckingRunner, self).handle_finished_list(source, list_name, num_tests, elapsed_time) 875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not self._finished_list_called: 89926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self._tester.assertEqual(self._remaining_locked_shards, []) 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._tester.assertFalse(self._has_http_lock) 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._finished_list_called = True 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class LayoutTestRunnerTests(unittest.TestCase): 955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _runner(self, port=None): 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need. 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0] 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) options.child_processes = '1' 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) host = MockHost() 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) port = port or host.port_factory.get(options.platform, options=options) 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return LockCheckingRunner(port, options, FakePrinter(), self, True) 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _run_tests(self, runner, tests): 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_inputs = [TestInput(test, 6000) for test in tests] 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expectations = TestExpectations(runner._port, tests) 107591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False) 1085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_interrupt_if_at_failure_limits(self): 1105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner = self._runner() 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_failures = None 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_crashes_or_times = None 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_names = ['passes/text.html', 'passes/image.html'] 114926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names] 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 116926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) run_results = TestRunResults(TestExpectations(runner._port, test_names), len(test_names)) 117926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) run_results.unexpected_failures = 100 118926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) run_results.unexpected_crashes = 50 119926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) run_results.unexpected_timeouts = 50 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # No exception when the exit_after* options are None. 121926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) runner._interrupt_if_at_failure_limits(run_results) 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # No exception when we haven't hit the limit yet. 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_failures = 101 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_crashes_or_timeouts = 101 126926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) runner._interrupt_if_at_failure_limits(run_results) 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Interrupt if we've exceeded either limit: 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_crashes_or_timeouts = 10 130926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results) 131926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP) 132926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP) 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_crashes_or_timeouts = None 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.exit_after_n_failures = 10 136926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results) 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_update_summary_with_result(self): 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Reftests expected to be image mismatch should be respected when pixel_tests=False. 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner = self._runner() 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._options.pixel_tests = False 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test = 'failures/expected/reftest.html' 1435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expectations = TestExpectations(runner._port, tests=[test]) 1445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) runner._expectations = expectations 1455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 146926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) run_results = TestRunResults(expectations, 1) 1475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!=']) 148926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) runner._update_summary_with_result(run_results, result) 149926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(1, run_results.expected) 150926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(0, run_results.unexpected) 1515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 152926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) run_results = TestRunResults(expectations, 1) 1535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) result = TestResult(test_name=test, failures=[], reftest_type=['==']) 154926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) runner._update_summary_with_result(run_results, result) 155926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(0, run_results.expected) 156926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(1, run_results.unexpected) 1575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class SharderTests(unittest.TestCase): 1605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_list = [ 1625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "http/tests/websocket/tests/unicode.htm", 1635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "animations/keyframes.html", 1645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "http/tests/security/view-source-no-refresh.html", 1655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "http/tests/websocket/tests/websocket-protocol-ignored.html", 1665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "fast/css/display-none-inline-style-change-crash.html", 1675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "http/tests/xmlhttprequest/supported-xml-content-types.html", 1685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "dom/html/level2/html/HTMLAnchorElement03.html", 1695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "ietestcenter/Javascript/11.1.5_4-4-c-1.html", 1705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "dom/html/level2/html/HTMLAnchorElement06.html", 1715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "perf/object-keys.html", 1721e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) "virtual/threaded/dir/test.html", 1731e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) "virtual/threaded/fast/foo/test.html", 1745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ] 1755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def get_test_input(self, test_file): 1775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) 1785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 179323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1): 180926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) port = TestPort(MockSystemHost()) 181926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.sharder = Sharder(port.split_test, max_locked_shards) 1825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_list = test_list or self.test_list 183323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], 184323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) num_workers, fully_parallel, run_singly) 1855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def assert_shards(self, actual_shards, expected_shard_names): 187926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(len(actual_shards), len(expected_shard_names)) 1885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for i, shard in enumerate(actual_shards): 1895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_shard_name, expected_test_names = expected_shard_names[i] 190926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(shard.name, expected_shard_name) 191926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual([test_input.test_name for test_input in shard.test_inputs], 1925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_test_names) 1935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_shard_by_dir(self): 195323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False) 1965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that although there are tests in multiple dirs that need locks, 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # they are crammed into a single shard in order to reduce the # of 1995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # workers hitting the server at once. 2005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(locked, 2015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) [('locked_shard_1', 2025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['http/tests/security/view-source-no-refresh.html', 2035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/unicode.htm', 2045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/websocket-protocol-ignored.html', 2055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/xmlhttprequest/supported-xml-content-types.html', 2065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'perf/object-keys.html'])]) 2075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(unlocked, 2081e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), 2091e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), 210591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch ('animations', ['animations/keyframes.html']), 2115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 2125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'dom/html/level2/html/HTMLAnchorElement06.html']), 2135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), 2145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) 2155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_shard_every_file(self): 217323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False) 2185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(locked, 2191e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) [('locked_shard_1', 2201e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ['http/tests/websocket/tests/unicode.htm', 2211e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'http/tests/security/view-source-no-refresh.html', 2221e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 2231e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ('locked_shard_2', 2241e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ['http/tests/xmlhttprequest/supported-xml-content-types.html', 2251e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'perf/object-keys.html'])]), 2265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(unlocked, 2271e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), 2281e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), 2291e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ('.', ['animations/keyframes.html']), 2305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('.', ['fast/css/display-none-inline-style-change-crash.html']), 2315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), 2325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), 2331e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) 2345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_shard_in_two(self): 236323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False) 2375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(locked, 2385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) [('locked_tests', 2395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['http/tests/websocket/tests/unicode.htm', 2405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/security/view-source-no-refresh.html', 2415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/websocket-protocol-ignored.html', 2425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/xmlhttprequest/supported-xml-content-types.html', 2435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'perf/object-keys.html'])]) 2445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(unlocked, 2455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) [('unlocked_tests', 2465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['animations/keyframes.html', 2475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'fast/css/display-none-inline-style-change-crash.html', 2485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'dom/html/level2/html/HTMLAnchorElement03.html', 2495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 250591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch 'dom/html/level2/html/HTMLAnchorElement06.html', 2511e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'virtual/threaded/dir/test.html', 2521e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'virtual/threaded/fast/foo/test.html'])]) 2535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_shard_in_two_has_no_locked_shards(self): 255323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, 2565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_list=['animations/keyframe.html']) 257926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(len(locked), 0) 258926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(len(unlocked), 1) 2595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_shard_in_two_has_no_unlocked_shards(self): 261323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, 2625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_list=['http/tests/websocket/tests/unicode.htm']) 263926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(len(locked), 1) 264926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(len(unlocked), 0) 2655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_multiple_locked_shards(self): 267323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False) 2685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(locked, 2695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) [('locked_shard_1', 2705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['http/tests/security/view-source-no-refresh.html', 2715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/unicode.htm', 2725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 2735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('locked_shard_2', 2745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['http/tests/xmlhttprequest/supported-xml-content-types.html', 2755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'perf/object-keys.html'])]) 2765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 277323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False) 2785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.assert_shards(locked, 2795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) [('locked_shard_1', 2805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['http/tests/security/view-source-no-refresh.html', 2815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/unicode.htm', 2825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/websocket/tests/websocket-protocol-ignored.html', 2835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'http/tests/xmlhttprequest/supported-xml-content-types.html', 2845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'perf/object-keys.html'])]) 285323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) 286323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) def test_virtual_shards(self): 287323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) # With run_singly=False, we try to keep all of the tests in a virtual suite together even 288323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) # when fully_parallel=True, so that we don't restart every time the command line args change. 289323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False, 290323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) 291323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) self.assert_shards(unlocked, 292323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])]) 293323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) 294323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism. 295323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True, 296323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) 297323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) self.assert_shards(unlocked, 298323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) [('.', ['virtual/foo/bar1.html']), 299323480423219ecd77329f8326dc5e0e3b50926d4Torne (Richard Coles) ('.', ['virtual/foo/bar2.html'])]) 300