1# Copyright (C) 2012 Google Inc. All rights reserved.
2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30import unittest
31
32from webkitpy.common.host_mock import MockHost
33from webkitpy.common.system.systemhost_mock import MockSystemHost
34from webkitpy.layout_tests import run_webkit_tests
35from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
36from webkitpy.layout_tests.models import test_expectations
37from webkitpy.layout_tests.models import test_failures
38from webkitpy.layout_tests.models.test_run_results import TestRunResults
39from webkitpy.layout_tests.models.test_input import TestInput
40from webkitpy.layout_tests.models.test_results import TestResult
41from webkitpy.layout_tests.port.test import TestPort
42
43
44TestExpectations = test_expectations.TestExpectations
45
46
47class FakePrinter(object):
48    num_completed = 0
49    num_tests = 0
50
51    def print_expected(self, run_results, get_tests_with_result_type):
52        pass
53
54    def print_workers_and_shards(self, num_workers, num_shards, num_locked_shards):
55        pass
56
57    def print_started_test(self, test_name):
58        pass
59
60    def print_finished_test(self, result, expected, exp_str, got_str):
61        pass
62
63    def write(self, msg):
64        pass
65
66    def write_update(self, msg):
67        pass
68
69    def flush(self):
70        pass
71
72
73class LockCheckingRunner(LayoutTestRunner):
74    def __init__(self, port, options, printer, tester, http_lock):
75        super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False)
76        self._finished_list_called = False
77        self._tester = tester
78        self._should_have_http_lock = http_lock
79
80    def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
81        if not self._finished_list_called:
82            self._tester.assertEqual(list_name, 'locked_tests')
83            self._tester.assertTrue(self._remaining_locked_shards)
84            self._tester.assertTrue(self._has_http_lock is self._should_have_http_lock)
85
86        super(LockCheckingRunner, self).handle_finished_list(source, list_name, num_tests, elapsed_time)
87
88        if not self._finished_list_called:
89            self._tester.assertEqual(self._remaining_locked_shards, [])
90            self._tester.assertFalse(self._has_http_lock)
91            self._finished_list_called = True
92
93
94class LayoutTestRunnerTests(unittest.TestCase):
95    def _runner(self, port=None):
96        # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
97        options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
98        options.child_processes = '1'
99
100        host = MockHost()
101        port = port or host.port_factory.get(options.platform, options=options)
102        return LockCheckingRunner(port, options, FakePrinter(), self, True)
103
104    def _run_tests(self, runner, tests):
105        test_inputs = [TestInput(test, 6000) for test in tests]
106        expectations = TestExpectations(runner._port, tests)
107        runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False)
108
109    def test_interrupt_if_at_failure_limits(self):
110        runner = self._runner()
111        runner._options.exit_after_n_failures = None
112        runner._options.exit_after_n_crashes_or_times = None
113        test_names = ['passes/text.html', 'passes/image.html']
114        runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]
115
116        run_results = TestRunResults(TestExpectations(runner._port, test_names), len(test_names))
117        run_results.unexpected_failures = 100
118        run_results.unexpected_crashes = 50
119        run_results.unexpected_timeouts = 50
120        # No exception when the exit_after* options are None.
121        runner._interrupt_if_at_failure_limits(run_results)
122
123        # No exception when we haven't hit the limit yet.
124        runner._options.exit_after_n_failures = 101
125        runner._options.exit_after_n_crashes_or_timeouts = 101
126        runner._interrupt_if_at_failure_limits(run_results)
127
128        # Interrupt if we've exceeded either limit:
129        runner._options.exit_after_n_crashes_or_timeouts = 10
130        self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
131        self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP)
132        self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP)
133
134        runner._options.exit_after_n_crashes_or_timeouts = None
135        runner._options.exit_after_n_failures = 10
136        exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
137
138    def test_update_summary_with_result(self):
139        # Reftests expected to be image mismatch should be respected when pixel_tests=False.
140        runner = self._runner()
141        runner._options.pixel_tests = False
142        test = 'failures/expected/reftest.html'
143        expectations = TestExpectations(runner._port, tests=[test])
144        runner._expectations = expectations
145
146        run_results = TestRunResults(expectations, 1)
147        result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!='])
148        runner._update_summary_with_result(run_results, result)
149        self.assertEqual(1, run_results.expected)
150        self.assertEqual(0, run_results.unexpected)
151
152        run_results = TestRunResults(expectations, 1)
153        result = TestResult(test_name=test, failures=[], reftest_type=['=='])
154        runner._update_summary_with_result(run_results, result)
155        self.assertEqual(0, run_results.expected)
156        self.assertEqual(1, run_results.unexpected)
157
158
159class SharderTests(unittest.TestCase):
160
161    test_list = [
162        "http/tests/websocket/tests/unicode.htm",
163        "animations/keyframes.html",
164        "http/tests/security/view-source-no-refresh.html",
165        "http/tests/websocket/tests/websocket-protocol-ignored.html",
166        "fast/css/display-none-inline-style-change-crash.html",
167        "http/tests/xmlhttprequest/supported-xml-content-types.html",
168        "dom/html/level2/html/HTMLAnchorElement03.html",
169        "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
170        "dom/html/level2/html/HTMLAnchorElement06.html",
171        "perf/object-keys.html",
172        "virtual/threaded/dir/test.html",
173        "virtual/threaded/fast/foo/test.html",
174    ]
175
176    def get_test_input(self, test_file):
177        return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')))
178
179    def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1):
180        port = TestPort(MockSystemHost())
181        self.sharder = Sharder(port.split_test, max_locked_shards)
182        test_list = test_list or self.test_list
183        return self.sharder.shard_tests([self.get_test_input(test) for test in test_list],
184            num_workers, fully_parallel, run_singly)
185
186    def assert_shards(self, actual_shards, expected_shard_names):
187        self.assertEqual(len(actual_shards), len(expected_shard_names))
188        for i, shard in enumerate(actual_shards):
189            expected_shard_name, expected_test_names = expected_shard_names[i]
190            self.assertEqual(shard.name, expected_shard_name)
191            self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
192                              expected_test_names)
193
194    def test_shard_by_dir(self):
195        locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False)
196
197        # Note that although there are tests in multiple dirs that need locks,
198        # they are crammed into a single shard in order to reduce the # of
199        # workers hitting the server at once.
200        self.assert_shards(locked,
201             [('locked_shard_1',
202               ['http/tests/security/view-source-no-refresh.html',
203                'http/tests/websocket/tests/unicode.htm',
204                'http/tests/websocket/tests/websocket-protocol-ignored.html',
205                'http/tests/xmlhttprequest/supported-xml-content-types.html',
206                'perf/object-keys.html'])])
207        self.assert_shards(unlocked,
208            [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
209             ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
210             ('animations', ['animations/keyframes.html']),
211             ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
212                                      'dom/html/level2/html/HTMLAnchorElement06.html']),
213             ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
214             ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
215
216    def test_shard_every_file(self):
217        locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False)
218        self.assert_shards(locked,
219            [('locked_shard_1',
220              ['http/tests/websocket/tests/unicode.htm',
221               'http/tests/security/view-source-no-refresh.html',
222               'http/tests/websocket/tests/websocket-protocol-ignored.html']),
223             ('locked_shard_2',
224              ['http/tests/xmlhttprequest/supported-xml-content-types.html',
225               'perf/object-keys.html'])]),
226        self.assert_shards(unlocked,
227            [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
228             ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
229             ('.', ['animations/keyframes.html']),
230             ('.', ['fast/css/display-none-inline-style-change-crash.html']),
231             ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
232             ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
233             ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
234
235    def test_shard_in_two(self):
236        locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False)
237        self.assert_shards(locked,
238            [('locked_tests',
239              ['http/tests/websocket/tests/unicode.htm',
240               'http/tests/security/view-source-no-refresh.html',
241               'http/tests/websocket/tests/websocket-protocol-ignored.html',
242               'http/tests/xmlhttprequest/supported-xml-content-types.html',
243               'perf/object-keys.html'])])
244        self.assert_shards(unlocked,
245            [('unlocked_tests',
246              ['animations/keyframes.html',
247               'fast/css/display-none-inline-style-change-crash.html',
248               'dom/html/level2/html/HTMLAnchorElement03.html',
249               'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
250               'dom/html/level2/html/HTMLAnchorElement06.html',
251               'virtual/threaded/dir/test.html',
252               'virtual/threaded/fast/foo/test.html'])])
253
254    def test_shard_in_two_has_no_locked_shards(self):
255        locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
256             test_list=['animations/keyframe.html'])
257        self.assertEqual(len(locked), 0)
258        self.assertEqual(len(unlocked), 1)
259
260    def test_shard_in_two_has_no_unlocked_shards(self):
261        locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
262             test_list=['http/tests/websocket/tests/unicode.htm'])
263        self.assertEqual(len(locked), 1)
264        self.assertEqual(len(unlocked), 0)
265
266    def test_multiple_locked_shards(self):
267        locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False)
268        self.assert_shards(locked,
269            [('locked_shard_1',
270              ['http/tests/security/view-source-no-refresh.html',
271               'http/tests/websocket/tests/unicode.htm',
272               'http/tests/websocket/tests/websocket-protocol-ignored.html']),
273             ('locked_shard_2',
274              ['http/tests/xmlhttprequest/supported-xml-content-types.html',
275               'perf/object-keys.html'])])
276
277        locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False)
278        self.assert_shards(locked,
279            [('locked_shard_1',
280              ['http/tests/security/view-source-no-refresh.html',
281               'http/tests/websocket/tests/unicode.htm',
282               'http/tests/websocket/tests/websocket-protocol-ignored.html',
283               'http/tests/xmlhttprequest/supported-xml-content-types.html',
284               'perf/object-keys.html'])])
285
286    def test_virtual_shards(self):
287        # With run_singly=False, we try to keep all of the tests in a virtual suite together even
288        # when fully_parallel=True, so that we don't restart every time the command line args change.
289        locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False,
290                test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
291        self.assert_shards(unlocked,
292            [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])])
293
294        # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism.
295        locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True,
296                test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
297        self.assert_shards(unlocked,
298            [('.', ['virtual/foo/bar1.html']),
299             ('.', ['virtual/foo/bar2.html'])])
300