1# Copyright (c) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29from datetime import datetime
30import unittest
31
32from webkitpy.common.net import bugzilla
33from webkitpy.common.net.layouttestresults import LayoutTestResults
34from webkitpy.common.system.deprecated_logging import error, log
35from webkitpy.common.system.outputcapture import OutputCapture
36from webkitpy.layout_tests.layout_package import test_results
37from webkitpy.layout_tests.layout_package import test_failures
38from webkitpy.thirdparty.mock import Mock
39from webkitpy.tool.bot.commitqueuetask import *
40from webkitpy.tool.mocktool import MockTool
41
42
43class MockCommitQueue(CommitQueueTaskDelegate):
44    def __init__(self, error_plan):
45        self._error_plan = error_plan
46
47    def run_command(self, command):
48        log("run_webkit_patch: %s" % command)
49        if self._error_plan:
50            error = self._error_plan.pop(0)
51            if error:
52                raise error
53
54    def command_passed(self, success_message, patch):
55        log("command_passed: success_message='%s' patch='%s'" % (
56            success_message, patch.id()))
57
58    def command_failed(self, failure_message, script_error, patch):
59        log("command_failed: failure_message='%s' script_error='%s' patch='%s'" % (
60            failure_message, script_error, patch.id()))
61        return 3947
62
63    def refetch_patch(self, patch):
64        return patch
65
66    def layout_test_results(self):
67        return None
68
69    def report_flaky_tests(self, patch, flaky_results, results_archive):
70        flaky_tests = [result.filename for result in flaky_results]
71        log("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), flaky_tests, results_archive.filename))
72
73    def archive_last_layout_test_results(self, patch):
74        log("archive_last_layout_test_results: patch='%s'" % patch.id())
75        archive = Mock()
76        archive.filename = "mock-archive-%s.zip" % patch.id()
77        return archive
78
79
80class CommitQueueTaskTest(unittest.TestCase):
81    def _run_through_task(self, commit_queue, expected_stderr, expected_exception=None, expect_retry=False):
82        tool = MockTool(log_executive=True)
83        patch = tool.bugs.fetch_attachment(197)
84        task = CommitQueueTask(commit_queue, patch)
85        success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr, expected_exception=expected_exception)
86        if not expected_exception:
87            self.assertEqual(success, not expect_retry)
88
89    def test_success_case(self):
90        commit_queue = MockCommitQueue([])
91        expected_stderr = """run_webkit_patch: ['clean']
92command_passed: success_message='Cleaned working directory' patch='197'
93run_webkit_patch: ['update']
94command_passed: success_message='Updated working directory' patch='197'
95run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
96command_passed: success_message='Applied patch' patch='197'
97run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
98command_passed: success_message='Built patch' patch='197'
99run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
100command_passed: success_message='Passed tests' patch='197'
101run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197]
102command_passed: success_message='Landed patch' patch='197'
103"""
104        self._run_through_task(commit_queue, expected_stderr)
105
106    def test_clean_failure(self):
107        commit_queue = MockCommitQueue([
108            ScriptError("MOCK clean failure"),
109        ])
110        expected_stderr = """run_webkit_patch: ['clean']
111command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='197'
112"""
113        self._run_through_task(commit_queue, expected_stderr, expect_retry=True)
114
115    def test_update_failure(self):
116        commit_queue = MockCommitQueue([
117            None,
118            ScriptError("MOCK update failure"),
119        ])
120        expected_stderr = """run_webkit_patch: ['clean']
121command_passed: success_message='Cleaned working directory' patch='197'
122run_webkit_patch: ['update']
123command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='197'
124"""
125        self._run_through_task(commit_queue, expected_stderr, expect_retry=True)
126
127    def test_apply_failure(self):
128        commit_queue = MockCommitQueue([
129            None,
130            None,
131            ScriptError("MOCK apply failure"),
132        ])
133        expected_stderr = """run_webkit_patch: ['clean']
134command_passed: success_message='Cleaned working directory' patch='197'
135run_webkit_patch: ['update']
136command_passed: success_message='Updated working directory' patch='197'
137run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
138command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='197'
139"""
140        self._run_through_task(commit_queue, expected_stderr, ScriptError)
141
142    def test_build_failure(self):
143        commit_queue = MockCommitQueue([
144            None,
145            None,
146            None,
147            ScriptError("MOCK build failure"),
148        ])
149        expected_stderr = """run_webkit_patch: ['clean']
150command_passed: success_message='Cleaned working directory' patch='197'
151run_webkit_patch: ['update']
152command_passed: success_message='Updated working directory' patch='197'
153run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
154command_passed: success_message='Applied patch' patch='197'
155run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
156command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='197'
157run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both']
158command_passed: success_message='Able to build without patch' patch='197'
159"""
160        self._run_through_task(commit_queue, expected_stderr, ScriptError)
161
162    def test_red_build_failure(self):
163        commit_queue = MockCommitQueue([
164            None,
165            None,
166            None,
167            ScriptError("MOCK build failure"),
168            ScriptError("MOCK clean build failure"),
169        ])
170        expected_stderr = """run_webkit_patch: ['clean']
171command_passed: success_message='Cleaned working directory' patch='197'
172run_webkit_patch: ['update']
173command_passed: success_message='Updated working directory' patch='197'
174run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
175command_passed: success_message='Applied patch' patch='197'
176run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
177command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='197'
178run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both']
179command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='197'
180"""
181        self._run_through_task(commit_queue, expected_stderr, expect_retry=True)
182
183    def test_flaky_test_failure(self):
184        commit_queue = MockCommitQueue([
185            None,
186            None,
187            None,
188            None,
189            ScriptError("MOCK tests failure"),
190        ])
191        # CommitQueueTask will only report flaky tests if we successfully parsed
192        # results.html and returned a LayoutTestResults object, so we fake one.
193        commit_queue.layout_test_results = lambda: LayoutTestResults([])
194        expected_stderr = """run_webkit_patch: ['clean']
195command_passed: success_message='Cleaned working directory' patch='197'
196run_webkit_patch: ['update']
197command_passed: success_message='Updated working directory' patch='197'
198run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
199command_passed: success_message='Applied patch' patch='197'
200run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
201command_passed: success_message='Built patch' patch='197'
202run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
203command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='197'
204archive_last_layout_test_results: patch='197'
205run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
206command_passed: success_message='Passed tests' patch='197'
207report_flaky_tests: patch='197' flaky_tests='[]' archive='mock-archive-197.zip'
208run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197]
209command_passed: success_message='Landed patch' patch='197'
210"""
211        self._run_through_task(commit_queue, expected_stderr)
212
213    def test_failed_archive(self):
214        commit_queue = MockCommitQueue([
215            None,
216            None,
217            None,
218            None,
219            ScriptError("MOCK tests failure"),
220        ])
221        commit_queue.layout_test_results = lambda: LayoutTestResults([])
222        # It's possible delegate to fail to archive layout tests, don't try to report
223        # flaky tests when that happens.
224        commit_queue.archive_last_layout_test_results = lambda patch: None
225        expected_stderr = """run_webkit_patch: ['clean']
226command_passed: success_message='Cleaned working directory' patch='197'
227run_webkit_patch: ['update']
228command_passed: success_message='Updated working directory' patch='197'
229run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
230command_passed: success_message='Applied patch' patch='197'
231run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
232command_passed: success_message='Built patch' patch='197'
233run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
234command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='197'
235run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
236command_passed: success_message='Passed tests' patch='197'
237run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197]
238command_passed: success_message='Landed patch' patch='197'
239"""
240        self._run_through_task(commit_queue, expected_stderr)
241
242    def test_double_flaky_test_failure(self):
243        class DoubleFlakyCommitQueue(MockCommitQueue):
244            def __init__(self, error_plan):
245                MockCommitQueue.__init__(self, error_plan)
246                self._double_flaky_test_counter = 0
247
248            def run_command(self, command):
249                self._double_flaky_test_counter += 1
250                MockCommitQueue.run_command(self, command)
251
252            def _mock_test_result(self, testname):
253                return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
254
255            def layout_test_results(self):
256                if self._double_flaky_test_counter % 2:
257                    return LayoutTestResults([self._mock_test_result('foo.html')])
258                return LayoutTestResults([self._mock_test_result('bar.html')])
259
260        commit_queue = DoubleFlakyCommitQueue([
261            None,
262            None,
263            None,
264            None,
265            ScriptError("MOCK test failure"),
266            ScriptError("MOCK test failure again"),
267        ])
268        # The (subtle) point of this test is that report_flaky_tests does not appear
269        # in the expected_stderr for this run.
270        # Note also that there is no attempt to run the tests w/o the patch.
271        expected_stderr = """run_webkit_patch: ['clean']
272command_passed: success_message='Cleaned working directory' patch='197'
273run_webkit_patch: ['update']
274command_passed: success_message='Updated working directory' patch='197'
275run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
276command_passed: success_message='Applied patch' patch='197'
277run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
278command_passed: success_message='Built patch' patch='197'
279run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
280command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197'
281archive_last_layout_test_results: patch='197'
282run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
283command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197'
284"""
285        tool = MockTool(log_executive=True)
286        patch = tool.bugs.fetch_attachment(197)
287        task = CommitQueueTask(commit_queue, patch)
288        success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr)
289        self.assertEqual(success, False)
290
291    def test_test_failure(self):
292        commit_queue = MockCommitQueue([
293            None,
294            None,
295            None,
296            None,
297            ScriptError("MOCK test failure"),
298            ScriptError("MOCK test failure again"),
299        ])
300        expected_stderr = """run_webkit_patch: ['clean']
301command_passed: success_message='Cleaned working directory' patch='197'
302run_webkit_patch: ['update']
303command_passed: success_message='Updated working directory' patch='197'
304run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
305command_passed: success_message='Applied patch' patch='197'
306run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
307command_passed: success_message='Built patch' patch='197'
308run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
309command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197'
310archive_last_layout_test_results: patch='197'
311run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
312command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197'
313archive_last_layout_test_results: patch='197'
314run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
315command_passed: success_message='Able to pass tests without patch' patch='197'
316"""
317        self._run_through_task(commit_queue, expected_stderr, ScriptError)
318
319    def test_red_test_failure(self):
320        commit_queue = MockCommitQueue([
321            None,
322            None,
323            None,
324            None,
325            ScriptError("MOCK test failure"),
326            ScriptError("MOCK test failure again"),
327            ScriptError("MOCK clean test failure"),
328        ])
329        expected_stderr = """run_webkit_patch: ['clean']
330command_passed: success_message='Cleaned working directory' patch='197'
331run_webkit_patch: ['update']
332command_passed: success_message='Updated working directory' patch='197'
333run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
334command_passed: success_message='Applied patch' patch='197'
335run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
336command_passed: success_message='Built patch' patch='197'
337run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
338command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197'
339archive_last_layout_test_results: patch='197'
340run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
341command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197'
342archive_last_layout_test_results: patch='197'
343run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive']
344command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='197'
345"""
346        self._run_through_task(commit_queue, expected_stderr, expect_retry=True)
347
348    def test_land_failure(self):
349        commit_queue = MockCommitQueue([
350            None,
351            None,
352            None,
353            None,
354            None,
355            ScriptError("MOCK land failure"),
356        ])
357        expected_stderr = """run_webkit_patch: ['clean']
358command_passed: success_message='Cleaned working directory' patch='197'
359run_webkit_patch: ['update']
360command_passed: success_message='Updated working directory' patch='197'
361run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197]
362command_passed: success_message='Applied patch' patch='197'
363run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both']
364command_passed: success_message='Built patch' patch='197'
365run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
366command_passed: success_message='Passed tests' patch='197'
367run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197]
368command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='197'
369"""
370        # FIXME: This should really be expect_retry=True for a better user experiance.
371        self._run_through_task(commit_queue, expected_stderr, ScriptError)
372
373    def _expect_validate(self, patch, is_valid):
374        class MockDelegate(object):
375            def refetch_patch(self, patch):
376                return patch
377
378        task = CommitQueueTask(MockDelegate(), patch)
379        self.assertEquals(task._validate(), is_valid)
380
381    def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"):
382        bug = bugzilla.Bug(bug_dict, None)
383        patch = bugzilla.Attachment(attachment_dict, bug)
384        patch._committer = committer
385        return patch
386
387    def test_validate(self):
388        self._expect_validate(self._mock_patch(), True)
389        self._expect_validate(self._mock_patch({'is_obsolete': True}), False)
390        self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False)
391        self._expect_validate(self._mock_patch(committer=None), False)
392        self._expect_validate(self._mock_patch({'review': '-'}), False)
393