1# Copyright (C) 2009 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#    * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#    * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#    * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import os
30import StringIO
31
32from webkitpy.common.checkout.scm import CheckoutNeedsUpdate
33from webkitpy.common.net.bugzilla import Attachment
34from webkitpy.common.system.filesystem_mock import MockFileSystem
35from webkitpy.common.system.outputcapture import OutputCapture
36from webkitpy.layout_tests.layout_package import test_results
37from webkitpy.layout_tests.layout_package import test_failures
38from webkitpy.thirdparty.mock import Mock
39from webkitpy.tool.commands.commandtest import CommandsTest
40from webkitpy.tool.commands.queues import *
41from webkitpy.tool.commands.queuestest import QueuesTest
42from webkitpy.tool.commands.stepsequence import StepSequence
43from webkitpy.tool.mocktool import MockTool, MockSCM, MockStatusServer
44
45
46class TestQueue(AbstractPatchQueue):
47    name = "test-queue"
48
49
50class TestReviewQueue(AbstractReviewQueue):
51    name = "test-review-queue"
52
53
54class TestFeederQueue(FeederQueue):
55    _sleep_duration = 0
56
57
58class AbstractQueueTest(CommandsTest):
59    def test_log_directory(self):
60        self.assertEquals(TestQueue()._log_directory(), os.path.join("..", "test-queue-logs"))
61
62    def _assert_run_webkit_patch(self, run_args, port=None):
63        queue = TestQueue()
64        tool = MockTool()
65        tool.status_server.bot_id = "gort"
66        tool.executive = Mock()
67        queue.bind_to_tool(tool)
68        queue._options = Mock()
69        queue._options.port = port
70
71        queue.run_webkit_patch(run_args)
72        expected_run_args = ["echo", "--status-host=example.com", "--bot-id=gort"]
73        if port:
74            expected_run_args.append("--port=%s" % port)
75        expected_run_args.extend(run_args)
76        tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args)
77
78    def test_run_webkit_patch(self):
79        self._assert_run_webkit_patch([1])
80        self._assert_run_webkit_patch(["one", 2])
81        self._assert_run_webkit_patch([1], port="mockport")
82
83    def test_iteration_count(self):
84        queue = TestQueue()
85        queue._options = Mock()
86        queue._options.iterations = 3
87        self.assertTrue(queue.should_continue_work_queue())
88        self.assertTrue(queue.should_continue_work_queue())
89        self.assertTrue(queue.should_continue_work_queue())
90        self.assertFalse(queue.should_continue_work_queue())
91
92    def test_no_iteration_count(self):
93        queue = TestQueue()
94        queue._options = Mock()
95        self.assertTrue(queue.should_continue_work_queue())
96        self.assertTrue(queue.should_continue_work_queue())
97        self.assertTrue(queue.should_continue_work_queue())
98        self.assertTrue(queue.should_continue_work_queue())
99
100    def _assert_log_message(self, script_error, log_message):
101        failure_log = AbstractQueue._log_from_script_error_for_upload(script_error, output_limit=10)
102        self.assertTrue(failure_log.read(), log_message)
103
104    def test_log_from_script_error_for_upload(self):
105        self._assert_log_message(ScriptError("test"), "test")
106        # In python 2.5 unicode(Exception) is busted. See:
107        # http://bugs.python.org/issue2517
108        # With no good workaround, we just ignore these tests.
109        if not hasattr(Exception, "__unicode__"):
110            return
111
112        unicode_tor = u"WebKit \u2661 Tor Arne Vestb\u00F8!"
113        utf8_tor = unicode_tor.encode("utf-8")
114        self._assert_log_message(ScriptError(unicode_tor), utf8_tor)
115        script_error = ScriptError(unicode_tor, output=unicode_tor)
116        expected_output = "%s\nLast %s characters of output:\n%s" % (utf8_tor, 10, utf8_tor[-10:])
117        self._assert_log_message(script_error, expected_output)
118
119
120class FeederQueueTest(QueuesTest):
121    def test_feeder_queue(self):
122        queue = TestFeederQueue()
123        tool = MockTool(log_executive=True)
124        expected_stderr = {
125            "begin_work_queue": self._default_begin_work_queue_stderr("feeder-queue", MockSCM.fake_checkout_root),
126            "should_proceed_with_work_item": "",
127            "next_work_item": "",
128            "process_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
129Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
130MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting attachment 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/Tools/Scripts/webkitpy/common/config/committers.py.
131
132- If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags.
133
134- If you have committer rights please correct the error in Tools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed).  The commit-queue restarts itself every 2 hours.  After restart the commit-queue will correctly respect your committer rights.'
135MOCK: update_work_items: commit-queue [106, 197]
136Feeding commit-queue items [106, 197]
137Feeding EWS (1 r? patch, 1 new)
138MOCK: submit_to_ews: 103
139""",
140            "handle_unexpected_error": "Mock error message\n",
141        }
142        self.assert_queue_outputs(queue, tool=tool, expected_stderr=expected_stderr)
143
144
145class AbstractPatchQueueTest(CommandsTest):
146    def test_next_patch(self):
147        queue = AbstractPatchQueue()
148        tool = MockTool()
149        queue.bind_to_tool(tool)
150        queue._options = Mock()
151        queue._options.port = None
152        self.assertEquals(queue._next_patch(), None)
153        tool.status_server = MockStatusServer(work_items=[2, 197])
154        expected_stdout = "MOCK: fetch_attachment: 2 is not a known attachment id\n"  # A mock-only message to prevent us from making mistakes.
155        expected_stderr = "MOCK: release_work_item: None 2\n"
156        patch_id = OutputCapture().assert_outputs(self, queue._next_patch, [], expected_stdout=expected_stdout, expected_stderr=expected_stderr)
157        self.assertEquals(patch_id, None)  # 2 is an invalid patch id
158        self.assertEquals(queue._next_patch().id(), 197)
159
160
161class NeedsUpdateSequence(StepSequence):
162    def _run(self, tool, options, state):
163        raise CheckoutNeedsUpdate([], 1, "", None)
164
165
166class AlwaysCommitQueueTool(object):
167    def __init__(self):
168        self.status_server = MockStatusServer()
169
170    def command_by_name(self, name):
171        return CommitQueue
172
173
174class SecondThoughtsCommitQueue(CommitQueue):
175    def __init__(self):
176        self._reject_patch = False
177        CommitQueue.__init__(self)
178
179    def run_command(self, command):
180        # We want to reject the patch after the first validation,
181        # so wait to reject it until after some other command has run.
182        self._reject_patch = True
183        return CommitQueue.run_command(self, command)
184
185    def refetch_patch(self, patch):
186        if not self._reject_patch:
187            return self._tool.bugs.fetch_attachment(patch.id())
188
189        attachment_dictionary = {
190            "id": patch.id(),
191            "bug_id": patch.bug_id(),
192            "name": "Rejected",
193            "is_obsolete": True,
194            "is_patch": False,
195            "review": "-",
196            "reviewer_email": "foo@bar.com",
197            "commit-queue": "-",
198            "committer_email": "foo@bar.com",
199            "attacher_email": "Contributer1",
200        }
201        return Attachment(attachment_dictionary, None)
202
203
204class CommitQueueTest(QueuesTest):
205    def _mock_test_result(self, testname):
206        return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
207
208    def test_commit_queue(self):
209        expected_stderr = {
210            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
211            "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n",
212            "next_work_item": "",
213            "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory
214MOCK: update_status: commit-queue Updated working directory
215MOCK: update_status: commit-queue Applied patch
216MOCK: update_status: commit-queue Built patch
217MOCK: update_status: commit-queue Passed tests
218MOCK: update_status: commit-queue Landed patch
219MOCK: update_status: commit-queue Pass
220MOCK: release_work_item: commit-queue 197
221""",
222            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n",
223            "handle_script_error": "ScriptError error message\n",
224        }
225        self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr)
226
227    def test_commit_queue_failure(self):
228        expected_stderr = {
229            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
230            "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n",
231            "next_work_item": "",
232            "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory
233MOCK: update_status: commit-queue Updated working directory
234MOCK: update_status: commit-queue Patch does not apply
235MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'MOCK script error'
236MOCK: update_status: commit-queue Fail
237MOCK: release_work_item: commit-queue 197
238""",
239            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n",
240            "handle_script_error": "ScriptError error message\n",
241        }
242        queue = CommitQueue()
243
244        def mock_run_webkit_patch(command):
245            if command == ['clean'] or command == ['update']:
246                # We want cleaning to succeed so we can error out on a step
247                # that causes the commit-queue to reject the patch.
248                return
249            raise ScriptError('MOCK script error')
250
251        queue.run_webkit_patch = mock_run_webkit_patch
252        self.assert_queue_outputs(queue, expected_stderr=expected_stderr)
253
254    def test_rollout(self):
255        tool = MockTool(log_executive=True)
256        tool.filesystem.write_text_file('/mock/results.html', '')  # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
257        tool.buildbot.light_tree_on_fire()
258        expected_stderr = {
259            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
260            "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n",
261            "next_work_item": "",
262            "process_work_item": """MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'clean']
263MOCK: update_status: commit-queue Cleaned working directory
264MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'update']
265MOCK: update_status: commit-queue Updated working directory
266MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachment', '--no-update', '--non-interactive', 197]
267MOCK: update_status: commit-queue Applied patch
268MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build', '--no-clean', '--no-update', '--build-style=both']
269MOCK: update_status: commit-queue Built patch
270MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive']
271MOCK: update_status: commit-queue Passed tests
272MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197]
273MOCK: update_status: commit-queue Landed patch
274MOCK: update_status: commit-queue Pass
275MOCK: release_work_item: commit-queue 197
276""",
277            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n",
278            "handle_script_error": "ScriptError error message\n",
279        }
280        self.assert_queue_outputs(CommitQueue(), tool=tool, expected_stderr=expected_stderr)
281
282    def test_rollout_lands(self):
283        tool = MockTool(log_executive=True)
284        tool.buildbot.light_tree_on_fire()
285        rollout_patch = tool.bugs.fetch_attachment(106)  # _patch6, a rollout patch.
286        assert(rollout_patch.is_rollout())
287        expected_stderr = {
288            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
289            "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing rollout patch\n",
290            "next_work_item": "",
291            "process_work_item": """MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'clean']
292MOCK: update_status: commit-queue Cleaned working directory
293MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'update']
294MOCK: update_status: commit-queue Updated working directory
295MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachment', '--no-update', '--non-interactive', 106]
296MOCK: update_status: commit-queue Applied patch
297MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 106]
298MOCK: update_status: commit-queue Landed patch
299MOCK: update_status: commit-queue Pass
300MOCK: release_work_item: commit-queue 106
301""",
302            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '106' with comment 'Rejecting attachment 106 from commit-queue.' and additional comment 'Mock error message'\n",
303            "handle_script_error": "ScriptError error message\n",
304        }
305        self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=rollout_patch, expected_stderr=expected_stderr)
306
307    def test_auto_retry(self):
308        queue = CommitQueue()
309        options = Mock()
310        options.parent_command = "commit-queue"
311        tool = AlwaysCommitQueueTool()
312        sequence = NeedsUpdateSequence(None)
313
314        expected_stderr = "Commit failed because the checkout is out of date.  Please update and try again.\nMOCK: update_status: commit-queue Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests.\n"
315        state = {'patch': None}
316        OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options, state], expected_exception=TryAgain, expected_stderr=expected_stderr)
317
318        self.assertEquals(options.update, True)
319        self.assertEquals(options.build, False)
320        self.assertEquals(options.test, False)
321
322    def test_manual_reject_during_processing(self):
323        queue = SecondThoughtsCommitQueue()
324        queue.bind_to_tool(MockTool())
325        queue._tool.filesystem.write_text_file('/mock/results.html', '')  # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem.
326        queue._options = Mock()
327        queue._options.port = None
328        expected_stderr = """MOCK: update_status: commit-queue Cleaned working directory
329MOCK: update_status: commit-queue Updated working directory
330MOCK: update_status: commit-queue Applied patch
331MOCK: update_status: commit-queue Built patch
332MOCK: update_status: commit-queue Passed tests
333MOCK: update_status: commit-queue Retry
334MOCK: release_work_item: commit-queue 197
335"""
336        OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_stderr=expected_stderr)
337
338    def test_report_flaky_tests(self):
339        queue = CommitQueue()
340        queue.bind_to_tool(MockTool())
341        expected_stderr = """MOCK bug comment: bug_id=76, cc=None
342--- Begin comment ---
343The commit-queue just saw foo/bar.html flake (Text diff mismatch) while processing attachment 197 on bug 42.
344Port: MockPort  Platform: MockPlatform 1.0
345--- End comment ---
346
347MOCK add_attachment_to_bug: bug_id=76, description=Failure diff from bot filename=failure.diff
348MOCK bug comment: bug_id=76, cc=None
349--- Begin comment ---
350The commit-queue just saw bar/baz.html flake (Text diff mismatch) while processing attachment 197 on bug 42.
351Port: MockPort  Platform: MockPlatform 1.0
352--- End comment ---
353
354MOCK add_attachment_to_bug: bug_id=76, description=Archive of layout-test-results from bot filename=layout-test-results.zip
355MOCK bug comment: bug_id=42, cc=None
356--- Begin comment ---
357The commit-queue encountered the following flaky tests while processing attachment 197:
358
359foo/bar.html bug 76 (author: abarth@webkit.org)
360bar/baz.html bug 76 (author: abarth@webkit.org)
361The commit-queue is continuing to process your patch.
362--- End comment ---
363
364"""
365        test_names = ["foo/bar.html", "bar/baz.html"]
366        test_results = [self._mock_test_result(name) for name in test_names]
367
368        class MockZipFile(object):
369            def __init__(self):
370                self.fp = StringIO()
371
372            def read(self, path):
373                return ""
374
375            def namelist(self):
376                # This is intentionally missing one diffs.txt to exercise the "upload the whole zip" codepath.
377                return ['foo/bar-diffs.txt']
378
379        OutputCapture().assert_outputs(self, queue.report_flaky_tests, [QueuesTest.mock_work_item, test_results, MockZipFile()], expected_stderr=expected_stderr)
380
381    def test_missing_layout_test_results(self):
382        queue = CommitQueue()
383        tool = MockTool()
384        results_path = '/mock/results.html'
385        tool.filesystem = MockFileSystem({results_path: None})
386        queue.bind_to_tool(tool)
387        # Make sure that our filesystem mock functions as we expect.
388        self.assertRaises(IOError, tool.filesystem.read_text_file, results_path)
389        # layout_test_results shouldn't raise even if the results.html file is missing.
390        self.assertEquals(queue.layout_test_results(), None)
391
392    def test_layout_test_results(self):
393        queue = CommitQueue()
394        queue.bind_to_tool(MockTool())
395        queue._read_file_contents = lambda path: None
396        self.assertEquals(queue.layout_test_results(), None)
397        queue._read_file_contents = lambda path: ""
398        self.assertEquals(queue.layout_test_results(), None)
399        queue._create_layout_test_results = lambda: LayoutTestResults([])
400        results = queue.layout_test_results()
401        self.assertNotEquals(results, None)
402        self.assertEquals(results.failure_limit_count(), 10)  # This value matches RunTests.NON_INTERACTIVE_FAILURE_LIMIT_COUNT
403
404    def test_archive_last_layout_test_results(self):
405        queue = CommitQueue()
406        queue.bind_to_tool(MockTool())
407        patch = queue._tool.bugs.fetch_attachment(128)
408        # This is just to test that the method doesn't raise.
409        queue.archive_last_layout_test_results(patch)
410
411    def test_upload_results_archive_for_patch(self):
412        queue = CommitQueue()
413        queue.bind_to_tool(MockTool())
414        patch = queue._tool.bugs.fetch_attachment(128)
415        expected_stderr = """MOCK add_attachment_to_bug: bug_id=42, description=Archive of layout-test-results from bot filename=layout-test-results.zip
416-- Begin comment --
417The attached test failures were seen while running run-webkit-tests on the commit-queue.
418Port: MockPort  Platform: MockPlatform 1.0
419-- End comment --
420"""
421        OutputCapture().assert_outputs(self, queue._upload_results_archive_for_patch, [patch, Mock()], expected_stderr=expected_stderr)
422
423
424class StyleQueueTest(QueuesTest):
425    def test_style_queue(self):
426        expected_stderr = {
427            "begin_work_queue": self._default_begin_work_queue_stderr("style-queue", MockSCM.fake_checkout_root),
428            "next_work_item": "",
429            "should_proceed_with_work_item": "MOCK: update_status: style-queue Checking style\n",
430            "process_work_item": "MOCK: update_status: style-queue Pass\nMOCK: release_work_item: style-queue 197\n",
431            "handle_unexpected_error": "Mock error message\n",
432            "handle_script_error": "MOCK: update_status: style-queue ScriptError error message\nMOCK bug comment: bug_id=42, cc=[]\n--- Begin comment ---\nAttachment 197 did not pass style-queue:\n\nScriptError error message\n\nIf any of these errors are false positives, please file a bug against check-webkit-style.\n--- End comment ---\n\n",
433        }
434        expected_exceptions = {
435            "handle_script_error": SystemExit,
436        }
437        self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr, expected_exceptions=expected_exceptions)
438