1# Copyright (c) 2009 Google Inc. All rights reserved.
2# Copyright (c) 2009 Apple Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31from optparse import make_option
32
33from webkitpy.tool import steps
34
35from webkitpy.common.checkout.commitinfo import CommitInfo
36from webkitpy.common.config.committers import CommitterList
37from webkitpy.common.net.buildbot import BuildBot
38from webkitpy.common.net.regressionwindow import RegressionWindow
39from webkitpy.common.system.user import User
40from webkitpy.tool.grammar import pluralize
41from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
42from webkitpy.common.system.deprecated_logging import log
43from webkitpy.layout_tests import port
44
45
46class SuggestReviewers(AbstractDeclarativeCommand):
47    name = "suggest-reviewers"
48    help_text = "Suggest reviewers for a patch based on recent changes to the modified files."
49
50    def __init__(self):
51        options = [
52            steps.Options.git_commit,
53        ]
54        AbstractDeclarativeCommand.__init__(self, options=options)
55
56    def execute(self, options, args, tool):
57        reviewers = tool.checkout().suggested_reviewers(options.git_commit)
58        print "\n".join([reviewer.full_name for reviewer in reviewers])
59
60
61class BugsToCommit(AbstractDeclarativeCommand):
62    name = "bugs-to-commit"
63    help_text = "List bugs in the commit-queue"
64
65    def execute(self, options, args, tool):
66        # FIXME: This command is poorly named.  It's fetching the commit-queue list here.  The name implies it's fetching pending-commit (all r+'d patches).
67        bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue()
68        for bug_id in bug_ids:
69            print "%s" % bug_id
70
71
72class PatchesInCommitQueue(AbstractDeclarativeCommand):
73    name = "patches-in-commit-queue"
74    help_text = "List patches in the commit-queue"
75
76    def execute(self, options, args, tool):
77        patches = tool.bugs.queries.fetch_patches_from_commit_queue()
78        log("Patches in commit queue:")
79        for patch in patches:
80            print patch.url()
81
82
83class PatchesToCommitQueue(AbstractDeclarativeCommand):
84    name = "patches-to-commit-queue"
85    help_text = "Patches which should be added to the commit queue"
86    def __init__(self):
87        options = [
88            make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"),
89        ]
90        AbstractDeclarativeCommand.__init__(self, options=options)
91
92    @staticmethod
93    def _needs_commit_queue(patch):
94        if patch.commit_queue() == "+": # If it's already cq+, ignore the patch.
95            log("%s already has cq=%s" % (patch.id(), patch.commit_queue()))
96            return False
97
98        # We only need to worry about patches from contributers who are not yet committers.
99        committer_record = CommitterList().committer_by_email(patch.attacher_email())
100        if committer_record:
101            log("%s committer = %s" % (patch.id(), committer_record))
102        return not committer_record
103
104    def execute(self, options, args, tool):
105        patches = tool.bugs.queries.fetch_patches_from_pending_commit_list()
106        patches_needing_cq = filter(self._needs_commit_queue, patches)
107        if options.bugs:
108            bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq)
109            bugs_needing_cq = sorted(set(bugs_needing_cq))
110            for bug_id in bugs_needing_cq:
111                print "%s" % tool.bugs.bug_url_for_bug_id(bug_id)
112        else:
113            for patch in patches_needing_cq:
114                print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit")
115
116
117class PatchesToReview(AbstractDeclarativeCommand):
118    name = "patches-to-review"
119    help_text = "List patches that are pending review"
120
121    def execute(self, options, args, tool):
122        patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue()
123        log("Patches pending review:")
124        for patch_id in patch_ids:
125            print patch_id
126
127
128class LastGreenRevision(AbstractDeclarativeCommand):
129    name = "last-green-revision"
130    help_text = "Prints the last known good revision"
131
132    def execute(self, options, args, tool):
133        print self._tool.buildbot.last_green_revision()
134
135
136class WhatBroke(AbstractDeclarativeCommand):
137    name = "what-broke"
138    help_text = "Print failing buildbots (%s) and what revisions broke them" % BuildBot.default_host
139
140    def _print_builder_line(self, builder_name, max_name_width, status_message):
141        print "%s : %s" % (builder_name.ljust(max_name_width), status_message)
142
143    def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True):
144        builder = self._tool.buildbot.builder_with_name(builder_status["name"])
145        red_build = builder.build(builder_status["build_number"])
146        regression_window = builder.find_regression_window(red_build)
147        if not regression_window.failing_build():
148            self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)")
149            return
150        if not regression_window.build_before_failure():
151            self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % regression_window.failing_build().revision())
152            return
153
154        revisions = regression_window.revisions()
155        first_failure_message = ""
156        if (regression_window.failing_build() == builder.build(builder_status["build_number"])):
157            first_failure_message = " FIRST FAILURE, possibly a flaky test"
158        self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (revisions, first_failure_message))
159        for revision in revisions:
160            commit_info = self._tool.checkout().commit_info_for_revision(revision)
161            if commit_info:
162                print commit_info.blame_string(self._tool.bugs)
163            else:
164                print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
165
166    def execute(self, options, args, tool):
167        builder_statuses = tool.buildbot.builder_statuses()
168        longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses)))
169        failing_builders = 0
170        for builder_status in builder_statuses:
171            # If the builder is green, print OK, exit.
172            if builder_status["is_green"]:
173                continue
174            self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name)
175            failing_builders += 1
176        if failing_builders:
177            print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses)))
178        else:
179            print "All builders are passing!"
180
181
182class ResultsFor(AbstractDeclarativeCommand):
183    name = "results-for"
184    help_text = "Print a list of failures for the passed revision from bots on %s" % BuildBot.default_host
185    argument_names = "REVISION"
186
187    def _print_layout_test_results(self, results):
188        if not results:
189            print " No results."
190            return
191        for title, files in results.parsed_results().items():
192            print " %s" % title
193            for filename in files:
194                print "  %s" % filename
195
196    def execute(self, options, args, tool):
197        builders = self._tool.buildbot.builders()
198        for builder in builders:
199            print "%s:" % builder.name()
200            build = builder.build_for_revision(args[0], allow_failed_lookups=True)
201            self._print_layout_test_results(build.layout_test_results())
202
203
204class FailureReason(AbstractDeclarativeCommand):
205    name = "failure-reason"
206    help_text = "Lists revisions where individual test failures started at %s" % BuildBot.default_host
207
208    def _blame_line_for_revision(self, revision):
209        try:
210            commit_info = self._tool.checkout().commit_info_for_revision(revision)
211        except Exception, e:
212            return "FAILED to fetch CommitInfo for r%s, exception: %s" % (revision, e)
213        if not commit_info:
214            return "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision
215        return commit_info.blame_string(self._tool.bugs)
216
217    def _print_blame_information_for_transition(self, regression_window, failing_tests):
218        red_build = regression_window.failing_build()
219        print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests)
220        print "Suspect revisions:"
221        for revision in regression_window.revisions():
222            print self._blame_line_for_revision(revision)
223
224    def _explain_failures_for_builder(self, builder, start_revision):
225        print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision)
226        revision_to_test = start_revision
227        build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
228        layout_test_results = build.layout_test_results()
229        if not layout_test_results:
230            # FIXME: This could be made more user friendly.
231            print "Failed to load layout test results; can't continue. (start revision = r%s)" % start_revision
232            return 1
233
234        results_to_explain = set(layout_test_results.failing_tests())
235        last_build_with_results = build
236        print "Starting at %s" % revision_to_test
237        while results_to_explain:
238            revision_to_test -= 1
239            new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True)
240            if not new_build:
241                print "No build for %s" % revision_to_test
242                continue
243            build = new_build
244            latest_results = build.layout_test_results()
245            if not latest_results:
246                print "No results build %s (r%s)" % (build._number, build.revision())
247                continue
248            failures = set(latest_results.failing_tests())
249            if len(failures) >= 20:
250                # FIXME: We may need to move this logic into the LayoutTestResults class.
251                # The buildbot stops runs after 20 failures so we don't have full results to work with here.
252                print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
253                continue
254            fixed_results = results_to_explain - failures
255            if not fixed_results:
256                print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures))
257                last_build_with_results = build
258                continue
259            regression_window = RegressionWindow(build, last_build_with_results)
260            self._print_blame_information_for_transition(regression_window, fixed_results)
261            last_build_with_results = build
262            results_to_explain -= fixed_results
263        if results_to_explain:
264            print "Failed to explain failures: %s" % results_to_explain
265            return 1
266        print "Explained all results for %s" % builder.name()
267        return 0
268
269    def _builder_to_explain(self):
270        builder_statuses = self._tool.buildbot.builder_statuses()
271        red_statuses = [status for status in builder_statuses if not status["is_green"]]
272        print "%s failing" % (pluralize("builder", len(red_statuses)))
273        builder_choices = [status["name"] for status in red_statuses]
274        # We could offer an "All" choice here.
275        chosen_name = self._tool.user.prompt_with_list("Which builder to diagnose:", builder_choices)
276        # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
277        for status in red_statuses:
278            if status["name"] == chosen_name:
279                return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
280
281    def execute(self, options, args, tool):
282        (builder, latest_revision) = self._builder_to_explain()
283        start_revision = self._tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision
284        if not start_revision:
285            print "Revision required."
286            return 1
287        return self._explain_failures_for_builder(builder, start_revision=int(start_revision))
288
289
290class FindFlakyTests(AbstractDeclarativeCommand):
291    name = "find-flaky-tests"
292    help_text = "Lists tests that often fail for a single build at %s" % BuildBot.default_host
293
294    def _find_failures(self, builder, revision):
295        build = builder.build_for_revision(revision, allow_failed_lookups=True)
296        if not build:
297            print "No build for %s" % revision
298            return (None, None)
299        results = build.layout_test_results()
300        if not results:
301            print "No results build %s (r%s)" % (build._number, build.revision())
302            return (None, None)
303        failures = set(results.failing_tests())
304        if len(failures) >= 20:
305            # FIXME: We may need to move this logic into the LayoutTestResults class.
306            # The buildbot stops runs after 20 failures so we don't have full results to work with here.
307            print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision())
308            return (None, None)
309        return (build, failures)
310
311    def _increment_statistics(self, flaky_tests, flaky_test_statistics):
312        for test in flaky_tests:
313            count = flaky_test_statistics.get(test, 0)
314            flaky_test_statistics[test] = count + 1
315
316    def _print_statistics(self, statistics):
317        print "=== Results ==="
318        print "Occurances Test name"
319        for value, key in sorted([(value, key) for key, value in statistics.items()]):
320            print "%10d %s" % (value, key)
321
322    def _walk_backwards_from(self, builder, start_revision, limit):
323        flaky_test_statistics = {}
324        all_previous_failures = set([])
325        one_time_previous_failures = set([])
326        previous_build = None
327        for i in range(limit):
328            revision = start_revision - i
329            print "Analyzing %s ... " % revision,
330            (build, failures) = self._find_failures(builder, revision)
331            if failures == None:
332                # Notice that we don't loop on the empty set!
333                continue
334            print "has %s failures" % len(failures)
335            flaky_tests = one_time_previous_failures - failures
336            if flaky_tests:
337                print "Flaky tests: %s %s" % (sorted(flaky_tests),
338                                              previous_build.results_url())
339            self._increment_statistics(flaky_tests, flaky_test_statistics)
340            one_time_previous_failures = failures - all_previous_failures
341            all_previous_failures = failures
342            previous_build = build
343        self._print_statistics(flaky_test_statistics)
344
345    def _builder_to_analyze(self):
346        statuses = self._tool.buildbot.builder_statuses()
347        choices = [status["name"] for status in statuses]
348        chosen_name = self._tool.user.prompt_with_list("Which builder to analyze:", choices)
349        for status in statuses:
350            if status["name"] == chosen_name:
351                return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
352
353    def execute(self, options, args, tool):
354        (builder, latest_revision) = self._builder_to_analyze()
355        limit = self._tool.user.prompt("How many revisions to look through? [10000] ") or 10000
356        return self._walk_backwards_from(builder, latest_revision, limit=int(limit))
357
358
359class TreeStatus(AbstractDeclarativeCommand):
360    name = "tree-status"
361    help_text = "Print the status of the %s buildbots" % BuildBot.default_host
362    long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder
363and displayes the status of each builder."""
364
365    def execute(self, options, args, tool):
366        for builder in tool.buildbot.builder_statuses():
367            status_string = "ok" if builder["is_green"] else "FAIL"
368            print "%s : %s" % (status_string.ljust(4), builder["name"])
369
370
371class SkippedPorts(AbstractDeclarativeCommand):
372    name = "skipped-ports"
373    help_text = "Print the list of ports skipping the given layout test(s)"
374    long_help = """Scans the the Skipped file of each port and figure
375out what ports are skipping the test(s). Categories are taken in account too."""
376    argument_names = "TEST_NAME"
377
378    def execute(self, options, args, tool):
379        results = dict([(test_name, []) for test_name in args])
380        for port_name, port_object in tool.port_factory.get_all().iteritems():
381            for test_name in args:
382                if port_object.skips_layout_test(test_name):
383                    results[test_name].append(port_name)
384
385        for test_name, ports in results.iteritems():
386            if ports:
387                print "Ports skipping test %r: %s" % (test_name, ', '.join(ports))
388            else:
389                print "Test %r is not skipped by any port." % test_name
390