1# Copyright (c) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import codecs
30import logging
31import platform
32import os.path
33
34from webkitpy.common.net.layouttestresults import path_for_layout_test, LayoutTestResults
35from webkitpy.common.config import urls
36from webkitpy.tool.bot.botinfo import BotInfo
37from webkitpy.tool.grammar import plural, pluralize, join_with_separators
38
39_log = logging.getLogger(__name__)
40
41
42class FlakyTestReporter(object):
43    def __init__(self, tool, bot_name):
44        self._tool = tool
45        self._bot_name = bot_name
46        self._bot_info = BotInfo(tool)
47
48    def _author_emails_for_test(self, flaky_test):
49        test_path = path_for_layout_test(flaky_test)
50        commit_infos = self._tool.checkout().recent_commit_infos_for_files([test_path])
51        # This ignores authors which are not committers because we don't have their bugzilla_email.
52        return set([commit_info.author().bugzilla_email() for commit_info in commit_infos if commit_info.author()])
53
54    def _bugzilla_email(self):
55        # FIXME: This is kinda a funny way to get the bugzilla email,
56        # we could also just create a Credentials object directly
57        # but some of the Credentials logic is in bugzilla.py too...
58        self._tool.bugs.authenticate()
59        return self._tool.bugs.username
60
61    # FIXME: This should move into common.config
62    _bot_emails = set([
63        "commit-queue@webkit.org",  # commit-queue
64        "eseidel@chromium.org",  # old commit-queue
65        "webkit.review.bot@gmail.com",  # style-queue, sheriff-bot, CrLx/Gtk EWS
66        "buildbot@hotmail.com",  # Win EWS
67        # Mac EWS currently uses eric@webkit.org, but that's not normally a bot
68    ])
69
70    def _lookup_bug_for_flaky_test(self, flaky_test):
71        bugs = self._tool.bugs.queries.fetch_bugs_matching_search(search_string=flaky_test)
72        if not bugs:
73            return None
74        # Match any bugs which are from known bots or the email this bot is using.
75        allowed_emails = self._bot_emails | set([self._bugzilla_email])
76        bugs = filter(lambda bug: bug.reporter_email() in allowed_emails, bugs)
77        if not bugs:
78            return None
79        if len(bugs) > 1:
80            # FIXME: There are probably heuristics we could use for finding
81            # the right bug instead of the first, like open vs. closed.
82            _log.warn("Found %s %s matching '%s' filed by a bot, using the first." % (pluralize('bug', len(bugs)), [bug.id() for bug in bugs], flaky_test))
83        return bugs[0]
84
85    def _view_source_url_for_test(self, test_path):
86        return urls.view_source_url("LayoutTests/%s" % test_path)
87
88    def _create_bug_for_flaky_test(self, flaky_test, author_emails, latest_flake_message):
89        format_values = {
90            'test': flaky_test,
91            'authors': join_with_separators(sorted(author_emails)),
92            'flake_message': latest_flake_message,
93            'test_url': self._view_source_url_for_test(flaky_test),
94            'bot_name': self._bot_name,
95        }
96        title = "Flaky Test: %(test)s" % format_values
97        description = """This is an automatically generated bug from the %(bot_name)s.
98%(test)s has been flaky on the %(bot_name)s.
99
100%(test)s was authored by %(authors)s.
101%(test_url)s
102
103%(flake_message)s
104
105The bots will update this with information from each new failure.
106
107If you believe this bug to be fixed or invalid, feel free to close.  The bots will re-open if the flake re-occurs.
108
109If you would like to track this test fix with another bug, please close this bug as a duplicate.  The bots will follow the duplicate chain when making future comments.
110""" % format_values
111
112        master_flake_bug = 50856  # MASTER: Flaky tests found by the commit-queue
113        return self._tool.bugs.create_bug(title, description,
114            component="Tools / Tests",
115            cc=",".join(author_emails),
116            blocked="50856")
117
118    # This is over-engineered, but it makes for pretty bug messages.
119    def _optional_author_string(self, author_emails):
120        if not author_emails:
121            return ""
122        heading_string = plural('author') if len(author_emails) > 1 else 'author'
123        authors_string = join_with_separators(sorted(author_emails))
124        return " (%s: %s)" % (heading_string, authors_string)
125
126    def _latest_flake_message(self, flaky_result, patch):
127        failure_messages = [failure.message() for failure in flaky_result.failures]
128        flake_message = "The %s just saw %s flake (%s) while processing attachment %s on bug %s." % (self._bot_name, flaky_result.filename, ", ".join(failure_messages), patch.id(), patch.bug_id())
129        return "%s\n%s" % (flake_message, self._bot_info.summary_text())
130
131    def _results_diff_path_for_test(self, test_path):
132        # FIXME: This is a big hack.  We should get this path from results.json
133        # except that old-run-webkit-tests doesn't produce a results.json
134        # so we just guess at the file path.
135        (test_path_root, _) = os.path.splitext(test_path)
136        return "%s-diffs.txt" % test_path_root
137
138    def _follow_duplicate_chain(self, bug):
139        while bug.is_closed() and bug.duplicate_of():
140            bug = self._tool.bugs.fetch_bug(bug.duplicate_of())
141        return bug
142
143    # Maybe this logic should move into Bugzilla? a reopen=True arg to post_comment?
144    def _update_bug_for_flaky_test(self, bug, latest_flake_message):
145        if bug.is_closed():
146            self._tool.bugs.reopen_bug(bug.id(), latest_flake_message)
147        else:
148            self._tool.bugs.post_comment_to_bug(bug.id(), latest_flake_message)
149
150    # This method is needed because our archive paths include a leading tmp/layout-test-results
151    def _find_in_archive(self, path, archive):
152        for archived_path in archive.namelist():
153            # Archives are currently created with full paths.
154            if archived_path.endswith(path):
155                return archived_path
156        return None
157
158    def _attach_failure_diff(self, flake_bug_id, flaky_test, results_archive_zip):
159        results_diff_path = self._results_diff_path_for_test(flaky_test)
160        # Check to make sure that the path makes sense.
161        # Since we're not actually getting this path from the results.html
162        # there is a chance it's wrong.
163        bot_id = self._tool.status_server.bot_id or "bot"
164        archive_path = self._find_in_archive(results_diff_path, results_archive_zip)
165        if archive_path:
166            results_diff = results_archive_zip.read(archive_path)
167            description = "Failure diff from %s" % bot_id
168            self._tool.bugs.add_attachment_to_bug(flake_bug_id, results_diff, description, filename="failure.diff")
169        else:
170            _log.warn("%s does not exist in results archive, uploading entire archive." % results_diff_path)
171            description = "Archive of layout-test-results from %s" % bot_id
172            # results_archive is a ZipFile object, grab the File object (.fp) to pass to Mechanize for uploading.
173            results_archive_file = results_archive_zip.fp
174            # Rewind the file object to start (since Mechanize won't do that automatically)
175            # See https://bugs.webkit.org/show_bug.cgi?id=54593
176            results_archive_file.seek(0)
177            self._tool.bugs.add_attachment_to_bug(flake_bug_id, results_archive_file, description, filename="layout-test-results.zip")
178
179    def report_flaky_tests(self, patch, flaky_test_results, results_archive):
180        message = "The %s encountered the following flaky tests while processing attachment %s:\n\n" % (self._bot_name, patch.id())
181        for flaky_result in flaky_test_results:
182            flaky_test = flaky_result.filename
183            bug = self._lookup_bug_for_flaky_test(flaky_test)
184            latest_flake_message = self._latest_flake_message(flaky_result, patch)
185            author_emails = self._author_emails_for_test(flaky_test)
186            if not bug:
187                _log.info("Bug does not already exist for %s, creating." % flaky_test)
188                flake_bug_id = self._create_bug_for_flaky_test(flaky_test, author_emails, latest_flake_message)
189            else:
190                bug = self._follow_duplicate_chain(bug)
191                # FIXME: Ideally we'd only make one comment per flake, not two.  But that's not possible
192                # in all cases (e.g. when reopening), so for now file attachment and comment are separate.
193                self._update_bug_for_flaky_test(bug, latest_flake_message)
194                flake_bug_id = bug.id()
195
196            self._attach_failure_diff(flake_bug_id, flaky_test, results_archive)
197            message += "%s bug %s%s\n" % (flaky_test, flake_bug_id, self._optional_author_string(author_emails))
198
199        message += "The %s is continuing to process your patch." % self._bot_name
200        self._tool.bugs.post_comment_to_bug(patch.bug_id(), message)
201