1# Copyright (c) 2011 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import os
30import optparse
31from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
32from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
33from webkitpy.layout_tests.models.test_expectations import TestExpectationParser, TestExpectationsModel, TestExpectations
34from webkitpy.layout_tests.port import builders
35from webkitpy.common.net import sheriff_calendar
36
37
38class FlakyTests(AbstractDeclarativeCommand):
39    name = "update-flaky-tests"
40    help_text = "Update FlakyTests file from the flakiness dashboard"
41    show_in_main_help = True
42
43    ALWAYS_CC = [
44        'ojan@chromium.org',
45        'dpranke@chromium.org',
46        'eseidel@chromium.org',
47    ]
48
49    COMMIT_MESSAGE = (
50        'Update FlakyTests to match current flakiness dashboard results\n\n'
51        'Automatically generated using:\n'
52        'webkit-patch update-flaky-tests\n\n'
53        'R=%s\n')
54
55    FLAKY_TEST_CONTENTS = (
56        '# This file is generated by webkit-patch update-flaky-tests from the flakiness dashboard data.\n'
57        '# Manual changes will be overwritten.\n\n'
58        '%s\n')
59
60    def __init__(self):
61        options = [
62            optparse.make_option('--upload', action='store_true',
63                help='upload the changed FlakyTest file for review'),
64            optparse.make_option('--reviewers', action='store',
65                help='comma-separated list of reviewers, defaults to blink gardeners'),
66        ]
67        AbstractDeclarativeCommand.__init__(self, options=options)
68        # This is sorta silly, but allows for unit testing:
69        self.expectations_factory = BotTestExpectationsFactory
70
71    def _collect_expectation_lines(self, builder_names, factory):
72        all_lines = []
73        for builder_name in builder_names:
74            model = TestExpectationsModel()
75            expectations = factory.expectations_for_builder(builder_name)
76            for line in expectations.expectation_lines(only_ignore_very_flaky=True):
77                model.add_expectation_line(line)
78            # FIXME: We need an official API to get all the test names or all test lines.
79            all_lines.extend(model._test_to_expectation_line.values())
80        return all_lines
81
82    def _commit_and_upload(self, tool, options):
83        files = tool.scm().changed_files()
84        flaky_tests_path = 'LayoutTests/FlakyTests'
85        if flaky_tests_path not in files:
86            print "%s is not changed, not uploading." % flaky_tests_path
87            return 0
88
89        if options.reviewers:
90            # FIXME: Could validate these as emails. sheriff_calendar has some code for that.
91            reviewer_emails = options.reviewers.split(',')
92        else:
93            reviewer_emails = sheriff_calendar.current_gardener_emails()
94            if not reviewer_emails:
95                print "No gardener, and --reviewers not specified, not bothering."
96                return 1
97
98        commit_message = self.COMMIT_MESSAGE % ','.join(reviewer_emails)
99        git_cmd = ['git', 'commit', '-m', commit_message,
100            tool.filesystem.join(tool.scm().checkout_root, flaky_tests_path)]
101        tool.executive.run_and_throw_if_fail(git_cmd)
102
103        git_cmd = ['git', 'cl', 'upload', '--send-mail', '-f',
104            '--cc', ','.join(self.ALWAYS_CC)]
105        tool.executive.run_and_throw_if_fail(git_cmd)
106
107    def execute(self, options, args, tool):
108        factory = self.expectations_factory()
109
110        # FIXME: WebKit Linux 32 and WebKit Linux have the same specifiers;
111        # if we include both of them, we'll get duplicate lines. Ideally
112        # Linux 32 would have unique speicifiers.
113        most_builders = builders.all_builder_names()
114        if 'WebKit Linux 32' in most_builders:
115            most_builders.remove('WebKit Linux 32')
116
117        lines = self._collect_expectation_lines(most_builders, factory)
118        lines.sort(key=lambda line: line.path)
119
120        port = tool.port_factory.get()
121        # Skip any tests which are mentioned in the dashboard but not in our checkout:
122        fs = tool.filesystem
123        lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), line.path)), lines)
124
125        # Note: This includes all flaky tests from the dashboard, even ones mentioned
126        # in existing TestExpectations. We could certainly load existing TestExpecations
127        # and filter accordingly, or update existing TestExpectations instead of FlakyTests.
128        flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests')
129        flaky_tests_contents = self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines)
130        fs.write_text_file(flaky_tests_path, flaky_tests_contents)
131        print "Updated %s" % flaky_tests_path
132
133        if options.upload:
134            return self._commit_and_upload(tool, options)
135
136