1# Copyright (c) 2011 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import os 30import optparse 31from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 32from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory 33from webkitpy.layout_tests.models.test_expectations import TestExpectationParser, TestExpectationsModel, TestExpectations 34from webkitpy.layout_tests.port import builders 35from webkitpy.common.net import sheriff_calendar 36 37 38class FlakyTests(AbstractDeclarativeCommand): 39 name = "update-flaky-tests" 40 help_text = "Update FlakyTests file from the flakiness dashboard" 41 show_in_main_help = True 42 43 ALWAYS_CC = [ 44 'ojan@chromium.org', 45 'dpranke@chromium.org', 46 'eseidel@chromium.org', 47 ] 48 49 COMMIT_MESSAGE = ( 50 'Update FlakyTests to match current flakiness dashboard results\n\n' 51 'Automatically generated using:\n' 52 'webkit-patch update-flaky-tests\n\n' 53 'R=%s\n') 54 55 FLAKY_TEST_CONTENTS = ( 56 '# This file is generated by webkit-patch update-flaky-tests from the flakiness dashboard data.\n' 57 '# Manual changes will be overwritten.\n\n' 58 '%s\n') 59 60 def __init__(self): 61 options = [ 62 optparse.make_option('--upload', action='store_true', 63 help='upload the changed FlakyTest file for review'), 64 optparse.make_option('--reviewers', action='store', 65 help='comma-separated list of reviewers, defaults to blink gardeners'), 66 ] 67 AbstractDeclarativeCommand.__init__(self, options=options) 68 # This is sorta silly, but allows for unit testing: 69 self.expectations_factory = BotTestExpectationsFactory 70 71 def _collect_expectation_lines(self, builder_names, factory): 72 all_lines = [] 73 for builder_name in builder_names: 74 model = TestExpectationsModel() 75 expectations = factory.expectations_for_builder(builder_name) 76 for line in expectations.expectation_lines(only_ignore_very_flaky=True): 77 model.add_expectation_line(line) 78 # FIXME: We need an official API to get all the test names or all test lines. 79 all_lines.extend(model._test_to_expectation_line.values()) 80 return all_lines 81 82 def _commit_and_upload(self, tool, options): 83 files = tool.scm().changed_files() 84 flaky_tests_path = 'LayoutTests/FlakyTests' 85 if flaky_tests_path not in files: 86 print "%s is not changed, not uploading." % flaky_tests_path 87 return 0 88 89 if options.reviewers: 90 # FIXME: Could validate these as emails. sheriff_calendar has some code for that. 91 reviewer_emails = options.reviewers.split(',') 92 else: 93 reviewer_emails = sheriff_calendar.current_gardener_emails() 94 if not reviewer_emails: 95 print "No gardener, and --reviewers not specified, not bothering." 96 return 1 97 98 commit_message = self.COMMIT_MESSAGE % ','.join(reviewer_emails) 99 git_cmd = ['git', 'commit', '-m', commit_message, 100 tool.filesystem.join(tool.scm().checkout_root, flaky_tests_path)] 101 tool.executive.run_and_throw_if_fail(git_cmd) 102 103 git_cmd = ['git', 'cl', 'upload', '--send-mail', '-f', 104 '--cc', ','.join(self.ALWAYS_CC)] 105 tool.executive.run_and_throw_if_fail(git_cmd) 106 107 def execute(self, options, args, tool): 108 factory = self.expectations_factory() 109 110 # FIXME: WebKit Linux 32 and WebKit Linux have the same specifiers; 111 # if we include both of them, we'll get duplicate lines. Ideally 112 # Linux 32 would have unique speicifiers. 113 most_builders = builders.all_builder_names() 114 if 'WebKit Linux 32' in most_builders: 115 most_builders.remove('WebKit Linux 32') 116 117 lines = self._collect_expectation_lines(most_builders, factory) 118 lines.sort(key=lambda line: line.path) 119 120 port = tool.port_factory.get() 121 # Skip any tests which are mentioned in the dashboard but not in our checkout: 122 fs = tool.filesystem 123 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), line.path)), lines) 124 125 # Note: This includes all flaky tests from the dashboard, even ones mentioned 126 # in existing TestExpectations. We could certainly load existing TestExpecations 127 # and filter accordingly, or update existing TestExpectations instead of FlakyTests. 128 flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests') 129 flaky_tests_contents = self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines) 130 fs.write_text_file(flaky_tests_path, flaky_tests_contents) 131 print "Updated %s" % flaky_tests_path 132 133 if options.upload: 134 return self._commit_and_upload(tool, options) 135 136