1#!/usr/bin/python
2
3# Copyright (c) 2014 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8"""Generate new bench expectations from results of trybots on a code review."""
9
10
11import collections
12import compare_codereview
13import json
14import os
15import re
16import shutil
17import subprocess
18import sys
19import urllib2
20
21
22BENCH_DATA_URL = 'gs://chromium-skia-gm/perfdata/%s/%s/bench_*_data_*'
23BUILD_STATUS_SUCCESS = 0
24BUILD_STATUS_WARNINGS = 1
25CHECKOUT_PATH = os.path.realpath(os.path.join(
26    os.path.dirname(os.path.abspath(__file__)), os.pardir))
27TMP_BENCH_DATA_DIR = os.path.join(CHECKOUT_PATH, '.bench_data')
28
29
30TryBuild = collections.namedtuple(
31    'TryBuild', ['builder_name', 'build_number', 'is_finished', 'json_url'])
32
33
34def find_all_builds(codereview_url):
35  """Finds and returns information about trybot runs for a code review.
36
37  Args:
38    codereview_url: URL of the codereview in question.
39
40  Returns:
41      List of NamedTuples: (builder_name, build_number, is_finished)
42  """
43  results = compare_codereview.CodeReviewHTMLParser().parse(codereview_url)
44  try_builds = []
45  for builder, data in results.iteritems():
46    if builder.startswith('Perf'):
47      build_num = None
48      json_url = None
49      if data.url:
50        split_url = data.url.split('/')
51        build_num = split_url[-1]
52        split_url.insert(split_url.index('builders'), 'json')
53        json_url = '/'.join(split_url)
54      is_finished = (data.status not in ('pending', 'try-pending') and
55                     build_num is not None)
56      try_builds.append(TryBuild(builder_name=builder,
57                                 build_number=build_num,
58                                 is_finished=is_finished,
59                                 json_url=json_url))
60  return try_builds
61
62
63def _all_trybots_finished(try_builds):
64  """Return True iff all of the given try jobs have finished.
65
66  Args:
67      try_builds: list of TryBuild instances.
68
69  Returns:
70      True if all of the given try jobs have finished, otherwise False.
71  """
72  for try_build in try_builds:
73    if not try_build.is_finished:
74      return False
75  return True
76
77
78def all_trybots_finished(codereview_url):
79  """Return True iff all of the try jobs on the given codereview have finished.
80
81  Args:
82      codereview_url: string; URL of the codereview.
83
84  Returns:
85      True if all of the try jobs have finished, otherwise False.
86  """
87  return _all_trybots_finished(find_all_builds(codereview_url))
88
89
90def get_bench_data(builder, build_num, dest_dir):
91  """Download the bench data for the given builder at the given build_num.
92
93  Args:
94      builder: string; name of the builder.
95      build_num: string; build number.
96      dest_dir: string; destination directory for the bench data.
97  """
98  url = BENCH_DATA_URL % (builder, build_num)
99  subprocess.check_call(['gsutil', 'cp', '-R', url, dest_dir])
100
101
102def find_revision_from_downloaded_data(dest_dir):
103  """Finds the revision at which the downloaded data was generated.
104
105  Args:
106      dest_dir: string; directory holding the downloaded data.
107
108  Returns:
109      The revision (git commit hash) at which the downloaded data was
110      generated, or None if no revision can be found.
111  """
112  for data_file in os.listdir(dest_dir):
113    match = re.match('bench_(?P<revision>[0-9a-fA-F]{2,40})_data.*', data_file)
114    if match:
115      return match.group('revision')
116  return None
117
118
119class TrybotNotFinishedError(Exception):
120  pass
121
122
123def _step_succeeded(try_build, step_name):
124  """Return True if the given step succeeded and False otherwise.
125
126  This function talks to the build master's JSON interface, which is slow.
127
128  TODO(borenet): There are now a few places which talk to the master's JSON
129  interface. Maybe it'd be worthwhile to create a module which does this.
130
131  Args:
132      try_build: TryBuild instance; the build we're concerned about.
133      step_name: string; name of the step we're concerned about.
134  """
135  step_url = '/'.join((try_build.json_url, 'steps', step_name))
136  step_data = json.load(urllib2.urlopen(step_url))
137  # step_data['results'] may not be present if the step succeeded. If present,
138  # it is a list whose first element is a result code, per the documentation:
139  # http://docs.buildbot.net/latest/developer/results.html
140  result = step_data.get('results', [BUILD_STATUS_SUCCESS])[0]
141  if result in (BUILD_STATUS_SUCCESS, BUILD_STATUS_WARNINGS):
142    return True
143  return False
144
145
146def gen_bench_expectations_from_codereview(codereview_url,
147                                           error_on_unfinished=True,
148                                           error_on_try_failure=True):
149  """Generate bench expectations from a code review.
150
151  Scans the given code review for Perf trybot runs. Downloads the results of
152  finished trybots and uses them to generate new expectations for their
153  waterfall counterparts.
154
155  Args:
156      url: string; URL of the code review.
157      error_on_unfinished: bool; throw an error if any trybot has not finished.
158      error_on_try_failure: bool; throw an error if any trybot failed an
159          important step.
160  """
161  try_builds = find_all_builds(codereview_url)
162
163  # Verify that all trybots have finished running.
164  if error_on_unfinished and not _all_trybots_finished(try_builds):
165    raise TrybotNotFinishedError('Not all trybots have finished.')
166
167  failed_run = []
168  failed_data_pull = []
169  failed_gen_expectations = []
170
171  # Don't even try to do anything if BenchPictures, PostBench, or
172  # UploadBenchResults failed.
173  for try_build in try_builds:
174    for step in ('BenchPictures', 'PostBench', 'UploadBenchResults'):
175      if not _step_succeeded(try_build, step):
176        msg = '%s failed on %s!' % (step, try_build.builder_name)
177        if error_on_try_failure:
178          raise Exception(msg)
179        print 'WARNING: %s Skipping.' % msg
180        failed_run.append(try_build.builder_name)
181
182  if os.path.isdir(TMP_BENCH_DATA_DIR):
183    shutil.rmtree(TMP_BENCH_DATA_DIR)
184
185  for try_build in try_builds:
186    try_builder = try_build.builder_name
187
188    # Even if we're not erroring out on try failures, we can't generate new
189    # expectations for failed bots.
190    if try_builder in failed_run:
191      continue
192
193    builder = try_builder.replace('-Trybot', '')
194
195    # Download the data.
196    dest_dir = os.path.join(TMP_BENCH_DATA_DIR, builder)
197    os.makedirs(dest_dir)
198    try:
199      get_bench_data(try_builder, try_build.build_number, dest_dir)
200    except subprocess.CalledProcessError:
201      failed_data_pull.append(try_builder)
202      continue
203
204    # Find the revision at which the data was generated.
205    revision = find_revision_from_downloaded_data(dest_dir)
206    if not revision:
207      # If we can't find a revision, then something is wrong with the data we
208      # downloaded. Skip this builder.
209      failed_data_pull.append(try_builder)
210      continue
211
212    # Generate new expectations.
213    output_file = os.path.join(CHECKOUT_PATH, 'expectations', 'bench',
214                               'bench_expectations_%s.txt' % builder)
215    try:
216      subprocess.check_call(['python',
217                             os.path.join(CHECKOUT_PATH, 'bench',
218                                          'gen_bench_expectations.py'),
219                             '-b', builder, '-o', output_file,
220                             '-d', dest_dir, '-r', revision])
221    except subprocess.CalledProcessError:
222      failed_gen_expectations.append(builder)
223
224  failure = ''
225  if failed_data_pull:
226    failure += 'Failed to load data for: %s\n\n' % ','.join(failed_data_pull)
227  if failed_gen_expectations:
228    failure += 'Failed to generate expectations for: %s\n\n' % ','.join(
229        failed_gen_expectations)
230  if failure:
231    raise Exception(failure)
232
233
234if __name__ == '__main__':
235  gen_bench_expectations_from_codereview(sys.argv[1])
236
237