1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Run Performance Test Bisect Tool
7
8This script is used by a trybot to run the src/tools/bisect-perf-regression.py
9script with the parameters specified in run-bisect-perf-regression.cfg. It will
10check out a copy of the depot in a subdirectory 'bisect' of the working
11directory provided, and run the bisect-perf-regression.py script there.
12
13"""
14
15import imp
16import optparse
17import os
18import subprocess
19import sys
20import traceback
21
22import bisect_utils
23bisect = imp.load_source('bisect-perf-regression',
24    os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
25        'bisect-perf-regression.py'))
26
27
28CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
29CROS_IP_ENV = 'BISECT_CROS_IP'
30
31
32class Goma(object):
33
34  def __init__(self, path_to_goma):
35    self._abs_path_to_goma = None
36    self._abs_path_to_goma_file = None
37    if path_to_goma:
38      self._abs_path_to_goma = os.path.abspath(path_to_goma)
39      self._abs_path_to_goma_file = self._GetExecutablePath(
40          self._abs_path_to_goma)
41
42  def __enter__(self):
43    if self._HasGOMAPath():
44      self._SetupAndStart()
45    return self
46
47  def __exit__(self, *_):
48    if self._HasGOMAPath():
49      self._Stop()
50
51  def _HasGOMAPath(self):
52    return bool(self._abs_path_to_goma)
53
54  def _GetExecutablePath(self, path_to_goma):
55    if os.name == 'nt':
56      return os.path.join(path_to_goma, 'goma_ctl.bat')
57    else:
58      return os.path.join(path_to_goma, 'goma_ctl.sh')
59
60  def _SetupEnvVars(self):
61    if os.name == 'nt':
62      os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
63          ' cl.exe')
64      os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
65          ' cl.exe')
66    else:
67      os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
68          os.environ['PATH']])
69
70  def _SetupAndStart(self):
71    """Sets up GOMA and launches it.
72
73    Args:
74      path_to_goma: Path to goma directory.
75
76    Returns:
77      True if successful."""
78    self._SetupEnvVars()
79
80    # Sometimes goma is lingering around if something went bad on a previous
81    # run. Stop it before starting a new process. Can ignore the return code
82    # since it will return an error if it wasn't running.
83    self._Stop()
84
85    if subprocess.call([self._abs_path_to_goma_file, 'start']):
86      raise RuntimeError('GOMA failed to start.')
87
88  def _Stop(self):
89    subprocess.call([self._abs_path_to_goma_file, 'stop'])
90
91
92
93def _LoadConfigFile(path_to_file):
94  """Attempts to load the specified config file as a module
95  and grab the global config dict.
96
97  Args:
98    path_to_file: Path to the file.
99
100  Returns:
101    The config dict which should be formatted as follows:
102    {'command': string, 'good_revision': string, 'bad_revision': string
103     'metric': string, etc...}.
104    Returns None on failure.
105  """
106  try:
107    local_vars = {}
108    execfile(path_to_file, local_vars)
109
110    return local_vars['config']
111  except:
112    print
113    traceback.print_exc()
114    print
115    return {}
116
117
118def _OutputFailedResults(text_to_print):
119  bisect_utils.OutputAnnotationStepStart('Results - Failed')
120  print
121  print text_to_print
122  print
123  bisect_utils.OutputAnnotationStepClosed()
124
125
126def _CreateBisectOptionsFromConfig(config):
127  opts_dict = {}
128  opts_dict['command'] = config['command']
129  opts_dict['metric'] = config['metric']
130
131  if config['repeat_count']:
132    opts_dict['repeat_test_count'] = int(config['repeat_count'])
133
134  if config['truncate_percent']:
135    opts_dict['truncate_percent'] = int(config['truncate_percent'])
136
137  if config['max_time_minutes']:
138    opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
139
140  if config.has_key('use_goma'):
141    opts_dict['use_goma'] = config['use_goma']
142
143  opts_dict['build_preference'] = 'ninja'
144  opts_dict['output_buildbot_annotations'] = True
145
146  if '--browser=cros' in config['command']:
147    opts_dict['target_platform'] = 'cros'
148
149    if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
150      opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
151      opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
152    else:
153      raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
154          'BISECT_CROS_BOARD undefined.')
155  elif 'android' in config['command']:
156    if 'android-chrome' in config['command']:
157      opts_dict['target_platform'] = 'android-chrome'
158    else:
159      opts_dict['target_platform'] = 'android'
160
161  return bisect.BisectOptions.FromDict(opts_dict)
162
163
164def _RunPerformanceTest(config, path_to_file):
165  # Bisect script expects to be run from src
166  os.chdir(os.path.join(path_to_file, '..'))
167
168  bisect_utils.OutputAnnotationStepStart('Building With Patch')
169
170  opts = _CreateBisectOptionsFromConfig(config)
171  b = bisect.BisectPerformanceMetrics(None, opts)
172
173  if bisect_utils.RunGClient(['runhooks']):
174    raise RuntimeError('Failed to run gclient runhooks')
175
176  if not b.BuildCurrentRevision('chromium'):
177    raise RuntimeError('Patched version failed to build.')
178
179  bisect_utils.OutputAnnotationStepClosed()
180  bisect_utils.OutputAnnotationStepStart('Running With Patch')
181
182  results_with_patch = b.RunPerformanceTestAndParseResults(
183      opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
184
185  if results_with_patch[1]:
186    raise RuntimeError('Patched version failed to run performance test.')
187
188  bisect_utils.OutputAnnotationStepClosed()
189
190  bisect_utils.OutputAnnotationStepStart('Reverting Patch')
191  if bisect_utils.RunGClient(['revert']):
192    raise RuntimeError('Failed to run gclient runhooks')
193  bisect_utils.OutputAnnotationStepClosed()
194
195  bisect_utils.OutputAnnotationStepStart('Building Without Patch')
196
197  if bisect_utils.RunGClient(['runhooks']):
198    raise RuntimeError('Failed to run gclient runhooks')
199
200  if not b.BuildCurrentRevision('chromium'):
201    raise RuntimeError('Unpatched version failed to build.')
202
203  bisect_utils.OutputAnnotationStepClosed()
204  bisect_utils.OutputAnnotationStepStart('Running Without Patch')
205
206  results_without_patch = b.RunPerformanceTestAndParseResults(
207      opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
208
209  if results_without_patch[1]:
210    raise RuntimeError('Unpatched version failed to run performance test.')
211
212  # Find the link to the cloud stored results file.
213  output = results_without_patch[2]
214  cloud_file_link = [t for t in output.splitlines()
215      if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
216  if cloud_file_link:
217    # What we're getting here is basically "View online at http://..." so parse
218    # out just the url portion.
219    cloud_file_link = cloud_file_link[0]
220    cloud_file_link = [t for t in cloud_file_link.split(' ')
221        if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
222    assert cloud_file_link, "Couldn't parse url from output."
223    cloud_file_link = cloud_file_link[0]
224  else:
225    cloud_file_link = ''
226
227  # Calculate the % difference in the means of the 2 runs.
228  percent_diff_in_means = (results_with_patch[0]['mean'] /
229      max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
230  std_err = bisect.CalculatePooledStandardError(
231      [results_with_patch[0]['values'], results_without_patch[0]['values']])
232
233  bisect_utils.OutputAnnotationStepClosed()
234  bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
235      (percent_diff_in_means, std_err))
236  print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
237      'Std. Error'.center(20, ' '))
238  print ' %s %s %s' % ('Patch'.center(10, ' '),
239      ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
240      ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
241  print ' %s %s %s' % ('No Patch'.center(10, ' '),
242      ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
243      ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
244  if cloud_file_link:
245    bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
246  bisect_utils.OutputAnnotationStepClosed()
247
248
249def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
250  """Attempts to build and run the current revision with and without the
251  current patch, with the parameters passed in.
252
253  Args:
254    config: The config read from run-perf-test.cfg.
255    path_to_file: Path to the bisect-perf-regression.py script.
256    path_to_goma: Path to goma directory.
257
258  Returns:
259    0 on success, otherwise 1.
260  """
261  try:
262    with Goma(path_to_goma) as goma:
263      config['use_goma'] = bool(path_to_goma)
264      _RunPerformanceTest(config, path_to_file)
265    return 0
266  except RuntimeError, e:
267    bisect_utils.OutputAnnotationStepClosed()
268    _OutputFailedResults('Error: %s' % e.message)
269    return 1
270
271
272def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
273    path_to_extra_src, dry_run):
274  """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
275  passed in.
276
277  Args:
278    config: A dict containing the parameters to pass to the script.
279    working_directory: A working directory to provide to the
280      bisect-perf-regression.py script, where it will store it's own copy of
281      the depot.
282    path_to_file: Path to the bisect-perf-regression.py script.
283    path_to_goma: Path to goma directory.
284    path_to_extra_src: Path to extra source file.
285    dry_run: Do a dry run, skipping sync, build, and performance testing steps.
286
287  Returns:
288    0 on success, otherwise 1.
289  """
290  bisect_utils.OutputAnnotationStepStart('Config')
291  print
292  for k, v in config.iteritems():
293    print '  %s : %s' % (k, v)
294  print
295  bisect_utils.OutputAnnotationStepClosed()
296
297  cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
298         '-c', config['command'],
299         '-g', config['good_revision'],
300         '-b', config['bad_revision'],
301         '-m', config['metric'],
302         '--working_directory', working_directory,
303         '--output_buildbot_annotations']
304
305  if config['repeat_count']:
306    cmd.extend(['-r', config['repeat_count']])
307
308  if config['truncate_percent']:
309    cmd.extend(['-t', config['truncate_percent']])
310
311  if config['max_time_minutes']:
312    cmd.extend(['--max_time_minutes', config['max_time_minutes']])
313
314  cmd.extend(['--build_preference', 'ninja'])
315
316  if '--browser=cros' in config['command']:
317    cmd.extend(['--target_platform', 'cros'])
318
319    if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
320      cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
321      cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
322    else:
323      print 'Error: Cros build selected, but BISECT_CROS_IP or'\
324            'BISECT_CROS_BOARD undefined.'
325      print
326      return 1
327
328  if 'android' in config['command']:
329    if 'android-chrome' in config['command']:
330      cmd.extend(['--target_platform', 'android-chrome'])
331    else:
332      cmd.extend(['--target_platform', 'android'])
333
334  if path_to_goma:
335    cmd.append('--use_goma')
336
337  if path_to_extra_src:
338    cmd.extend(['--extra_src', path_to_extra_src])
339
340  if dry_run:
341    cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
342        '--debug_ignore_perf_test'])
343  cmd = [str(c) for c in cmd]
344
345  with Goma(path_to_goma) as goma:
346    return_code = subprocess.call(cmd)
347
348  if return_code:
349    print 'Error: bisect-perf-regression.py returned with error %d' %\
350        return_code
351    print
352
353  return return_code
354
355
356def main():
357
358  usage = ('%prog [options] [-- chromium-options]\n'
359           'Used by a trybot to run the bisection script using the parameters'
360           ' provided in the run-bisect-perf-regression.cfg file.')
361
362  parser = optparse.OptionParser(usage=usage)
363  parser.add_option('-w', '--working_directory',
364                    type='str',
365                    help='A working directory to supply to the bisection '
366                    'script, which will use it as the location to checkout '
367                    'a copy of the chromium depot.')
368  parser.add_option('-p', '--path_to_goma',
369                    type='str',
370                    help='Path to goma directory. If this is supplied, goma '
371                    'builds will be enabled.')
372  parser.add_option('--extra_src',
373                    type='str',
374                    help='Path to extra source file. If this is supplied, '
375                    'bisect script will use this to override default behavior.')
376  parser.add_option('--dry_run',
377                    action="store_true",
378                    help='The script will perform the full bisect, but '
379                    'without syncing, building, or running the performance '
380                    'tests.')
381  (opts, args) = parser.parse_args()
382
383  path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
384  path_to_bisect_cfg = os.path.join(path_to_current_directory,
385      'run-bisect-perf-regression.cfg')
386
387  config = _LoadConfigFile(path_to_bisect_cfg)
388
389  # Check if the config is empty
390  config_has_values = [v for v in config.values() if v]
391
392  if config and config_has_values:
393    if not opts.working_directory:
394      print 'Error: missing required parameter: --working_directory'
395      print
396      parser.print_help()
397      return 1
398
399    return _RunBisectionScript(config, opts.working_directory,
400        path_to_current_directory, opts.path_to_goma, opts.extra_src,
401        opts.dry_run)
402  else:
403    perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party',
404        'WebKit', 'Tools', 'run-perf-test.cfg')]
405
406    for current_perf_cfg_file in perf_cfg_files:
407      path_to_perf_cfg = os.path.join(
408          os.path.abspath(os.path.dirname(sys.argv[0])), current_perf_cfg_file)
409
410      config = _LoadConfigFile(path_to_perf_cfg)
411      config_has_values = [v for v in config.values() if v]
412
413      if config and config_has_values:
414        return _SetupAndRunPerformanceTest(config, path_to_current_directory,
415            opts.path_to_goma)
416
417    print 'Error: Could not load config file. Double check your changes to '\
418          'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
419    print
420    return 1
421
422
423if __name__ == '__main__':
424  sys.exit(main())
425