1#!/usr/bin/env python
2
3
4# Copyright (c) 2014 The Chromium Authors. All rights reserved.
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8
9"""greenify.py: standalone script to correct flaky bench expectations.
10
11    Requires Rietveld credentials on the running machine.
12
13    Usage:
14      Copy script to a separate dir outside Skia repo. The script will create a
15          skia dir on the first run to host the repo, and will create/delete
16          temp dirs as needed.
17      ./greenify.py --url <the stdio url from failed CheckForRegressions step>
18"""
19
20import argparse
21import filecmp
22import os
23import re
24import shutil
25import subprocess
26import time
27import urllib2
28
29
30# Regular expression for matching exception data.
31EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) '
32                'vs (\d+.\d+), ')
33EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE)
34
35
36def clean_dir(d):
37  if os.path.exists(d):
38    shutil.rmtree(d)
39  os.makedirs(d)
40
41def checkout_or_update_skia(repo_dir):
42  status = True
43  old_cwd = os.getcwd()
44  os.chdir(repo_dir)
45  print 'CHECK SKIA REPO...'
46  if subprocess.call(['git', 'pull'],
47                     stderr=subprocess.PIPE):
48    print 'Checking out Skia from git, please be patient...'
49    os.chdir(old_cwd)
50    clean_dir(repo_dir)
51    os.chdir(repo_dir)
52    if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch',
53                        'https://skia.googlesource.com/skia.git', '.']):
54      status = False
55  subprocess.call(['git', 'checkout', 'master'])
56  subprocess.call(['git', 'pull'])
57  os.chdir(old_cwd)
58  return status
59
60def git_commit_expectations(repo_dir, exp_dir, bot, build, commit):
61  commit_msg = """Greenify bench bot %s at build %s
62
63TBR=bsalomon@google.com
64
65Bypassing trybots:
66NOTRY=true""" % (bot, build)
67  old_cwd = os.getcwd()
68  os.chdir(repo_dir)
69  upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks',
70            '--bypass-watchlists', '-m', commit_msg]
71  if commit:
72    upload.append('--use-commit-queue')
73  branch = exp_dir[exp_dir.rfind('/') + 1:]
74  filename = 'bench_expectations_%s.txt' % bot
75  cmds = ([['git', 'checkout', 'master'],
76           ['git', 'pull'],
77           ['git', 'checkout', '-b', branch, '-t', 'origin/master'],
78           ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'],
79           ['git', 'add', 'expectations/bench/' + filename],
80           ['git', 'commit', '-m', commit_msg],
81           upload,
82           ['git', 'checkout', 'master'],
83           ['git', 'branch', '-D', branch],
84          ])
85  status = True
86  for cmd in cmds:
87    print 'Running ' + ' '.join(cmd)
88    if subprocess.call(cmd):
89      print 'FAILED. Please check if skia git repo is present.'
90      subprocess.call(['git', 'checkout', 'master'])
91      status = False
92      break
93  os.chdir(old_cwd)
94  return status
95
96def delete_dirs(li):
97  for d in li:
98    print 'Deleting directory %s' % d
99    shutil.rmtree(d)
100
101def widen_bench_ranges(url, bot, repo_dir, exp_dir):
102  fname = 'bench_expectations_%s.txt' % bot
103  src = os.path.join(repo_dir, 'expectations', 'bench', fname)
104  if not os.path.isfile(src):
105    print 'This bot has no expectations! %s' % bot
106    return False
107  row_dic = {}
108  for l in urllib2.urlopen(url).read().split('\n'):
109    data = EXCEPTION_RE_COMPILED.search(l)
110    if data:
111      row = data.group(1)
112      lb = float(data.group(2))
113      ub = float(data.group(3))
114      actual = float(data.group(4))
115      exp = float(data.group(5))
116      avg = (actual + exp) / 2
117      shift = avg - exp
118      lb = lb + shift
119      ub = ub + shift
120      # In case outlier really fluctuates a lot
121      if actual < lb:
122        lb = actual - abs(shift) * 0.1 + 0.5
123      elif actual > ub:
124        ub = actual + abs(shift) * 0.1 + 0.5
125      row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub)
126  if not row_dic:
127    print 'NO out-of-range benches found at %s' % url
128    return False
129
130  changed = 0
131  li = []
132  for l in open(src).readlines():
133    parts = l.strip().split(',')
134    if parts[0].startswith('#') or len(parts) != 5:
135      li.append(l.strip())
136      continue
137    if ','.join(parts[:2]) in row_dic:
138      li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])])
139      changed += 1
140    else:
141      li.append(l.strip())
142  if not changed:
143    print 'Not in source file:\n' + '\n'.join(row_dic.keys())
144    return False
145
146  dst = os.path.join(exp_dir, fname)
147  with open(dst, 'w+') as f:
148    f.write('\n'.join(li))
149  return True
150
151
152def main():
153  d = os.path.dirname(os.path.abspath(__file__))
154  os.chdir(d)
155  if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE):
156    print 'Please copy script to a separate dir outside git repos to use.'
157    return
158  ts_str = '%s' % time.time()
159
160  parser = argparse.ArgumentParser()
161  parser.add_argument('--url',
162                      help='Broken bench build CheckForRegressions page url.')
163  parser.add_argument('--commit', action='store_true',
164                      help='Whether to commit changes automatically.')
165  args = parser.parse_args()
166  repo_dir = os.path.join(d, 'skia')
167  if not os.path.exists(repo_dir):
168    os.makedirs(repo_dir)
169  if not checkout_or_update_skia(repo_dir):
170    print 'ERROR setting up Skia repo at %s' % repo_dir
171    return 1
172
173  file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py')
174  if not filecmp.cmp(__file__, file_in_repo):
175    shutil.copy(file_in_repo, __file__)
176    print 'Updated this script from repo; please run again.'
177    return
178
179  if not args.url:
180    raise Exception('Please provide a url with broken CheckForRegressions.')
181  path = args.url.split('/')
182  if len(path) != 11 or not path[6].isdigit():
183    raise Exception('Unexpected url format: %s' % args.url)
184  bot = path[4]
185  build = path[6]
186  commit = False
187  if args.commit:
188    commit = True
189
190  exp_dir = os.path.join(d, 'exp' + ts_str)
191  clean_dir(exp_dir)
192  if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir):
193    print 'NO bench exceptions found! %s' % args.url
194  elif not git_commit_expectations(
195      repo_dir, exp_dir, bot, build, commit):
196    print 'ERROR uploading expectations using git.'
197  elif not commit:
198    print 'CL created. Please take a look at the link above.'
199  else:
200    print 'New bench baselines should be in CQ now.'
201  delete_dirs([exp_dir])
202
203
204if __name__ == "__main__":
205  main()
206