1#!/usr/bin/pyton
2
3# Copyright 2017 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8import os
9import sys
10import subprocess
11import multiprocessing
12
13from argparse import ArgumentParser
14
15
16README = """
17Simply run
18\033[36m
19    python {0} TEST_GIT_BRANCH
20\033[0m
21to see if TEST_GIT_BRANCH has performance regressions against master in 8888.
22
23To compare a specific config with svg and skp resources included, add --config
24and --extraarg option. For exampe,
25\033[36m
26    python {0} TEST_GIT_BRANCH --config gl \\
27        --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps"
28\033[0m
29For more options, please see
30
31    python {0} --help
32""".format(__file__)
33
34
35CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
36AB_SCRIPT = "ab.py"
37
38
39def parse_args():
40  if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help':
41    print README
42
43  parser = ArgumentParser(
44    description='Noiselessly (hence calm) becnhmark a git branch against ' +
45                'another baseline branch (e.g., master) using multiple ' +
46                ' nanobench runs.'
47  )
48
49  default_threads = max(1, multiprocessing.cpu_count() / 2);
50  default_skiadir = os.path.normpath(CURRENT_DIR + "/../../")
51
52  config_help = (
53      'nanobench config; we currently support only one config '
54      'at a time (default: %(default)s)')
55  reps_help = (
56      'initial repititions of the nanobench run; this may be '
57      'overridden when we have many threads (default: %(default)s)')
58  extraarg_help = (
59      'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps '
60      '~/Desktop/bots/skps)')
61  baseline_help = (
62      'baseline branch to compare against (default: %(default)s)')
63  basearg_help = (
64      'nanobench arg for the baseline branch; if not given, we use '
65      ' the same arg for both the test branch and the baseline branch')
66  threads_help = (
67      'number of threads to be used (default: %(default)s); '
68      'for GPU config, this will always be 1')
69  no_compile_help = (
70      'whether NOT to compile nanobench and copy it to WRITEDIR '
71      '(i.e., reuse previous nanobench compiled)')
72  skip_base_help = (
73      'whether NOT to run nanobench on baseline branch '
74      '(i.e., reuse previous baseline measurements)')
75  noinit_help = (
76      'whether to skip initial nanobench runs (default: %(default)s)')
77  branch_help = (
78      "the test branch to benchmark; if it's 'modified', we'll benchmark the "
79      "current modified code against 'git stash'.")
80
81  definitions = [
82    # argname, type, default value, help
83    ['--config',    str, '8888', config_help],
84    ['--skiadir',   str, default_skiadir, 'default: %(default)s'],
85    ['--ninjadir',  str, 'out/Release', 'default: %(default)s'],
86    ['--writedir',  str, '/var/tmp', 'default: %(default)s'],
87    ['--extraarg',  str, '', extraarg_help],
88    ['--baseline',  str, 'master', baseline_help],
89    ['--basearg',   str, '', basearg_help],
90    ['--reps',      int, 2, reps_help],
91    ['--threads',   int, default_threads, threads_help],
92  ]
93
94  for d in definitions:
95    parser.add_argument(d[0], type=d[1], default=d[2], help=d[3])
96
97  parser.add_argument('branch', type=str, help=branch_help)
98  parser.add_argument('--no-compile', dest='no_compile', action="store_true",
99      help=no_compile_help)
100  parser.add_argument('--skip-base', dest='skipbase', action="store_true",
101      help=skip_base_help)
102  parser.add_argument('--noinit', dest='noinit', action="store_true",
103      help=noinit_help)
104  parser.add_argument('--concise', dest='concise', action="store_true",
105      help="If set, no verbose thread info will be printed.")
106  parser.set_defaults(no_compile=False);
107  parser.set_defaults(skipbase=False);
108  parser.set_defaults(noinit=False);
109  parser.set_defaults(concise=False);
110
111  # Additional args for bots
112  BHELP = "bot specific options"
113  parser.add_argument('--githash', type=str, help=BHELP)
114  parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP)
115
116  args = parser.parse_args()
117  if not args.basearg:
118    args.basearg = args.extraarg
119
120  return args
121
122
123def nano_path(args, branch):
124  return args.writedir + '/nanobench_' + branch
125
126
127def compile_branch(args, branch):
128  print "Compiling branch %s" % args.branch
129
130  commands = [
131    ['git', 'checkout', branch],
132    ['gclient', 'sync'],
133    ['ninja', '-C', args.ninjadir, 'nanobench'],
134    ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)]
135  ]
136  for command in commands:
137    subprocess.check_call(command, cwd=args.skiadir)
138
139
140def compile_modified(args):
141  print "Compiling modified code"
142  subprocess.check_call(
143      ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir)
144  subprocess.check_call(
145      ['cp', args.ninjadir + '/nanobench', nano_path(args, args.branch)],
146      cwd=args.skiadir)
147
148  print "Compiling stashed code"
149  stash_output = subprocess.check_output(['git', 'stash'], cwd=args.skiadir)
150  if 'No local changes to save' in stash_output:
151    subprocess.check_call(['git', 'reset', 'HEAD^', '--soft'])
152    subprocess.check_call(['git', 'stash'])
153
154  subprocess.check_call(['gclient', 'sync'], cwd=args.skiadir)
155  subprocess.check_call(
156      ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir)
157  subprocess.check_call(
158      ['cp', args.ninjadir + '/nanobench', nano_path(args, args.baseline)],
159      cwd=args.skiadir)
160  subprocess.check_call(['git', 'stash', 'pop'], cwd=args.skiadir)
161
162def compile_nanobench(args):
163  if args.branch == 'modified':
164    compile_modified(args)
165  else:
166    compile_branch(args, args.branch)
167    compile_branch(args, args.baseline)
168
169
170def main():
171  args = parse_args()
172
173  # copy in case that it will be gone after git branch switching
174  orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT
175  temp_ab_name = args.writedir + "/" + AB_SCRIPT
176  subprocess.check_call(['cp', orig_ab_name, temp_ab_name])
177
178  if not args.no_compile:
179    compile_nanobench(args)
180
181  command = [
182    'python',
183    temp_ab_name,
184    args.writedir,
185    args.branch + ("_A" if args.branch == args.baseline else ""),
186    args.baseline + ("_B" if args.branch == args.baseline else ""),
187    nano_path(args, args.branch),
188    nano_path(args, args.baseline),
189    args.extraarg,
190    args.basearg,
191    str(args.reps),
192    "true" if args.skipbase else "false",
193    args.config,
194    str(args.threads if args.config in ["8888", "565"] else 1),
195    "true" if args.noinit else "false"
196  ]
197
198  if args.githash:
199    command += ['--githash', args.githash]
200  if args.keys:
201    command += (['--keys'] + args.keys)
202
203  if args.concise:
204    command.append("--concise")
205
206  p = subprocess.Popen(command, cwd=args.skiadir)
207  try:
208    p.wait()
209  except KeyboardInterrupt:
210    try:
211      p.terminate()
212    except OSError as e:
213      print e
214
215
216if __name__ == "__main__":
217  main()
218