1980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li#!/usr/bin/pyton 2980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 3980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li# Copyright 2017 Google Inc. 4980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li# 5980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li# Use of this source code is governed by a BSD-style license that can be 6980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li# found in the LICENSE file. 7980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 8980379d47589f06719c4f3545c412789bf4d43f4Yuqian Liimport os 9980379d47589f06719c4f3545c412789bf4d43f4Yuqian Liimport sys 10980379d47589f06719c4f3545c412789bf4d43f4Yuqian Liimport subprocess 11980379d47589f06719c4f3545c412789bf4d43f4Yuqian Liimport multiprocessing 12980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 13980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lifrom argparse import ArgumentParser 14980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 15980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 16980379d47589f06719c4f3545c412789bf4d43f4Yuqian LiREADME = """ 17980379d47589f06719c4f3545c412789bf4d43f4Yuqian LiSimply run 18980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li\033[36m 19980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li python {0} TEST_GIT_BRANCH 20980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li\033[0m 21980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lito see if TEST_GIT_BRANCH has performance regressions against master in 8888. 22980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 23980379d47589f06719c4f3545c412789bf4d43f4Yuqian LiTo compare a specific config with svg and skp resources included, add --config 24980379d47589f06719c4f3545c412789bf4d43f4Yuqian Liand --extraarg option. For exampe, 25980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li\033[36m 26980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li python {0} TEST_GIT_BRANCH --config gl \\ 27980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps" 28980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li\033[0m 29980379d47589f06719c4f3545c412789bf4d43f4Yuqian LiFor more options, please see 30980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 31980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li python {0} --help 32980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li""".format(__file__) 33980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 34980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 35980379d47589f06719c4f3545c412789bf4d43f4Yuqian LiCURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) 36980379d47589f06719c4f3545c412789bf4d43f4Yuqian LiAB_SCRIPT = "ab.py" 37980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 38980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 39980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lidef parse_args(): 40980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help': 41980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li print README 42980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 43980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser = ArgumentParser( 44980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li description='Noiselessly (hence calm) becnhmark a git branch against ' + 45980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'another baseline branch (e.g., master) using multiple ' + 46980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ' nanobench runs.' 47980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ) 48980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 49980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li default_threads = max(1, multiprocessing.cpu_count() / 2); 50980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li default_skiadir = os.path.normpath(CURRENT_DIR + "/../../") 51980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 52980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li config_help = ( 53980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'nanobench config; we currently support only one config ' 54980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'at a time (default: %(default)s)') 55980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li reps_help = ( 56980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'initial repititions of the nanobench run; this may be ' 57980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'overridden when we have many threads (default: %(default)s)') 58980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li extraarg_help = ( 59980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps ' 60980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li '~/Desktop/bots/skps)') 61980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li baseline_help = ( 62980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'baseline branch to compare against (default: %(default)s)') 63980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li basearg_help = ( 64980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'nanobench arg for the baseline branch; if not given, we use ' 65980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ' the same arg for both the test branch and the baseline branch') 66980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li threads_help = ( 67980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'number of threads to be used (default: %(default)s); ' 68980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'for GPU config, this will always be 1') 69980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li no_compile_help = ( 70980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'whether NOT to compile nanobench and copy it to WRITEDIR ' 71980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li '(i.e., reuse previous nanobench compiled)') 72980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li skip_base_help = ( 73980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'whether NOT to run nanobench on baseline branch ' 74980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li '(i.e., reuse previous baseline measurements)') 75980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li noinit_help = ( 76980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'whether to skip initial nanobench runs (default: %(default)s)') 77c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li branch_help = ( 78c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li "the test branch to benchmark; if it's 'modified', we'll benchmark the " 79c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li "current modified code against 'git stash'.") 80980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 81980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li definitions = [ 82980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li # argname, type, default value, help 83980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--config', str, '8888', config_help], 84980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--skiadir', str, default_skiadir, 'default: %(default)s'], 85980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--ninjadir', str, 'out/Release', 'default: %(default)s'], 86980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--writedir', str, '/var/tmp', 'default: %(default)s'], 87980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--extraarg', str, '', extraarg_help], 88980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--baseline', str, 'master', baseline_help], 89980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--basearg', str, '', basearg_help], 90980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['--reps', int, 2, reps_help], 91228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li ['--threads', int, default_threads, threads_help], 92980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ] 93980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 94980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li for d in definitions: 95980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.add_argument(d[0], type=d[1], default=d[2], help=d[3]) 96980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 97c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li parser.add_argument('branch', type=str, help=branch_help) 98980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.add_argument('--no-compile', dest='no_compile', action="store_true", 99980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li help=no_compile_help) 100980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.add_argument('--skip-base', dest='skipbase', action="store_true", 101980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li help=skip_base_help) 102980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.add_argument('--noinit', dest='noinit', action="store_true", 103980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li help=noinit_help) 10484366d22d6535dd955d9aa14bb0f364381c3f45dYuqian Li parser.add_argument('--concise', dest='concise', action="store_true", 10584366d22d6535dd955d9aa14bb0f364381c3f45dYuqian Li help="If set, no verbose thread info will be printed.") 106980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.set_defaults(no_compile=False); 107980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.set_defaults(skipbase=False); 108980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li parser.set_defaults(noinit=False); 10984366d22d6535dd955d9aa14bb0f364381c3f45dYuqian Li parser.set_defaults(concise=False); 110980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 111228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li # Additional args for bots 112228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li BHELP = "bot specific options" 113228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li parser.add_argument('--githash', type=str, help=BHELP) 114228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) 115228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li 116980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args = parser.parse_args() 117980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li if not args.basearg: 118980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.basearg = args.extraarg 119980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 120980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li return args 121980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 122980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 123980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lidef nano_path(args, branch): 124980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li return args.writedir + '/nanobench_' + branch 125980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 126980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 127980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lidef compile_branch(args, branch): 128980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li print "Compiling branch %s" % args.branch 129980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 130980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li commands = [ 131980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['git', 'checkout', branch], 132632156d835940df95a900d8e8c9851eda0cb917cYuqian Li ['gclient', 'sync'], 133980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['ninja', '-C', args.ninjadir, 'nanobench'], 134980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)] 135980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ] 136980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li for command in commands: 137980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li subprocess.check_call(command, cwd=args.skiadir) 138980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 139980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 140c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Lidef compile_modified(args): 141c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li print "Compiling modified code" 142c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li subprocess.check_call( 143c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 144c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li subprocess.check_call( 145c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li ['cp', args.ninjadir + '/nanobench', nano_path(args, args.branch)], 146c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li cwd=args.skiadir) 147c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li 148c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li print "Compiling stashed code" 1495be3a8e061c951dabd94ce7100ca551a16dbe012Yuqian Li stash_output = subprocess.check_output(['git', 'stash'], cwd=args.skiadir) 1505be3a8e061c951dabd94ce7100ca551a16dbe012Yuqian Li if 'No local changes to save' in stash_output: 1515be3a8e061c951dabd94ce7100ca551a16dbe012Yuqian Li subprocess.check_call(['git', 'reset', 'HEAD^', '--soft']) 1525be3a8e061c951dabd94ce7100ca551a16dbe012Yuqian Li subprocess.check_call(['git', 'stash']) 1535be3a8e061c951dabd94ce7100ca551a16dbe012Yuqian Li 154632156d835940df95a900d8e8c9851eda0cb917cYuqian Li subprocess.check_call(['gclient', 'sync'], cwd=args.skiadir) 155c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li subprocess.check_call( 156c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 157c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li subprocess.check_call( 158c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li ['cp', args.ninjadir + '/nanobench', nano_path(args, args.baseline)], 159c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li cwd=args.skiadir) 160c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li subprocess.check_call(['git', 'stash', 'pop'], cwd=args.skiadir) 161c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li 162980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lidef compile_nanobench(args): 163c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li if args.branch == 'modified': 164c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li compile_modified(args) 165c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li else: 166c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li compile_branch(args, args.branch) 167c81aaaad0b1ea0cf77882a4b89adc5b9e58c05bbYuqian Li compile_branch(args, args.baseline) 168980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 169980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 170980379d47589f06719c4f3545c412789bf4d43f4Yuqian Lidef main(): 171980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args = parse_args() 172980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 173980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li # copy in case that it will be gone after git branch switching 174980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT 175980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li temp_ab_name = args.writedir + "/" + AB_SCRIPT 176980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li subprocess.check_call(['cp', orig_ab_name, temp_ab_name]) 177980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 178980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li if not args.no_compile: 179980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li compile_nanobench(args) 180980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 181980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li command = [ 182980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 'python', 183980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li temp_ab_name, 184980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.writedir, 185980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.branch + ("_A" if args.branch == args.baseline else ""), 186980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.baseline + ("_B" if args.branch == args.baseline else ""), 187980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li nano_path(args, args.branch), 188980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li nano_path(args, args.baseline), 189980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.extraarg, 190980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.basearg, 191980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li str(args.reps), 192980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li "true" if args.skipbase else "false", 193980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li args.config, 194980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li str(args.threads if args.config in ["8888", "565"] else 1), 195980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li "true" if args.noinit else "false" 196980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li ] 197980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 198228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li if args.githash: 199228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li command += ['--githash', args.githash] 200228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li if args.keys: 201228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li command += (['--keys'] + args.keys) 202228da62fa791e1532826f8e17b945c3d8cbc1300Yuqian Li 20384366d22d6535dd955d9aa14bb0f364381c3f45dYuqian Li if args.concise: 20484366d22d6535dd955d9aa14bb0f364381c3f45dYuqian Li command.append("--concise") 20584366d22d6535dd955d9aa14bb0f364381c3f45dYuqian Li 206980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li p = subprocess.Popen(command, cwd=args.skiadir) 207980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li try: 208980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li p.wait() 209980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li except KeyboardInterrupt: 210980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li try: 211980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li p.terminate() 212980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li except OSError as e: 213980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li print e 214980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 215980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li 216980379d47589f06719c4f3545c412789bf4d43f4Yuqian Liif __name__ == "__main__": 217980379d47589f06719c4f3545c412789bf4d43f4Yuqian Li main() 218