1#!/usr/bin/env python 2 3# Copyright 2016 Google Inc. 4# 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8from __future__ import print_function 9from _adb import Adb 10from _benchresult import BenchResult 11from _hardware import HardwareException, Hardware 12from argparse import ArgumentParser 13from multiprocessing import Queue 14from threading import Thread, Timer 15import collections 16import glob 17import math 18import re 19import subprocess 20import sys 21import time 22 23__argparse = ArgumentParser(description=""" 24 25Executes the skpbench binary with various configs and skps. 26 27Also monitors the output in order to filter out and re-run results that have an 28unacceptable stddev. 29 30""") 31 32__argparse.add_argument('skpbench', 33 help="path to the skpbench binary") 34__argparse.add_argument('--adb', 35 action='store_true', help="execute skpbench over adb") 36__argparse.add_argument('--adb_binary', default='adb', 37 help="The name of the adb binary to use.") 38__argparse.add_argument('-s', '--device-serial', 39 help="if using adb, ID of the specific device to target " 40 "(only required if more than 1 device is attached)") 41__argparse.add_argument('-m', '--max-stddev', 42 type=float, default=4, 43 help="initial max allowable relative standard deviation") 44__argparse.add_argument('-x', '--suffix', 45 help="suffix to append on config (e.g. '_before', '_after')") 46__argparse.add_argument('-w','--write-path', 47 help="directory to save .png proofs to disk.") 48__argparse.add_argument('-v','--verbosity', 49 type=int, default=1, help="level of verbosity (0=none to 5=debug)") 50__argparse.add_argument('-d', '--duration', 51 type=int, help="number of milliseconds to run each benchmark") 52__argparse.add_argument('-l', '--sample-ms', 53 type=int, help="duration of a sample (minimum)") 54__argparse.add_argument('--gpu', 55 action='store_true', 56 help="perform timing on the gpu clock instead of cpu (gpu work only)") 57__argparse.add_argument('--fps', 58 action='store_true', help="use fps instead of ms") 59__argparse.add_argument('--pr', 60 help="comma- or space-separated list of GPU path renderers, including: " 61 "[[~]all [~]default [~]dashline [~]nvpr [~]msaa [~]aaconvex " 62 "[~]aalinearizing [~]small [~]tess]") 63__argparse.add_argument('--nocache', 64 action='store_true', help="disable caching of path mask textures") 65__argparse.add_argument('-c', '--config', 66 default='gl', help="comma- or space-separated list of GPU configs") 67__argparse.add_argument('-a', '--resultsfile', 68 help="optional file to append results into") 69__argparse.add_argument('skps', 70 nargs='+', 71 help=".skp files or directories to expand for .skp files") 72 73FLAGS = __argparse.parse_args() 74if FLAGS.adb: 75 import _adb_path as _path 76 _path.init(FLAGS.device_serial, FLAGS.adb_binary) 77else: 78 import _os_path as _path 79 80def dump_commandline_if_verbose(commandline): 81 if FLAGS.verbosity >= 5: 82 quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline] 83 print(' '.join(quoted), file=sys.stderr) 84 85 86class StddevException(Exception): 87 pass 88 89class Message: 90 READLINE = 0, 91 POLL_HARDWARE = 1, 92 EXIT = 2 93 def __init__(self, message, value=None): 94 self.message = message 95 self.value = value 96 97class SubprocessMonitor(Thread): 98 def __init__(self, queue, proc): 99 self._queue = queue 100 self._proc = proc 101 Thread.__init__(self) 102 103 def run(self): 104 """Runs on the background thread.""" 105 for line in iter(self._proc.stdout.readline, b''): 106 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) 107 self._queue.put(Message(Message.EXIT)) 108 109class SKPBench: 110 ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)] 111 if FLAGS.duration: 112 ARGV.extend(['--duration', str(FLAGS.duration)]) 113 if FLAGS.sample_ms: 114 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) 115 if FLAGS.gpu: 116 ARGV.extend(['--gpuClock', 'true']) 117 if FLAGS.fps: 118 ARGV.extend(['--fps', 'true']) 119 if FLAGS.pr: 120 ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr)) 121 if FLAGS.nocache: 122 ARGV.extend(['--cachePathMasks', 'false']) 123 if FLAGS.adb: 124 if FLAGS.device_serial is None: 125 ARGV[:0] = [FLAGS.adb_binary, 'shell'] 126 else: 127 ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell'] 128 129 @classmethod 130 def get_header(cls, outfile=sys.stdout): 131 commandline = cls.ARGV + ['--duration', '0'] 132 dump_commandline_if_verbose(commandline) 133 out = subprocess.check_output(commandline, stderr=subprocess.STDOUT) 134 return out.rstrip() 135 136 @classmethod 137 def run_warmup(cls, warmup_time, config): 138 if not warmup_time: 139 return 140 print('running %i second warmup...' % warmup_time, file=sys.stderr) 141 commandline = cls.ARGV + ['--duration', str(warmup_time * 1000), 142 '--config', config, 143 '--skp', 'warmup'] 144 dump_commandline_if_verbose(commandline) 145 output = subprocess.check_output(commandline, stderr=subprocess.STDOUT) 146 147 # validate the warmup run output. 148 for line in output.decode('utf-8').split('\n'): 149 match = BenchResult.match(line.rstrip()) 150 if match and match.bench == 'warmup': 151 return 152 raise Exception('Invalid warmup output:\n%s' % output) 153 154 def __init__(self, skp, config, max_stddev, best_result=None): 155 self.skp = skp 156 self.config = config 157 self.max_stddev = max_stddev 158 self.best_result = best_result 159 self._queue = Queue() 160 self._proc = None 161 self._monitor = None 162 self._hw_poll_timer = None 163 164 def __enter__(self): 165 return self 166 167 def __exit__(self, exception_type, exception_value, traceback): 168 if self._proc: 169 self.terminate() 170 if self._hw_poll_timer: 171 self._hw_poll_timer.cancel() 172 173 def execute(self, hardware): 174 hardware.sanity_check() 175 self._schedule_hardware_poll() 176 177 commandline = self.ARGV + ['--config', self.config, 178 '--skp', self.skp, 179 '--suppressHeader', 'true'] 180 if FLAGS.write_path: 181 pngfile = _path.join(FLAGS.write_path, self.config, 182 _path.basename(self.skp) + '.png') 183 commandline.extend(['--png', pngfile]) 184 dump_commandline_if_verbose(commandline) 185 self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE, 186 stderr=subprocess.STDOUT) 187 self._monitor = SubprocessMonitor(self._queue, self._proc) 188 self._monitor.start() 189 190 while True: 191 message = self._queue.get() 192 if message.message == Message.READLINE: 193 result = BenchResult.match(message.value) 194 if result: 195 hardware.sanity_check() 196 self._process_result(result) 197 elif hardware.filter_line(message.value): 198 print(message.value, file=sys.stderr) 199 continue 200 if message.message == Message.POLL_HARDWARE: 201 hardware.sanity_check() 202 self._schedule_hardware_poll() 203 continue 204 if message.message == Message.EXIT: 205 self._monitor.join() 206 self._proc.wait() 207 if self._proc.returncode != 0: 208 raise Exception("skpbench exited with nonzero exit code %i" % 209 self._proc.returncode) 210 self._proc = None 211 break 212 213 def _schedule_hardware_poll(self): 214 if self._hw_poll_timer: 215 self._hw_poll_timer.cancel() 216 self._hw_poll_timer = \ 217 Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE))) 218 self._hw_poll_timer.start() 219 220 def _process_result(self, result): 221 if not self.best_result or result.stddev <= self.best_result.stddev: 222 self.best_result = result 223 elif FLAGS.verbosity >= 2: 224 print("reusing previous result for %s/%s with lower stddev " 225 "(%s%% instead of %s%%)." % 226 (result.config, result.bench, self.best_result.stddev, 227 result.stddev), file=sys.stderr) 228 if self.max_stddev and self.best_result.stddev > self.max_stddev: 229 raise StddevException() 230 231 def terminate(self): 232 if self._proc: 233 self._proc.terminate() 234 self._monitor.join() 235 self._proc.wait() 236 self._proc = None 237 238def emit_result(line, resultsfile=None): 239 print(line) 240 sys.stdout.flush() 241 if resultsfile: 242 print(line, file=resultsfile) 243 resultsfile.flush() 244 245def run_benchmarks(configs, skps, hardware, resultsfile=None): 246 hasheader = False 247 benches = collections.deque([(skp, config, FLAGS.max_stddev) 248 for skp in skps 249 for config in configs]) 250 while benches: 251 try: 252 with hardware: 253 SKPBench.run_warmup(hardware.warmup_time, configs[0]) 254 if not hasheader: 255 emit_result(SKPBench.get_header(), resultsfile) 256 hasheader = True 257 while benches: 258 benchargs = benches.popleft() 259 with SKPBench(*benchargs) as skpbench: 260 try: 261 skpbench.execute(hardware) 262 if skpbench.best_result: 263 emit_result(skpbench.best_result.format(FLAGS.suffix), 264 resultsfile) 265 else: 266 print("WARNING: no result for %s with config %s" % 267 (skpbench.skp, skpbench.config), file=sys.stderr) 268 269 except StddevException: 270 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) 271 if FLAGS.verbosity >= 1: 272 print("stddev is too high for %s/%s (%s%%, max=%.2f%%), " 273 "re-queuing with max=%.2f%%." % 274 (skpbench.best_result.config, skpbench.best_result.bench, 275 skpbench.best_result.stddev, skpbench.max_stddev, 276 retry_max_stddev), 277 file=sys.stderr) 278 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, 279 skpbench.best_result)) 280 281 except HardwareException as exception: 282 skpbench.terminate() 283 if FLAGS.verbosity >= 4: 284 hardware.print_debug_diagnostics() 285 if FLAGS.verbosity >= 1: 286 print("%s; rebooting and taking a %i second nap..." % 287 (exception.message, exception.sleeptime), file=sys.stderr) 288 benches.appendleft(benchargs) # retry the same bench next time. 289 raise # wake hw up from benchmarking mode before the nap. 290 291 except HardwareException as exception: 292 time.sleep(exception.sleeptime) 293 294def main(): 295 # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)). 296 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' 297 configs = re.split(DELIMITER, FLAGS.config) 298 skps = _path.find_skps(FLAGS.skps) 299 300 if FLAGS.adb: 301 adb = Adb(FLAGS.device_serial, FLAGS.adb_binary, 302 echo=(FLAGS.verbosity >= 5)) 303 model = adb.check('getprop ro.product.model').strip() 304 if model == 'Pixel C': 305 from _hardware_pixel_c import HardwarePixelC 306 hardware = HardwarePixelC(adb) 307 elif model == 'Pixel': 308 from _hardware_pixel import HardwarePixel 309 hardware = HardwarePixel(adb) 310 elif model == 'Nexus 6P': 311 from _hardware_nexus_6p import HardwareNexus6P 312 hardware = HardwareNexus6P(adb) 313 else: 314 from _hardware_android import HardwareAndroid 315 print("WARNING: %s: don't know how to monitor this hardware; results " 316 "may be unreliable." % model, file=sys.stderr) 317 hardware = HardwareAndroid(adb) 318 else: 319 hardware = Hardware() 320 321 if FLAGS.resultsfile: 322 with open(FLAGS.resultsfile, mode='a+') as resultsfile: 323 run_benchmarks(configs, skps, hardware, resultsfile=resultsfile) 324 else: 325 run_benchmarks(configs, skps, hardware) 326 327 328if __name__ == '__main__': 329 main() 330