1#!/usr/bin/env python
2
3# Copyright 2016 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8from __future__ import print_function
9from _adb import Adb
10from _benchresult import BenchResult
11from _hardware import HardwareException, Hardware
12from argparse import ArgumentParser
13from multiprocessing import Queue
14from threading import Thread, Timer
15import collections
16import glob
17import math
18import re
19import subprocess
20import sys
21import time
22
23__argparse = ArgumentParser(description="""
24
25Executes the skpbench binary with various configs and skps.
26
27Also monitors the output in order to filter out and re-run results that have an
28unacceptable stddev.
29
30""")
31
32__argparse.add_argument('skpbench',
33  help="path to the skpbench binary")
34__argparse.add_argument('--adb',
35  action='store_true', help="execute skpbench over adb")
36__argparse.add_argument('--adb_binary', default='adb',
37  help="The name of the adb binary to use.")
38__argparse.add_argument('-s', '--device-serial',
39  help="if using adb, ID of the specific device to target "
40       "(only required if more than 1 device is attached)")
41__argparse.add_argument('-m', '--max-stddev',
42  type=float, default=4,
43  help="initial max allowable relative standard deviation")
44__argparse.add_argument('-x', '--suffix',
45  help="suffix to append on config (e.g. '_before', '_after')")
46__argparse.add_argument('-w','--write-path',
47  help="directory to save .png proofs to disk.")
48__argparse.add_argument('-v','--verbosity',
49  type=int, default=1, help="level of verbosity (0=none to 5=debug)")
50__argparse.add_argument('-d', '--duration',
51  type=int, help="number of milliseconds to run each benchmark")
52__argparse.add_argument('-l', '--sample-ms',
53  type=int, help="duration of a sample (minimum)")
54__argparse.add_argument('--gpu',
55  action='store_true',
56  help="perform timing on the gpu clock instead of cpu (gpu work only)")
57__argparse.add_argument('--fps',
58  action='store_true', help="use fps instead of ms")
59__argparse.add_argument('--pr',
60  help="comma- or space-separated list of GPU path renderers, including: "
61       "[[~]all [~]default [~]dashline [~]nvpr [~]msaa [~]aaconvex "
62       "[~]aalinearizing [~]small [~]tess]")
63__argparse.add_argument('--nocache',
64  action='store_true', help="disable caching of path mask textures")
65__argparse.add_argument('-c', '--config',
66  default='gl', help="comma- or space-separated list of GPU configs")
67__argparse.add_argument('-a', '--resultsfile',
68  help="optional file to append results into")
69__argparse.add_argument('skps',
70  nargs='+',
71  help=".skp files or directories to expand for .skp files")
72
73FLAGS = __argparse.parse_args()
74if FLAGS.adb:
75  import _adb_path as _path
76  _path.init(FLAGS.device_serial, FLAGS.adb_binary)
77else:
78  import _os_path as _path
79
80def dump_commandline_if_verbose(commandline):
81  if FLAGS.verbosity >= 5:
82    quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
83    print(' '.join(quoted), file=sys.stderr)
84
85
86class StddevException(Exception):
87  pass
88
89class Message:
90  READLINE = 0,
91  POLL_HARDWARE = 1,
92  EXIT = 2
93  def __init__(self, message, value=None):
94    self.message = message
95    self.value = value
96
97class SubprocessMonitor(Thread):
98  def __init__(self, queue, proc):
99    self._queue = queue
100    self._proc = proc
101    Thread.__init__(self)
102
103  def run(self):
104    """Runs on the background thread."""
105    for line in iter(self._proc.stdout.readline, b''):
106      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
107    self._queue.put(Message(Message.EXIT))
108
109class SKPBench:
110  ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
111  if FLAGS.duration:
112    ARGV.extend(['--duration', str(FLAGS.duration)])
113  if FLAGS.sample_ms:
114    ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
115  if FLAGS.gpu:
116    ARGV.extend(['--gpuClock', 'true'])
117  if FLAGS.fps:
118    ARGV.extend(['--fps', 'true'])
119  if FLAGS.pr:
120    ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr))
121  if FLAGS.nocache:
122    ARGV.extend(['--cachePathMasks', 'false'])
123  if FLAGS.adb:
124    if FLAGS.device_serial is None:
125      ARGV[:0] = [FLAGS.adb_binary, 'shell']
126    else:
127      ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell']
128
129  @classmethod
130  def get_header(cls, outfile=sys.stdout):
131    commandline = cls.ARGV + ['--duration', '0']
132    dump_commandline_if_verbose(commandline)
133    out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
134    return out.rstrip()
135
136  @classmethod
137  def run_warmup(cls, warmup_time, config):
138    if not warmup_time:
139      return
140    print('running %i second warmup...' % warmup_time, file=sys.stderr)
141    commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
142                              '--config', config,
143                              '--skp', 'warmup']
144    dump_commandline_if_verbose(commandline)
145    output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
146
147    # validate the warmup run output.
148    for line in output.decode('utf-8').split('\n'):
149      match = BenchResult.match(line.rstrip())
150      if match and match.bench == 'warmup':
151        return
152    raise Exception('Invalid warmup output:\n%s' % output)
153
154  def __init__(self, skp, config, max_stddev, best_result=None):
155    self.skp = skp
156    self.config = config
157    self.max_stddev = max_stddev
158    self.best_result = best_result
159    self._queue = Queue()
160    self._proc = None
161    self._monitor = None
162    self._hw_poll_timer = None
163
164  def __enter__(self):
165    return self
166
167  def __exit__(self, exception_type, exception_value, traceback):
168    if self._proc:
169      self.terminate()
170    if self._hw_poll_timer:
171      self._hw_poll_timer.cancel()
172
173  def execute(self, hardware):
174    hardware.sanity_check()
175    self._schedule_hardware_poll()
176
177    commandline = self.ARGV + ['--config', self.config,
178                               '--skp', self.skp,
179                               '--suppressHeader', 'true']
180    if FLAGS.write_path:
181      pngfile = _path.join(FLAGS.write_path, self.config,
182                           _path.basename(self.skp) + '.png')
183      commandline.extend(['--png', pngfile])
184    dump_commandline_if_verbose(commandline)
185    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
186                                  stderr=subprocess.STDOUT)
187    self._monitor = SubprocessMonitor(self._queue, self._proc)
188    self._monitor.start()
189
190    while True:
191      message = self._queue.get()
192      if message.message == Message.READLINE:
193        result = BenchResult.match(message.value)
194        if result:
195          hardware.sanity_check()
196          self._process_result(result)
197        elif hardware.filter_line(message.value):
198          print(message.value, file=sys.stderr)
199        continue
200      if message.message == Message.POLL_HARDWARE:
201        hardware.sanity_check()
202        self._schedule_hardware_poll()
203        continue
204      if message.message == Message.EXIT:
205        self._monitor.join()
206        self._proc.wait()
207        if self._proc.returncode != 0:
208          raise Exception("skpbench exited with nonzero exit code %i" %
209                          self._proc.returncode)
210        self._proc = None
211        break
212
213  def _schedule_hardware_poll(self):
214    if self._hw_poll_timer:
215      self._hw_poll_timer.cancel()
216    self._hw_poll_timer = \
217      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
218    self._hw_poll_timer.start()
219
220  def _process_result(self, result):
221    if not self.best_result or result.stddev <= self.best_result.stddev:
222      self.best_result = result
223    elif FLAGS.verbosity >= 2:
224      print("reusing previous result for %s/%s with lower stddev "
225            "(%s%% instead of %s%%)." %
226            (result.config, result.bench, self.best_result.stddev,
227             result.stddev), file=sys.stderr)
228    if self.max_stddev and self.best_result.stddev > self.max_stddev:
229      raise StddevException()
230
231  def terminate(self):
232    if self._proc:
233      self._proc.terminate()
234      self._monitor.join()
235      self._proc.wait()
236      self._proc = None
237
238def emit_result(line, resultsfile=None):
239  print(line)
240  sys.stdout.flush()
241  if resultsfile:
242    print(line, file=resultsfile)
243    resultsfile.flush()
244
245def run_benchmarks(configs, skps, hardware, resultsfile=None):
246  hasheader = False
247  benches = collections.deque([(skp, config, FLAGS.max_stddev)
248                               for skp in skps
249                               for config in configs])
250  while benches:
251    try:
252      with hardware:
253        SKPBench.run_warmup(hardware.warmup_time, configs[0])
254        if not hasheader:
255          emit_result(SKPBench.get_header(), resultsfile)
256          hasheader = True
257        while benches:
258          benchargs = benches.popleft()
259          with SKPBench(*benchargs) as skpbench:
260            try:
261              skpbench.execute(hardware)
262              if skpbench.best_result:
263                emit_result(skpbench.best_result.format(FLAGS.suffix),
264                            resultsfile)
265              else:
266                print("WARNING: no result for %s with config %s" %
267                      (skpbench.skp, skpbench.config), file=sys.stderr)
268
269            except StddevException:
270              retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
271              if FLAGS.verbosity >= 1:
272                print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
273                      "re-queuing with max=%.2f%%." %
274                      (skpbench.best_result.config, skpbench.best_result.bench,
275                       skpbench.best_result.stddev, skpbench.max_stddev,
276                       retry_max_stddev),
277                      file=sys.stderr)
278              benches.append((skpbench.skp, skpbench.config, retry_max_stddev,
279                              skpbench.best_result))
280
281            except HardwareException as exception:
282              skpbench.terminate()
283              if FLAGS.verbosity >= 4:
284                hardware.print_debug_diagnostics()
285              if FLAGS.verbosity >= 1:
286                print("%s; rebooting and taking a %i second nap..." %
287                      (exception.message, exception.sleeptime), file=sys.stderr)
288              benches.appendleft(benchargs) # retry the same bench next time.
289              raise # wake hw up from benchmarking mode before the nap.
290
291    except HardwareException as exception:
292      time.sleep(exception.sleeptime)
293
294def main():
295  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
296  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
297  configs = re.split(DELIMITER, FLAGS.config)
298  skps = _path.find_skps(FLAGS.skps)
299
300  if FLAGS.adb:
301    adb = Adb(FLAGS.device_serial, FLAGS.adb_binary,
302              echo=(FLAGS.verbosity >= 5))
303    model = adb.check('getprop ro.product.model').strip()
304    if model == 'Pixel C':
305      from _hardware_pixel_c import HardwarePixelC
306      hardware = HardwarePixelC(adb)
307    elif model == 'Pixel':
308      from _hardware_pixel import HardwarePixel
309      hardware = HardwarePixel(adb)
310    elif model == 'Nexus 6P':
311      from _hardware_nexus_6p import HardwareNexus6P
312      hardware = HardwareNexus6P(adb)
313    else:
314      from _hardware_android import HardwareAndroid
315      print("WARNING: %s: don't know how to monitor this hardware; results "
316            "may be unreliable." % model, file=sys.stderr)
317      hardware = HardwareAndroid(adb)
318  else:
319    hardware = Hardware()
320
321  if FLAGS.resultsfile:
322    with open(FLAGS.resultsfile, mode='a+') as resultsfile:
323      run_benchmarks(configs, skps, hardware, resultsfile=resultsfile)
324  else:
325    run_benchmarks(configs, skps, hardware)
326
327
328if __name__ == '__main__':
329  main()
330