1#!/usr/bin/env python
2
3# Copyright 2016 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8from __future__ import print_function
9from _adb import Adb
10from _benchresult import BenchResult
11from _hardware import HardwareException, Hardware
12from argparse import ArgumentParser
13from multiprocessing import Queue
14from threading import Thread, Timer
15import collections
16import glob
17import math
18import re
19import subprocess
20import sys
21import time
22
23__argparse = ArgumentParser(description="""
24
25Executes the skpbench binary with various configs and skps.
26
27Also monitors the output in order to filter out and re-run results that have an
28unacceptable stddev.
29
30""")
31
32__argparse.add_argument('skpbench',
33  help="path to the skpbench binary")
34__argparse.add_argument('--adb',
35  action='store_true', help="execute skpbench over adb")
36__argparse.add_argument('--adb_binary', default='adb',
37  help="The name of the adb binary to use.")
38__argparse.add_argument('-s', '--device-serial',
39  help="if using adb, ID of the specific device to target "
40       "(only required if more than 1 device is attached)")
41__argparse.add_argument('-m', '--max-stddev',
42  type=float, default=4,
43  help="initial max allowable relative standard deviation")
44__argparse.add_argument('-x', '--suffix',
45  help="suffix to append on config (e.g. '_before', '_after')")
46__argparse.add_argument('-w','--write-path',
47  help="directory to save .png proofs to disk.")
48__argparse.add_argument('-v','--verbosity',
49  type=int, default=1, help="level of verbosity (0=none to 5=debug)")
50__argparse.add_argument('-d', '--duration',
51  type=int, help="number of milliseconds to run each benchmark")
52__argparse.add_argument('-l', '--sample-ms',
53  type=int, help="duration of a sample (minimum)")
54__argparse.add_argument('--gpu',
55  action='store_true',
56  help="perform timing on the gpu clock instead of cpu (gpu work only)")
57__argparse.add_argument('--fps',
58  action='store_true', help="use fps instead of ms")
59__argparse.add_argument('--pr',
60  help="comma- or space-separated list of GPU path renderers, including: "
61       "[[~]all [~]default [~]dashline [~]nvpr [~]msaa [~]aaconvex "
62       "[~]aalinearizing [~]small [~]tess]")
63__argparse.add_argument('--nocache',
64  action='store_true', help="disable caching of path mask textures")
65__argparse.add_argument('-c', '--config',
66  default='gl', help="comma- or space-separated list of GPU configs")
67__argparse.add_argument('-a', '--resultsfile',
68  help="optional file to append results into")
69__argparse.add_argument('--ddl',
70  action='store_true', help="record the skp into DDLs before rendering")
71__argparse.add_argument('--ddlNumAdditionalThreads',
72  type=int, default=0,
73  help="number of DDL recording threads in addition to main one")
74__argparse.add_argument('--ddlTilingWidthHeight',
75  type=int, default=0, help="number of tiles along one edge when in DDL mode")
76__argparse.add_argument('--ddlRecordTime',
77  action='store_true', help="report just the cpu time spent recording DDLs")
78__argparse.add_argument('--gpuThreads',
79  type=int, default=-1,
80  help="Create this many extra threads to assist with GPU work, including"
81       " software path rendering. Defaults to two.")
82__argparse.add_argument('srcs',
83  nargs='+',
84  help=".skp files or directories to expand for .skp files, and/or .svg files")
85
86FLAGS = __argparse.parse_args()
87if FLAGS.adb:
88  import _adb_path as _path
89  _path.init(FLAGS.device_serial, FLAGS.adb_binary)
90else:
91  import _os_path as _path
92
93def dump_commandline_if_verbose(commandline):
94  if FLAGS.verbosity >= 5:
95    quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
96    print(' '.join(quoted), file=sys.stderr)
97
98
99class StddevException(Exception):
100  pass
101
102class Message:
103  READLINE = 0,
104  POLL_HARDWARE = 1,
105  EXIT = 2
106  def __init__(self, message, value=None):
107    self.message = message
108    self.value = value
109
110class SubprocessMonitor(Thread):
111  def __init__(self, queue, proc):
112    self._queue = queue
113    self._proc = proc
114    Thread.__init__(self)
115
116  def run(self):
117    """Runs on the background thread."""
118    for line in iter(self._proc.stdout.readline, b''):
119      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
120    self._queue.put(Message(Message.EXIT))
121
122class SKPBench:
123  ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
124  if FLAGS.duration:
125    ARGV.extend(['--duration', str(FLAGS.duration)])
126  if FLAGS.sample_ms:
127    ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
128  if FLAGS.gpu:
129    ARGV.extend(['--gpuClock', 'true'])
130  if FLAGS.fps:
131    ARGV.extend(['--fps', 'true'])
132  if FLAGS.pr:
133    ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr))
134  if FLAGS.nocache:
135    ARGV.extend(['--cachePathMasks', 'false'])
136  if FLAGS.gpuThreads != -1:
137    ARGV.extend(['--gpuThreads', str(FLAGS.gpuThreads)])
138
139  # DDL parameters
140  if FLAGS.ddl:
141    ARGV.extend(['--ddl', 'true'])
142  if FLAGS.ddlNumAdditionalThreads:
143    ARGV.extend(['--ddlNumAdditionalThreads',
144                 str(FLAGS.ddlNumAdditionalThreads)])
145  if FLAGS.ddlTilingWidthHeight:
146    ARGV.extend(['--ddlTilingWidthHeight', str(FLAGS.ddlTilingWidthHeight)])
147  if FLAGS.ddlRecordTime:
148    ARGV.extend(['--ddlRecordTime', 'true'])
149
150  if FLAGS.adb:
151    if FLAGS.device_serial is None:
152      ARGV[:0] = [FLAGS.adb_binary, 'shell']
153    else:
154      ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell']
155
156  @classmethod
157  def get_header(cls, outfile=sys.stdout):
158    commandline = cls.ARGV + ['--duration', '0']
159    dump_commandline_if_verbose(commandline)
160    out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
161    return out.rstrip()
162
163  @classmethod
164  def run_warmup(cls, warmup_time, config):
165    if not warmup_time:
166      return
167    print('running %i second warmup...' % warmup_time, file=sys.stderr)
168    commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
169                              '--config', config,
170                              '--src', 'warmup']
171    dump_commandline_if_verbose(commandline)
172    output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
173
174    # validate the warmup run output.
175    for line in output.decode('utf-8').split('\n'):
176      match = BenchResult.match(line.rstrip())
177      if match and match.bench == 'warmup':
178        return
179    raise Exception('Invalid warmup output:\n%s' % output)
180
181  def __init__(self, src, config, max_stddev, best_result=None):
182    self.src = src
183    self.config = config
184    self.max_stddev = max_stddev
185    self.best_result = best_result
186    self._queue = Queue()
187    self._proc = None
188    self._monitor = None
189    self._hw_poll_timer = None
190
191  def __enter__(self):
192    return self
193
194  def __exit__(self, exception_type, exception_value, traceback):
195    if self._proc:
196      self.terminate()
197    if self._hw_poll_timer:
198      self._hw_poll_timer.cancel()
199
200  def execute(self, hardware):
201    hardware.sanity_check()
202    self._schedule_hardware_poll()
203
204    commandline = self.ARGV + ['--config', self.config,
205                               '--src', self.src,
206                               '--suppressHeader', 'true']
207    if FLAGS.write_path:
208      pngfile = _path.join(FLAGS.write_path, self.config,
209                           _path.basename(self.src) + '.png')
210      commandline.extend(['--png', pngfile])
211    dump_commandline_if_verbose(commandline)
212    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
213                                  stderr=subprocess.STDOUT)
214    self._monitor = SubprocessMonitor(self._queue, self._proc)
215    self._monitor.start()
216
217    while True:
218      message = self._queue.get()
219      if message.message == Message.READLINE:
220        result = BenchResult.match(message.value)
221        if result:
222          hardware.sanity_check()
223          self._process_result(result)
224        elif hardware.filter_line(message.value):
225          print(message.value, file=sys.stderr)
226        continue
227      if message.message == Message.POLL_HARDWARE:
228        hardware.sanity_check()
229        self._schedule_hardware_poll()
230        continue
231      if message.message == Message.EXIT:
232        self._monitor.join()
233        self._proc.wait()
234        if self._proc.returncode != 0:
235          raise Exception("skpbench exited with nonzero exit code %i" %
236                          self._proc.returncode)
237        self._proc = None
238        break
239
240  def _schedule_hardware_poll(self):
241    if self._hw_poll_timer:
242      self._hw_poll_timer.cancel()
243    self._hw_poll_timer = \
244      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
245    self._hw_poll_timer.start()
246
247  def _process_result(self, result):
248    if not self.best_result or result.stddev <= self.best_result.stddev:
249      self.best_result = result
250    elif FLAGS.verbosity >= 2:
251      print("reusing previous result for %s/%s with lower stddev "
252            "(%s%% instead of %s%%)." %
253            (result.config, result.bench, self.best_result.stddev,
254             result.stddev), file=sys.stderr)
255    if self.max_stddev and self.best_result.stddev > self.max_stddev:
256      raise StddevException()
257
258  def terminate(self):
259    if self._proc:
260      self._proc.terminate()
261      self._monitor.join()
262      self._proc.wait()
263      self._proc = None
264
265def emit_result(line, resultsfile=None):
266  print(line)
267  sys.stdout.flush()
268  if resultsfile:
269    print(line, file=resultsfile)
270    resultsfile.flush()
271
272def run_benchmarks(configs, srcs, hardware, resultsfile=None):
273  hasheader = False
274  benches = collections.deque([(src, config, FLAGS.max_stddev)
275                               for src in srcs
276                               for config in configs])
277  while benches:
278    try:
279      with hardware:
280        SKPBench.run_warmup(hardware.warmup_time, configs[0])
281        if not hasheader:
282          emit_result(SKPBench.get_header(), resultsfile)
283          hasheader = True
284        while benches:
285          benchargs = benches.popleft()
286          with SKPBench(*benchargs) as skpbench:
287            try:
288              skpbench.execute(hardware)
289              if skpbench.best_result:
290                emit_result(skpbench.best_result.format(FLAGS.suffix),
291                            resultsfile)
292              else:
293                print("WARNING: no result for %s with config %s" %
294                      (skpbench.src, skpbench.config), file=sys.stderr)
295
296            except StddevException:
297              retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
298              if FLAGS.verbosity >= 1:
299                print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
300                      "re-queuing with max=%.2f%%." %
301                      (skpbench.best_result.config, skpbench.best_result.bench,
302                       skpbench.best_result.stddev, skpbench.max_stddev,
303                       retry_max_stddev),
304                      file=sys.stderr)
305              benches.append((skpbench.src, skpbench.config, retry_max_stddev,
306                              skpbench.best_result))
307
308            except HardwareException as exception:
309              skpbench.terminate()
310              if FLAGS.verbosity >= 4:
311                hardware.print_debug_diagnostics()
312              if FLAGS.verbosity >= 1:
313                print("%s; rebooting and taking a %i second nap..." %
314                      (exception.message, exception.sleeptime), file=sys.stderr)
315              benches.appendleft(benchargs) # retry the same bench next time.
316              raise # wake hw up from benchmarking mode before the nap.
317
318    except HardwareException as exception:
319      time.sleep(exception.sleeptime)
320
321def main():
322  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
323  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
324  configs = re.split(DELIMITER, FLAGS.config)
325  srcs = _path.find_skps(FLAGS.srcs)
326
327  if FLAGS.adb:
328    adb = Adb(FLAGS.device_serial, FLAGS.adb_binary,
329              echo=(FLAGS.verbosity >= 5))
330    model = adb.check('getprop ro.product.model').strip()
331    if model == 'Pixel C':
332      from _hardware_pixel_c import HardwarePixelC
333      hardware = HardwarePixelC(adb)
334    elif model == 'Pixel':
335      from _hardware_pixel import HardwarePixel
336      hardware = HardwarePixel(adb)
337    elif model == 'Pixel 2':
338      from _hardware_pixel2 import HardwarePixel2
339      hardware = HardwarePixel2(adb)
340    elif model == 'Nexus 6P':
341      from _hardware_nexus_6p import HardwareNexus6P
342      hardware = HardwareNexus6P(adb)
343    else:
344      from _hardware_android import HardwareAndroid
345      print("WARNING: %s: don't know how to monitor this hardware; results "
346            "may be unreliable." % model, file=sys.stderr)
347      hardware = HardwareAndroid(adb)
348  else:
349    hardware = Hardware()
350
351  if FLAGS.resultsfile:
352    with open(FLAGS.resultsfile, mode='a+') as resultsfile:
353      run_benchmarks(configs, srcs, hardware, resultsfile=resultsfile)
354  else:
355    run_benchmarks(configs, srcs, hardware)
356
357
358if __name__ == '__main__':
359  main()
360