1#!/usr/bin/env python
2# Copyright 2014 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""
7Performance runner for d8.
8
9Call e.g. with tools/run-perf.py --arch ia32 some_suite.json
10
11The suite json format is expected to be:
12{
13  "path": <relative path chunks to perf resources and main file>,
14  "name": <optional suite name, file name is default>,
15  "archs": [<architecture name for which this suite is run>, ...],
16  "binary": <name of binary to run, default "d8">,
17  "flags": [<flag to d8>, ...],
18  "run_count": <how often will this suite run (optional)>,
19  "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
20  "resources": [<js file to be loaded before main>, ...]
21  "main": <main js perf runner file>,
22  "results_regexp": <optional regexp>,
23  "results_processor": <optional python results processor script>,
24  "units": <the unit specification for the performance dashboard>,
25  "tests": [
26    {
27      "name": <name of the trace>,
28      "results_regexp": <optional more specific regexp>,
29      "results_processor": <optional python results processor script>,
30      "units": <the unit specification for the performance dashboard>,
31    }, ...
32  ]
33}
34
35The tests field can also nest other suites in arbitrary depth. A suite
36with a "main" file is a leaf suite that can contain one more level of
37tests.
38
39A suite's results_regexp is expected to have one string place holder
40"%s" for the trace name. A trace's results_regexp overwrites suite
41defaults.
42
43A suite's results_processor may point to an optional python script. If
44specified, it is called after running the tests like this (with a path
45relatve to the suite level's path):
46<results_processor file> <same flags as for d8> <suite level name> <output>
47
48The <output> is a temporary file containing d8 output. The results_regexp will
49be applied to the output of this script.
50
51A suite without "tests" is considered a performance test itself.
52
53Full example (suite with one runner):
54{
55  "path": ["."],
56  "flags": ["--expose-gc"],
57  "archs": ["ia32", "x64"],
58  "run_count": 5,
59  "run_count_ia32": 3,
60  "main": "run.js",
61  "results_regexp": "^%s: (.+)$",
62  "units": "score",
63  "tests": [
64    {"name": "Richards"},
65    {"name": "DeltaBlue"},
66    {"name": "NavierStokes",
67     "results_regexp": "^NavierStokes: (.+)$"}
68  ]
69}
70
71Full example (suite with several runners):
72{
73  "path": ["."],
74  "flags": ["--expose-gc"],
75  "archs": ["ia32", "x64"],
76  "run_count": 5,
77  "units": "score",
78  "tests": [
79    {"name": "Richards",
80     "path": ["richards"],
81     "main": "run.js",
82     "run_count": 3,
83     "results_regexp": "^Richards: (.+)$"},
84    {"name": "NavierStokes",
85     "path": ["navier_stokes"],
86     "main": "run.js",
87     "results_regexp": "^NavierStokes: (.+)$"}
88  ]
89}
90
91Path pieces are concatenated. D8 is always run with the suite's path as cwd.
92"""
93
94import json
95import math
96import optparse
97import os
98import re
99import sys
100
101from testrunner.local import commands
102from testrunner.local import utils
103
104ARCH_GUESS = utils.DefaultArch()
105SUPPORTED_ARCHS = ["android_arm",
106                   "android_arm64",
107                   "android_ia32",
108                   "arm",
109                   "ia32",
110                   "mips",
111                   "mipsel",
112                   "nacl_ia32",
113                   "nacl_x64",
114                   "x64",
115                   "arm64"]
116
117GENERIC_RESULTS_RE = re.compile(
118    r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$")
119
120
121def GeometricMean(values):
122  """Returns the geometric mean of a list of values.
123
124  The mean is calculated using log to avoid overflow.
125  """
126  values = map(float, values)
127  return str(math.exp(sum(map(math.log, values)) / len(values)))
128
129
130class Results(object):
131  """Place holder for result traces."""
132  def __init__(self, traces=None, errors=None):
133    self.traces = traces or []
134    self.errors = errors or []
135
136  def ToDict(self):
137    return {"traces": self.traces, "errors": self.errors}
138
139  def WriteToFile(self, file_name):
140    with open(file_name, "w") as f:
141      f.write(json.dumps(self.ToDict()))
142
143  def __add__(self, other):
144    self.traces += other.traces
145    self.errors += other.errors
146    return self
147
148  def __str__(self):  # pragma: no cover
149    return str(self.ToDict())
150
151
152class Node(object):
153  """Represents a node in the suite tree structure."""
154  def __init__(self, *args):
155    self._children = []
156
157  def AppendChild(self, child):
158    self._children.append(child)
159
160
161class DefaultSentinel(Node):
162  """Fake parent node with all default values."""
163  def __init__(self):
164    super(DefaultSentinel, self).__init__()
165    self.binary = "d8"
166    self.run_count = 10
167    self.timeout = 60
168    self.path = []
169    self.graphs = []
170    self.flags = []
171    self.resources = []
172    self.results_regexp = None
173    self.stddev_regexp = None
174    self.units = "score"
175    self.total = False
176
177
178class Graph(Node):
179  """Represents a suite definition.
180
181  Can either be a leaf or an inner node that provides default values.
182  """
183  def __init__(self, suite, parent, arch):
184    super(Graph, self).__init__()
185    self._suite = suite
186
187    assert isinstance(suite.get("path", []), list)
188    assert isinstance(suite["name"], basestring)
189    assert isinstance(suite.get("flags", []), list)
190    assert isinstance(suite.get("resources", []), list)
191
192    # Accumulated values.
193    self.path = parent.path[:] + suite.get("path", [])
194    self.graphs = parent.graphs[:] + [suite["name"]]
195    self.flags = parent.flags[:] + suite.get("flags", [])
196    self.resources = parent.resources[:] + suite.get("resources", [])
197
198    # Descrete values (with parent defaults).
199    self.binary = suite.get("binary", parent.binary)
200    self.run_count = suite.get("run_count", parent.run_count)
201    self.run_count = suite.get("run_count_%s" % arch, self.run_count)
202    self.timeout = suite.get("timeout", parent.timeout)
203    self.units = suite.get("units", parent.units)
204    self.total = suite.get("total", parent.total)
205
206    # A regular expression for results. If the parent graph provides a
207    # regexp and the current suite has none, a string place holder for the
208    # suite name is expected.
209    # TODO(machenbach): Currently that makes only sense for the leaf level.
210    # Multiple place holders for multiple levels are not supported.
211    if parent.results_regexp:
212      regexp_default = parent.results_regexp % re.escape(suite["name"])
213    else:
214      regexp_default = None
215    self.results_regexp = suite.get("results_regexp", regexp_default)
216
217    # A similar regular expression for the standard deviation (optional).
218    if parent.stddev_regexp:
219      stddev_default = parent.stddev_regexp % re.escape(suite["name"])
220    else:
221      stddev_default = None
222    self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
223
224
225class Trace(Graph):
226  """Represents a leaf in the suite tree structure.
227
228  Handles collection of measurements.
229  """
230  def __init__(self, suite, parent, arch):
231    super(Trace, self).__init__(suite, parent, arch)
232    assert self.results_regexp
233    self.results = []
234    self.errors = []
235    self.stddev = ""
236
237  def ConsumeOutput(self, stdout):
238    try:
239      self.results.append(
240          re.search(self.results_regexp, stdout, re.M).group(1))
241    except:
242      self.errors.append("Regexp \"%s\" didn't match for test %s."
243                         % (self.results_regexp, self.graphs[-1]))
244
245    try:
246      if self.stddev_regexp and self.stddev:
247        self.errors.append("Test %s should only run once since a stddev "
248                           "is provided by the test." % self.graphs[-1])
249      if self.stddev_regexp:
250        self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
251    except:
252      self.errors.append("Regexp \"%s\" didn't match for test %s."
253                         % (self.stddev_regexp, self.graphs[-1]))
254
255  def GetResults(self):
256    return Results([{
257      "graphs": self.graphs,
258      "units": self.units,
259      "results": self.results,
260      "stddev": self.stddev,
261    }], self.errors)
262
263
264class Runnable(Graph):
265  """Represents a runnable suite definition (i.e. has a main file).
266  """
267  @property
268  def main(self):
269    return self._suite.get("main", "")
270
271  def ChangeCWD(self, suite_path):
272    """Changes the cwd to to path defined in the current graph.
273
274    The tests are supposed to be relative to the suite configuration.
275    """
276    suite_dir = os.path.abspath(os.path.dirname(suite_path))
277    bench_dir = os.path.normpath(os.path.join(*self.path))
278    os.chdir(os.path.join(suite_dir, bench_dir))
279
280  def GetCommand(self, shell_dir):
281    # TODO(machenbach): This requires +.exe if run on windows.
282    return (
283      [os.path.join(shell_dir, self.binary)] +
284      self.flags +
285      self.resources +
286      [self.main]
287    )
288
289  def Run(self, runner):
290    """Iterates over several runs and handles the output for all traces."""
291    for stdout in runner():
292      for trace in self._children:
293        trace.ConsumeOutput(stdout)
294    res = reduce(lambda r, t: r + t.GetResults(), self._children, Results())
295
296    if not res.traces or not self.total:
297      return res
298
299    # Assume all traces have the same structure.
300    if len(set(map(lambda t: len(t["results"]), res.traces))) != 1:
301      res.errors.append("Not all traces have the same number of results.")
302      return res
303
304    # Calculate the geometric means for all traces. Above we made sure that
305    # there is at least one trace and that the number of results is the same
306    # for each trace.
307    n_results = len(res.traces[0]["results"])
308    total_results = [GeometricMean(t["results"][i] for t in res.traces)
309                     for i in range(0, n_results)]
310    res.traces.append({
311      "graphs": self.graphs + ["Total"],
312      "units": res.traces[0]["units"],
313      "results": total_results,
314      "stddev": "",
315    })
316    return res
317
318class RunnableTrace(Trace, Runnable):
319  """Represents a runnable suite definition that is a leaf."""
320  def __init__(self, suite, parent, arch):
321    super(RunnableTrace, self).__init__(suite, parent, arch)
322
323  def Run(self, runner):
324    """Iterates over several runs and handles the output."""
325    for stdout in runner():
326      self.ConsumeOutput(stdout)
327    return self.GetResults()
328
329
330class RunnableGeneric(Runnable):
331  """Represents a runnable suite definition with generic traces."""
332  def __init__(self, suite, parent, arch):
333    super(RunnableGeneric, self).__init__(suite, parent, arch)
334
335  def Run(self, runner):
336    """Iterates over several runs and handles the output."""
337    traces = {}
338    for stdout in runner():
339      for line in stdout.strip().splitlines():
340        match = GENERIC_RESULTS_RE.match(line)
341        if match:
342          trace = match.group(1)
343          result = match.group(2)
344          stddev = match.group(3)
345          trace_result = traces.setdefault(trace, Results([{
346            "graphs": self.graphs + [trace],
347            "units": self.units,
348            "results": [],
349            "stddev": "",
350          }], []))
351          trace_result.traces[0]["results"].append(result)
352          trace_result.traces[0]["stddev"] = stddev
353
354    return reduce(lambda r, t: r + t, traces.itervalues(), Results())
355
356
357def MakeGraph(suite, arch, parent):
358  """Factory method for making graph objects."""
359  if isinstance(parent, Runnable):
360    # Below a runnable can only be traces.
361    return Trace(suite, parent, arch)
362  elif suite.get("main"):
363    # A main file makes this graph runnable.
364    if suite.get("tests"):
365      # This graph has subgraphs (traces).
366      return Runnable(suite, parent, arch)
367    else:
368      # This graph has no subgraphs, it's a leaf.
369      return RunnableTrace(suite, parent, arch)
370  elif suite.get("generic"):
371    # This is a generic suite definition. It is either a runnable executable
372    # or has a main js file.
373    return RunnableGeneric(suite, parent, arch)
374  elif suite.get("tests"):
375    # This is neither a leaf nor a runnable.
376    return Graph(suite, parent, arch)
377  else:  # pragma: no cover
378    raise Exception("Invalid suite configuration.")
379
380
381def BuildGraphs(suite, arch, parent=None):
382  """Builds a tree structure of graph objects that corresponds to the suite
383  configuration.
384  """
385  parent = parent or DefaultSentinel()
386
387  # TODO(machenbach): Implement notion of cpu type?
388  if arch not in suite.get("archs", ["ia32", "x64"]):
389    return None
390
391  graph = MakeGraph(suite, arch, parent)
392  for subsuite in suite.get("tests", []):
393    BuildGraphs(subsuite, arch, graph)
394  parent.AppendChild(graph)
395  return graph
396
397
398def FlattenRunnables(node):
399  """Generator that traverses the tree structure and iterates over all
400  runnables.
401  """
402  if isinstance(node, Runnable):
403    yield node
404  elif isinstance(node, Node):
405    for child in node._children:
406      for result in FlattenRunnables(child):
407        yield result
408  else:  # pragma: no cover
409    raise Exception("Invalid suite configuration.")
410
411
412# TODO: Implement results_processor.
413def Main(args):
414  parser = optparse.OptionParser()
415  parser.add_option("--arch",
416                    help=("The architecture to run tests for, "
417                          "'auto' or 'native' for auto-detect"),
418                    default="x64")
419  parser.add_option("--buildbot",
420                    help="Adapt to path structure used on buildbots",
421                    default=False, action="store_true")
422  parser.add_option("--json-test-results",
423                    help="Path to a file for storing json results.")
424  parser.add_option("--outdir", help="Base directory with compile output",
425                    default="out")
426  (options, args) = parser.parse_args(args)
427
428  if len(args) == 0:  # pragma: no cover
429    parser.print_help()
430    return 1
431
432  if options.arch in ["auto", "native"]:  # pragma: no cover
433    options.arch = ARCH_GUESS
434
435  if not options.arch in SUPPORTED_ARCHS:  # pragma: no cover
436    print "Unknown architecture %s" % options.arch
437    return 1
438
439  workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
440
441  if options.buildbot:
442    shell_dir = os.path.join(workspace, options.outdir, "Release")
443  else:
444    shell_dir = os.path.join(workspace, options.outdir,
445                             "%s.release" % options.arch)
446
447  results = Results()
448  for path in args:
449    path = os.path.abspath(path)
450
451    if not os.path.exists(path):  # pragma: no cover
452      results.errors.append("Configuration file %s does not exist." % path)
453      continue
454
455    with open(path) as f:
456      suite = json.loads(f.read())
457
458    # If no name is given, default to the file name without .json.
459    suite.setdefault("name", os.path.splitext(os.path.basename(path))[0])
460
461    for runnable in FlattenRunnables(BuildGraphs(suite, options.arch)):
462      print ">>> Running suite: %s" % "/".join(runnable.graphs)
463      runnable.ChangeCWD(path)
464
465      def Runner():
466        """Output generator that reruns several times."""
467        for i in xrange(0, max(1, runnable.run_count)):
468          # TODO(machenbach): Allow timeout per arch like with run_count per
469          # arch.
470          output = commands.Execute(runnable.GetCommand(shell_dir),
471                                    timeout=runnable.timeout)
472          print ">>> Stdout (#%d):" % (i + 1)
473          print output.stdout
474          if output.stderr:  # pragma: no cover
475            # Print stderr for debugging.
476            print ">>> Stderr (#%d):" % (i + 1)
477            print output.stderr
478          if output.timed_out:
479            print ">>> Test timed out after %ss." % runnable.timeout
480          yield output.stdout
481
482      # Let runnable iterate over all runs and handle output.
483      results += runnable.Run(Runner)
484
485  if options.json_test_results:
486    results.WriteToFile(options.json_test_results)
487  else:  # pragma: no cover
488    print results
489
490  return min(1, len(results.errors))
491
492if __name__ == "__main__":  # pragma: no cover
493  sys.exit(Main(sys.argv[1:]))
494