1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2016 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Given a specially-formatted JSON object, generates results report(s).
8
9The JSON object should look like:
10{"data": BenchmarkData, "platforms": BenchmarkPlatforms}
11
12BenchmarkPlatforms is a [str], each of which names a platform the benchmark
13  was run on (e.g. peppy, shamu, ...). Note that the order of this list is
14  related with the order of items in BenchmarkData.
15
16BenchmarkData is a {str: [PlatformData]}. The str is the name of the benchmark,
17and a PlatformData is a set of data for a given platform. There must be one
18PlatformData for each benchmark, for each element in BenchmarkPlatforms.
19
20A PlatformData is a [{str: float}], where each str names a metric we recorded,
21and the float is the value for that metric. Each element is considered to be
22the metrics collected from an independent run of this benchmark. NOTE: Each
23PlatformData is expected to have a "retval" key, with the return value of
24the benchmark. If the benchmark is successful, said return value should be 0.
25Otherwise, this will break some of our JSON functionality.
26
27Putting it all together, a JSON object will end up looking like:
28  { "platforms": ["peppy", "peppy-new-crosstool"],
29    "data": {
30      "bench_draw_line": [
31        [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0},
32         {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}],
33        [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0},
34         {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}]
35      ]
36    }
37  }
38
39Which says that we ran a benchmark on platforms named peppy, and
40  peppy-new-crosstool.
41We ran one benchmark, named bench_draw_line.
42It was run twice on each platform.
43Peppy's runs took 1.321ms and 1.920ms, while peppy-new-crosstool's took 1.221ms
44  and 1.423ms. None of the runs failed to complete.
45"""
46
47from __future__ import division
48from __future__ import print_function
49
50import argparse
51import functools
52import json
53import os
54import sys
55import traceback
56
57from results_report import BenchmarkResults
58from results_report import HTMLResultsReport
59from results_report import JSONResultsReport
60from results_report import TextResultsReport
61
62
63def CountBenchmarks(benchmark_runs):
64  """Counts the number of iterations for each benchmark in benchmark_runs."""
65
66  # Example input for benchmark_runs:
67  # {"bench": [[run1, run2, run3], [run1, run2, run3, run4]]}
68  def _MaxLen(results):
69    return 0 if not results else max(len(r) for r in results)
70
71  return [(name, _MaxLen(results)) for name, results in benchmark_runs.items()]
72
73
74def CutResultsInPlace(results, max_keys=50, complain_on_update=True):
75  """Limits the given benchmark results to max_keys keys in-place.
76
77  This takes the `data` field from the benchmark input, and mutates each
78  benchmark run to contain `max_keys` elements (ignoring special elements, like
79  "retval"). At the moment, it just selects the first `max_keys` keyvals,
80  alphabetically.
81
82  If complain_on_update is true, this will print a message noting that a
83  truncation occurred.
84
85  This returns the `results` object that was passed in, for convenience.
86
87  e.g.
88  >>> benchmark_data = {
89  ...   "bench_draw_line": [
90  ...     [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0},
91  ...      {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}],
92  ...     [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0},
93  ...      {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}]
94  ...   ]
95  ... }
96  >>> CutResultsInPlace(benchmark_data, max_keys=1, complain_on_update=False)
97  {
98    'bench_draw_line': [
99      [{'memory (mb)': 128.1, 'retval': 0},
100       {'memory (mb)': 128.4, 'retval': 0}],
101      [{'memory (mb)': 124.3, 'retval': 0},
102       {'memory (mb)': 123.9, 'retval': 0}]
103    ]
104  }
105  """
106  actually_updated = False
107  for bench_results in results.values():
108    for platform_results in bench_results:
109      for i, result in enumerate(platform_results):
110        # Keep the keys that come earliest when sorted alphabetically.
111        # Forcing alphabetical order is arbitrary, but necessary; otherwise,
112        # the keyvals we'd emit would depend on our iteration order through a
113        # map.
114        removable_keys = sorted(k for k in result if k != 'retval')
115        retained_keys = removable_keys[:max_keys]
116        platform_results[i] = {k: result[k] for k in retained_keys}
117        # retval needs to be passed through all of the time.
118        retval = result.get('retval')
119        if retval is not None:
120          platform_results[i]['retval'] = retval
121        actually_updated = actually_updated or \
122          len(retained_keys) != len(removable_keys)
123
124  if actually_updated and complain_on_update:
125    print(
126        'Warning: Some benchmark keyvals have been truncated.', file=sys.stderr)
127  return results
128
129
130def _PositiveInt(s):
131  i = int(s)
132  if i < 0:
133    raise argparse.ArgumentTypeError('%d is not a positive integer.' % (i,))
134  return i
135
136
137def _AccumulateActions(args):
138  """Given program arguments, determines what actions we want to run.
139
140  Returns [(ResultsReportCtor, str)], where ResultsReportCtor can construct a
141  ResultsReport, and the str is the file extension for the given report.
142  """
143  results = []
144  # The order of these is arbitrary.
145  if args.json:
146    results.append((JSONResultsReport, 'json'))
147  if args.text:
148    results.append((TextResultsReport, 'txt'))
149  if args.email:
150    email_ctor = functools.partial(TextResultsReport, email=True)
151    results.append((email_ctor, 'email'))
152  # We emit HTML if nothing else was specified.
153  if args.html or not results:
154    results.append((HTMLResultsReport, 'html'))
155  return results
156
157
158# Note: get_contents is a function, because it may be expensive (generating some
159# HTML reports takes O(seconds) on my machine, depending on the size of the
160# input data).
161def WriteFile(output_prefix, extension, get_contents, overwrite, verbose):
162  """Writes `contents` to a file named "${output_prefix}.${extension}".
163
164  get_contents should be a zero-args function that returns a string (of the
165  contents to write).
166  If output_prefix == '-', this writes to stdout.
167  If overwrite is False, this will not overwrite files.
168  """
169  if output_prefix == '-':
170    if verbose:
171      print('Writing %s report to stdout' % (extension,), file=sys.stderr)
172    sys.stdout.write(get_contents())
173    return
174
175  file_name = '%s.%s' % (output_prefix, extension)
176  if not overwrite and os.path.exists(file_name):
177    raise IOError('Refusing to write %s -- it already exists' % (file_name,))
178
179  with open(file_name, 'w') as out_file:
180    if verbose:
181      print('Writing %s report to %s' % (extension, file_name), file=sys.stderr)
182    out_file.write(get_contents())
183
184
185def RunActions(actions, benchmark_results, output_prefix, overwrite, verbose):
186  """Runs `actions`, returning True if all succeeded."""
187  failed = False
188
189  report_ctor = None  # Make the linter happy
190  for report_ctor, extension in actions:
191    try:
192      get_contents = lambda: report_ctor(benchmark_results).GetReport()
193      WriteFile(output_prefix, extension, get_contents, overwrite, verbose)
194    except Exception:
195      # Complain and move along; we may have more actions that might complete
196      # successfully.
197      failed = True
198      traceback.print_exc()
199  return not failed
200
201
202def PickInputFile(input_name):
203  """Given program arguments, returns file to read for benchmark input."""
204  return sys.stdin if input_name == '-' else open(input_name)
205
206
207def _NoPerfReport(_label_name, _benchmark_name, _benchmark_iteration):
208  return {}
209
210
211def _ParseArgs(argv):
212  parser = argparse.ArgumentParser(description='Turns JSON into results '
213                                   'report(s).')
214  parser.add_argument(
215      '-v',
216      '--verbose',
217      action='store_true',
218      help='Be a tiny bit more verbose.')
219  parser.add_argument(
220      '-f',
221      '--force',
222      action='store_true',
223      help='Overwrite existing results files.')
224  parser.add_argument(
225      '-o',
226      '--output',
227      default='report',
228      type=str,
229      help='Prefix of the output filename (default: report). '
230      '- means stdout.')
231  parser.add_argument(
232      '-i',
233      '--input',
234      required=True,
235      type=str,
236      help='Where to read the JSON from. - means stdin.')
237  parser.add_argument(
238      '-l',
239      '--statistic-limit',
240      default=0,
241      type=_PositiveInt,
242      help='The maximum number of benchmark statistics to '
243      'display from a single run. 0 implies unlimited.')
244  parser.add_argument(
245      '--json', action='store_true', help='Output a JSON report.')
246  parser.add_argument(
247      '--text', action='store_true', help='Output a text report.')
248  parser.add_argument(
249      '--email',
250      action='store_true',
251      help='Output a text report suitable for email.')
252  parser.add_argument(
253      '--html',
254      action='store_true',
255      help='Output an HTML report (this is the default if no '
256      'other output format is specified).')
257  return parser.parse_args(argv)
258
259
260def Main(argv):
261  args = _ParseArgs(argv)
262  with PickInputFile(args.input) as in_file:
263    raw_results = json.load(in_file)
264
265  platform_names = raw_results['platforms']
266  results = raw_results['data']
267  if args.statistic_limit:
268    results = CutResultsInPlace(results, max_keys=args.statistic_limit)
269  benches = CountBenchmarks(results)
270  # In crosperf, a label is essentially a platform+configuration. So, a name of
271  # a label and a name of a platform are equivalent for our purposes.
272  bench_results = BenchmarkResults(
273      label_names=platform_names,
274      benchmark_names_and_iterations=benches,
275      run_keyvals=results,
276      read_perf_report=_NoPerfReport)
277  actions = _AccumulateActions(args)
278  ok = RunActions(actions, bench_results, args.output, args.force, args.verbose)
279  return 0 if ok else 1
280
281
282if __name__ == '__main__':
283  sys.exit(Main(sys.argv[1:]))
284