1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2016 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Given a specially-formatted JSON object, generates results report(s). 8 9The JSON object should look like: 10{"data": BenchmarkData, "platforms": BenchmarkPlatforms} 11 12BenchmarkPlatforms is a [str], each of which names a platform the benchmark 13 was run on (e.g. peppy, shamu, ...). Note that the order of this list is 14 related with the order of items in BenchmarkData. 15 16BenchmarkData is a {str: [PlatformData]}. The str is the name of the benchmark, 17and a PlatformData is a set of data for a given platform. There must be one 18PlatformData for each benchmark, for each element in BenchmarkPlatforms. 19 20A PlatformData is a [{str: float}], where each str names a metric we recorded, 21and the float is the value for that metric. Each element is considered to be 22the metrics collected from an independent run of this benchmark. NOTE: Each 23PlatformData is expected to have a "retval" key, with the return value of 24the benchmark. If the benchmark is successful, said return value should be 0. 25Otherwise, this will break some of our JSON functionality. 26 27Putting it all together, a JSON object will end up looking like: 28 { "platforms": ["peppy", "peppy-new-crosstool"], 29 "data": { 30 "bench_draw_line": [ 31 [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0}, 32 {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}], 33 [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0}, 34 {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}] 35 ] 36 } 37 } 38 39Which says that we ran a benchmark on platforms named peppy, and 40 peppy-new-crosstool. 41We ran one benchmark, named bench_draw_line. 42It was run twice on each platform. 43Peppy's runs took 1.321ms and 1.920ms, while peppy-new-crosstool's took 1.221ms 44 and 1.423ms. None of the runs failed to complete. 45""" 46 47from __future__ import division 48from __future__ import print_function 49 50import argparse 51import functools 52import json 53import os 54import sys 55import traceback 56 57from results_report import BenchmarkResults 58from results_report import HTMLResultsReport 59from results_report import JSONResultsReport 60from results_report import TextResultsReport 61 62 63def CountBenchmarks(benchmark_runs): 64 """Counts the number of iterations for each benchmark in benchmark_runs.""" 65 66 # Example input for benchmark_runs: 67 # {"bench": [[run1, run2, run3], [run1, run2, run3, run4]]} 68 def _MaxLen(results): 69 return 0 if not results else max(len(r) for r in results) 70 71 return [(name, _MaxLen(results)) for name, results in benchmark_runs.items()] 72 73 74def CutResultsInPlace(results, max_keys=50, complain_on_update=True): 75 """Limits the given benchmark results to max_keys keys in-place. 76 77 This takes the `data` field from the benchmark input, and mutates each 78 benchmark run to contain `max_keys` elements (ignoring special elements, like 79 "retval"). At the moment, it just selects the first `max_keys` keyvals, 80 alphabetically. 81 82 If complain_on_update is true, this will print a message noting that a 83 truncation occurred. 84 85 This returns the `results` object that was passed in, for convenience. 86 87 e.g. 88 >>> benchmark_data = { 89 ... "bench_draw_line": [ 90 ... [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0}, 91 ... {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}], 92 ... [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0}, 93 ... {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}] 94 ... ] 95 ... } 96 >>> CutResultsInPlace(benchmark_data, max_keys=1, complain_on_update=False) 97 { 98 'bench_draw_line': [ 99 [{'memory (mb)': 128.1, 'retval': 0}, 100 {'memory (mb)': 128.4, 'retval': 0}], 101 [{'memory (mb)': 124.3, 'retval': 0}, 102 {'memory (mb)': 123.9, 'retval': 0}] 103 ] 104 } 105 """ 106 actually_updated = False 107 for bench_results in results.values(): 108 for platform_results in bench_results: 109 for i, result in enumerate(platform_results): 110 # Keep the keys that come earliest when sorted alphabetically. 111 # Forcing alphabetical order is arbitrary, but necessary; otherwise, 112 # the keyvals we'd emit would depend on our iteration order through a 113 # map. 114 removable_keys = sorted(k for k in result if k != 'retval') 115 retained_keys = removable_keys[:max_keys] 116 platform_results[i] = {k: result[k] for k in retained_keys} 117 # retval needs to be passed through all of the time. 118 retval = result.get('retval') 119 if retval is not None: 120 platform_results[i]['retval'] = retval 121 actually_updated = actually_updated or \ 122 len(retained_keys) != len(removable_keys) 123 124 if actually_updated and complain_on_update: 125 print( 126 'Warning: Some benchmark keyvals have been truncated.', file=sys.stderr) 127 return results 128 129 130def _PositiveInt(s): 131 i = int(s) 132 if i < 0: 133 raise argparse.ArgumentTypeError('%d is not a positive integer.' % (i,)) 134 return i 135 136 137def _AccumulateActions(args): 138 """Given program arguments, determines what actions we want to run. 139 140 Returns [(ResultsReportCtor, str)], where ResultsReportCtor can construct a 141 ResultsReport, and the str is the file extension for the given report. 142 """ 143 results = [] 144 # The order of these is arbitrary. 145 if args.json: 146 results.append((JSONResultsReport, 'json')) 147 if args.text: 148 results.append((TextResultsReport, 'txt')) 149 if args.email: 150 email_ctor = functools.partial(TextResultsReport, email=True) 151 results.append((email_ctor, 'email')) 152 # We emit HTML if nothing else was specified. 153 if args.html or not results: 154 results.append((HTMLResultsReport, 'html')) 155 return results 156 157 158# Note: get_contents is a function, because it may be expensive (generating some 159# HTML reports takes O(seconds) on my machine, depending on the size of the 160# input data). 161def WriteFile(output_prefix, extension, get_contents, overwrite, verbose): 162 """Writes `contents` to a file named "${output_prefix}.${extension}". 163 164 get_contents should be a zero-args function that returns a string (of the 165 contents to write). 166 If output_prefix == '-', this writes to stdout. 167 If overwrite is False, this will not overwrite files. 168 """ 169 if output_prefix == '-': 170 if verbose: 171 print('Writing %s report to stdout' % (extension,), file=sys.stderr) 172 sys.stdout.write(get_contents()) 173 return 174 175 file_name = '%s.%s' % (output_prefix, extension) 176 if not overwrite and os.path.exists(file_name): 177 raise IOError('Refusing to write %s -- it already exists' % (file_name,)) 178 179 with open(file_name, 'w') as out_file: 180 if verbose: 181 print('Writing %s report to %s' % (extension, file_name), file=sys.stderr) 182 out_file.write(get_contents()) 183 184 185def RunActions(actions, benchmark_results, output_prefix, overwrite, verbose): 186 """Runs `actions`, returning True if all succeeded.""" 187 failed = False 188 189 report_ctor = None # Make the linter happy 190 for report_ctor, extension in actions: 191 try: 192 get_contents = lambda: report_ctor(benchmark_results).GetReport() 193 WriteFile(output_prefix, extension, get_contents, overwrite, verbose) 194 except Exception: 195 # Complain and move along; we may have more actions that might complete 196 # successfully. 197 failed = True 198 traceback.print_exc() 199 return not failed 200 201 202def PickInputFile(input_name): 203 """Given program arguments, returns file to read for benchmark input.""" 204 return sys.stdin if input_name == '-' else open(input_name) 205 206 207def _NoPerfReport(_label_name, _benchmark_name, _benchmark_iteration): 208 return {} 209 210 211def _ParseArgs(argv): 212 parser = argparse.ArgumentParser(description='Turns JSON into results ' 213 'report(s).') 214 parser.add_argument( 215 '-v', 216 '--verbose', 217 action='store_true', 218 help='Be a tiny bit more verbose.') 219 parser.add_argument( 220 '-f', 221 '--force', 222 action='store_true', 223 help='Overwrite existing results files.') 224 parser.add_argument( 225 '-o', 226 '--output', 227 default='report', 228 type=str, 229 help='Prefix of the output filename (default: report). ' 230 '- means stdout.') 231 parser.add_argument( 232 '-i', 233 '--input', 234 required=True, 235 type=str, 236 help='Where to read the JSON from. - means stdin.') 237 parser.add_argument( 238 '-l', 239 '--statistic-limit', 240 default=0, 241 type=_PositiveInt, 242 help='The maximum number of benchmark statistics to ' 243 'display from a single run. 0 implies unlimited.') 244 parser.add_argument( 245 '--json', action='store_true', help='Output a JSON report.') 246 parser.add_argument( 247 '--text', action='store_true', help='Output a text report.') 248 parser.add_argument( 249 '--email', 250 action='store_true', 251 help='Output a text report suitable for email.') 252 parser.add_argument( 253 '--html', 254 action='store_true', 255 help='Output an HTML report (this is the default if no ' 256 'other output format is specified).') 257 return parser.parse_args(argv) 258 259 260def Main(argv): 261 args = _ParseArgs(argv) 262 with PickInputFile(args.input) as in_file: 263 raw_results = json.load(in_file) 264 265 platform_names = raw_results['platforms'] 266 results = raw_results['data'] 267 if args.statistic_limit: 268 results = CutResultsInPlace(results, max_keys=args.statistic_limit) 269 benches = CountBenchmarks(results) 270 # In crosperf, a label is essentially a platform+configuration. So, a name of 271 # a label and a name of a platform are equivalent for our purposes. 272 bench_results = BenchmarkResults( 273 label_names=platform_names, 274 benchmark_names_and_iterations=benches, 275 run_keyvals=results, 276 read_perf_report=_NoPerfReport) 277 actions = _AccumulateActions(args) 278 ok = RunActions(actions, bench_results, args.output, args.force, args.verbose) 279 return 0 if ok else 1 280 281 282if __name__ == '__main__': 283 sys.exit(Main(sys.argv[1:])) 284