• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2015 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5'''
6python %prog
7
8Convert a perf trybot JSON file into a pleasing HTML page. It can read
9from standard input or via the --filename option. Examples:
10
11  cat results.json | %prog --title "ia32 results"
12  %prog -f results.json -t "ia32 results" -o results.html
13'''
14
15import json
16import math
17from optparse import OptionParser
18import os
19import shutil
20import sys
21import tempfile
22
23PERCENT_CONSIDERED_SIGNIFICANT = 0.5
24PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
25PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
26
27
28def ComputeZ(baseline_avg, baseline_sigma, mean, n):
29  if baseline_sigma == 0:
30    return 1000.0;
31  return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
32
33
34# Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
35def ComputeProbability(z):
36  if z > 2.575829: # p 0.005: two sided < 0.01
37    return 0
38  if z > 2.326348: # p 0.010
39    return 0.01
40  if z > 2.170091: # p 0.015
41    return 0.02
42  if z > 2.053749: # p 0.020
43    return 0.03
44  if z > 1.959964: # p 0.025: two sided < 0.05
45    return 0.04
46  if z > 1.880793: # p 0.030
47    return 0.05
48  if z > 1.811910: # p 0.035
49    return 0.06
50  if z > 1.750686: # p 0.040
51    return 0.07
52  if z > 1.695397: # p 0.045
53    return 0.08
54  if z > 1.644853: # p 0.050: two sided < 0.10
55    return 0.09
56  if z > 1.281551: # p 0.100: two sided < 0.20
57    return 0.10
58  return 0.20 # two sided p >= 0.20
59
60
61class Result:
62  def __init__(self, test_name, count, hasScoreUnits, result, sigma,
63               master_result, master_sigma):
64    self.result_ = float(result)
65    self.sigma_ = float(sigma)
66    self.master_result_ = float(master_result)
67    self.master_sigma_ = float(master_sigma)
68    self.significant_ = False
69    self.notable_ = 0
70    self.percentage_string_ = ""
71    # compute notability and significance.
72    try:
73      if hasScoreUnits:
74        compare_num = 100*self.result_/self.master_result_ - 100
75      else:
76        compare_num = 100*self.master_result_/self.result_ - 100
77      if abs(compare_num) > 0.1:
78        self.percentage_string_ = "%3.1f" % (compare_num)
79        z = ComputeZ(self.master_result_, self.master_sigma_, self.result_, count)
80        p = ComputeProbability(z)
81        if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
82          self.significant_ = True
83        if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
84          self.notable_ = 1
85        elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
86          self.notable_ = -1
87    except ZeroDivisionError:
88      self.percentage_string_ = "NaN"
89      self.significant_ = True
90
91  def result(self):
92    return self.result_
93
94  def sigma(self):
95    return self.sigma_
96
97  def master_result(self):
98    return self.master_result_
99
100  def master_sigma(self):
101    return self.master_sigma_
102
103  def percentage_string(self):
104    return self.percentage_string_;
105
106  def isSignificant(self):
107    return self.significant_
108
109  def isNotablyPositive(self):
110    return self.notable_ > 0
111
112  def isNotablyNegative(self):
113    return self.notable_ < 0
114
115
116class Benchmark:
117  def __init__(self, name, data):
118    self.name_ = name
119    self.tests_ = {}
120    for test in data:
121      # strip off "<name>/" prefix, allowing for subsequent "/"s
122      test_name = test.split("/", 1)[1]
123      self.appendResult(test_name, data[test])
124
125  # tests is a dictionary of Results
126  def tests(self):
127    return self.tests_
128
129  def SortedTestKeys(self):
130    keys = self.tests_.keys()
131    keys.sort()
132    t = "Total"
133    if t in keys:
134      keys.remove(t)
135      keys.append(t)
136    return keys
137
138  def name(self):
139    return self.name_
140
141  def appendResult(self, test_name, test_data):
142    with_string = test_data["result with patch   "]
143    data = with_string.split()
144    master_string = test_data["result without patch"]
145    master_data = master_string.split()
146    runs = int(test_data["runs"])
147    units = test_data["units"]
148    hasScoreUnits = units == "score"
149    self.tests_[test_name] = Result(test_name,
150                                    runs,
151                                    hasScoreUnits,
152                                    data[0], data[2],
153                                    master_data[0], master_data[2])
154
155
156class BenchmarkRenderer:
157  def __init__(self, output_file):
158    self.print_output_ = []
159    self.output_file_ = output_file
160
161  def Print(self, str_data):
162    self.print_output_.append(str_data)
163
164  def FlushOutput(self):
165    string_data = "\n".join(self.print_output_)
166    print_output = []
167    if self.output_file_:
168      # create a file
169      with open(self.output_file_, "w") as text_file:
170        text_file.write(string_data)
171    else:
172      print(string_data)
173
174  def RenderOneBenchmark(self, benchmark):
175    self.Print("<h2>")
176    self.Print("<a name=\"" + benchmark.name() + "\">")
177    self.Print(benchmark.name() + "</a> <a href=\"#top\">(top)</a>")
178    self.Print("</h2>");
179    self.Print("<table class=\"benchmark\">")
180    self.Print("<thead>")
181    self.Print("  <th>Test</th>")
182    self.Print("  <th>Result</th>")
183    self.Print("  <th>Master</th>")
184    self.Print("  <th>%</th>")
185    self.Print("</thead>")
186    self.Print("<tbody>")
187    tests = benchmark.tests()
188    for test in benchmark.SortedTestKeys():
189      t = tests[test]
190      self.Print("  <tr>")
191      self.Print("    <td>" + test + "</td>")
192      self.Print("    <td>" + str(t.result()) + "</td>")
193      self.Print("    <td>" + str(t.master_result()) + "</td>")
194      t = tests[test]
195      res = t.percentage_string()
196      if t.isSignificant():
197        res = self.bold(res)
198      if t.isNotablyPositive():
199        res = self.green(res)
200      elif t.isNotablyNegative():
201        res = self.red(res)
202      self.Print("    <td>" + res + "</td>")
203      self.Print("  </tr>")
204    self.Print("</tbody>")
205    self.Print("</table>")
206
207  def ProcessJSONData(self, data, title):
208    self.Print("<h1>" + title + "</h1>")
209    self.Print("<ul>")
210    for benchmark in data:
211     if benchmark != "errors":
212       self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
213    self.Print("</ul>")
214    for benchmark in data:
215      if benchmark != "errors":
216        benchmark_object = Benchmark(benchmark, data[benchmark])
217        self.RenderOneBenchmark(benchmark_object)
218
219  def bold(self, data):
220    return "<b>" + data + "</b>"
221
222  def red(self, data):
223    return "<font color=\"red\">" + data + "</font>"
224
225
226  def green(self, data):
227    return "<font color=\"green\">" + data + "</font>"
228
229  def PrintHeader(self):
230    data = """<html>
231<head>
232<title>Output</title>
233<style type="text/css">
234/*
235Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
236*/
237body {
238  font-family: Helvetica, arial, sans-serif;
239  font-size: 14px;
240  line-height: 1.6;
241  padding-top: 10px;
242  padding-bottom: 10px;
243  background-color: white;
244  padding: 30px;
245}
246h1, h2, h3, h4, h5, h6 {
247  margin: 20px 0 10px;
248  padding: 0;
249  font-weight: bold;
250  -webkit-font-smoothing: antialiased;
251  cursor: text;
252  position: relative;
253}
254h1 {
255  font-size: 28px;
256  color: black;
257}
258
259h2 {
260  font-size: 24px;
261  border-bottom: 1px solid #cccccc;
262  color: black;
263}
264
265h3 {
266  font-size: 18px;
267}
268
269h4 {
270  font-size: 16px;
271}
272
273h5 {
274  font-size: 14px;
275}
276
277h6 {
278  color: #777777;
279  font-size: 14px;
280}
281
282p, blockquote, ul, ol, dl, li, table, pre {
283  margin: 15px 0;
284}
285
286li p.first {
287  display: inline-block;
288}
289
290ul, ol {
291  padding-left: 30px;
292}
293
294ul :first-child, ol :first-child {
295  margin-top: 0;
296}
297
298ul :last-child, ol :last-child {
299  margin-bottom: 0;
300}
301
302table {
303  padding: 0;
304}
305
306table tr {
307  border-top: 1px solid #cccccc;
308  background-color: white;
309  margin: 0;
310  padding: 0;
311}
312
313table tr:nth-child(2n) {
314  background-color: #f8f8f8;
315}
316
317table tr th {
318  font-weight: bold;
319  border: 1px solid #cccccc;
320  text-align: left;
321  margin: 0;
322  padding: 6px 13px;
323}
324table tr td {
325  border: 1px solid #cccccc;
326  text-align: left;
327  margin: 0;
328  padding: 6px 13px;
329}
330table tr th :first-child, table tr td :first-child {
331  margin-top: 0;
332}
333table tr th :last-child, table tr td :last-child {
334  margin-bottom: 0;
335}
336</style>
337</head>
338<body>
339"""
340    self.Print(data)
341
342  def PrintFooter(self):
343    data = """</body>
344</html>
345"""
346    self.Print(data)
347
348
349def Render(opts, args):
350  if opts.filename:
351    with open(opts.filename) as json_data:
352      data = json.load(json_data)
353  else:
354    # load data from stdin
355    data = json.load(sys.stdin)
356
357  if opts.title:
358    title = opts.title
359  elif opts.filename:
360    title = opts.filename
361  else:
362    title = "Benchmark results"
363  renderer = BenchmarkRenderer(opts.output)
364  renderer.PrintHeader()
365  renderer.ProcessJSONData(data, title)
366  renderer.PrintFooter()
367  renderer.FlushOutput()
368
369
370if __name__ == '__main__':
371  parser = OptionParser(usage=__doc__)
372  parser.add_option("-f", "--filename", dest="filename",
373                    help="Specifies the filename for the JSON results "
374                         "rather than reading from stdin.")
375  parser.add_option("-t", "--title", dest="title",
376                    help="Optional title of the web page.")
377  parser.add_option("-o", "--output", dest="output",
378                    help="Write html output to this file rather than stdout.")
379
380  (opts, args) = parser.parse_args()
381  Render(opts, args)
382