1// Copyright 2012 the V8 project authors. All rights reserved.
2// Redistribution and use in source and binary forms, with or without
3// modification, are permitted provided that the following conditions are
4// met:
5//
6//     * Redistributions of source code must retain the above copyright
7//       notice, this list of conditions and the following disclaimer.
8//     * Redistributions in binary form must reproduce the above
9//       copyright notice, this list of conditions and the following
10//       disclaimer in the documentation and/or other materials provided
11//       with the distribution.
12//     * Neither the name of Google Inc. nor the names of its
13//       contributors may be used to endorse or promote products derived
14//       from this software without specific prior written permission.
15//
16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29// Simple framework for running the benchmark suites and
30// computing a score based on the timing measurements.
31
32
33// A benchmark has a name (string) and a function that will be run to
34// do the performance measurement. The optional setup and tearDown
35// arguments are functions that will be invoked before and after
36// running the benchmark, but the running time of these functions will
37// not be accounted for in the benchmark score.
38function Benchmark(name, run, setup, tearDown, minIterations) {
39  this.name = name;
40  this.run = run;
41  this.Setup = setup ? setup : function() { };
42  this.TearDown = tearDown ? tearDown : function() { };
43  this.minIterations = minIterations ? minIterations : 32;
44}
45
46
47// Benchmark results hold the benchmark and the measured time used to
48// run the benchmark. The benchmark score is computed later once a
49// full benchmark suite has run to completion.
50function BenchmarkResult(benchmark, time) {
51  this.benchmark = benchmark;
52  this.time = time;
53}
54
55
56// Automatically convert results to numbers. Used by the geometric
57// mean computation.
58BenchmarkResult.prototype.valueOf = function() {
59  return this.time;
60}
61
62
63// Suites of benchmarks consist of a name and the set of benchmarks in
64// addition to the reference timing that the final score will be based
65// on. This way, all scores are relative to a reference run and higher
66// scores implies better performance.
67function BenchmarkSuite(name, reference, benchmarks) {
68  this.name = name;
69  this.reference = reference;
70  this.benchmarks = benchmarks;
71  BenchmarkSuite.suites.push(this);
72}
73
74
75// Keep track of all declared benchmark suites.
76BenchmarkSuite.suites = [];
77
78
79// Scores are not comparable across versions. Bump the version if
80// you're making changes that will affect that scores, e.g. if you add
81// a new benchmark or change an existing one.
82BenchmarkSuite.version = '8';
83
84// Override the alert function to throw an exception instead.
85alert = function(s) {
86  throw "Alert called with argument: " + s;
87};
88
89// To make the benchmark results predictable, we replace Math.random
90// with a 100% deterministic alternative.
91Math.random = (function() {
92  var seed = 49734321;
93  return function() {
94    // Robert Jenkins' 32 bit integer hash function.
95    seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
96    seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
97    seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
98    seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
99    seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
100    seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
101    return (seed & 0xfffffff) / 0x10000000;
102  };
103})();
104
105
106// Runs all registered benchmark suites and optionally yields between
107// each individual benchmark to avoid running for too long in the
108// context of browsers. Once done, the final score is reported to the
109// runner.
110BenchmarkSuite.RunSuites = function(runner) {
111  var continuation = null;
112  var suites = BenchmarkSuite.suites;
113  var length = suites.length;
114  BenchmarkSuite.scores = [];
115  var index = 0;
116  function RunStep() {
117    while (continuation || index < length) {
118      if (continuation) {
119        continuation = continuation();
120      } else {
121        var suite = suites[index++];
122        if (runner.NotifyStart) runner.NotifyStart(suite.name);
123        continuation = suite.RunStep(runner);
124      }
125      if (continuation && typeof window != 'undefined' && window.setTimeout) {
126        window.setTimeout(RunStep, 25);
127        return;
128      }
129    }
130    if (runner.NotifyScore) {
131      var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores);
132      var formatted = BenchmarkSuite.FormatScore(100 * score);
133      runner.NotifyScore(formatted);
134    }
135  }
136  RunStep();
137}
138
139
140// Counts the total number of registered benchmarks. Useful for
141// showing progress as a percentage.
142BenchmarkSuite.CountBenchmarks = function() {
143  var result = 0;
144  var suites = BenchmarkSuite.suites;
145  for (var i = 0; i < suites.length; i++) {
146    result += suites[i].benchmarks.length;
147  }
148  return result;
149}
150
151
152// Computes the geometric mean of a set of numbers.
153BenchmarkSuite.GeometricMean = function(numbers) {
154  var log = 0;
155  for (var i = 0; i < numbers.length; i++) {
156    log += Math.log(numbers[i]);
157  }
158  return Math.pow(Math.E, log / numbers.length);
159}
160
161
162// Converts a score value to a string with at least three significant
163// digits.
164BenchmarkSuite.FormatScore = function(value) {
165  if (value > 100) {
166    return value.toFixed(0);
167  } else {
168    return value.toPrecision(3);
169  }
170}
171
172// Notifies the runner that we're done running a single benchmark in
173// the benchmark suite. This can be useful to report progress.
174BenchmarkSuite.prototype.NotifyStep = function(result) {
175  this.results.push(result);
176  if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name);
177}
178
179
180// Notifies the runner that we're done with running a suite and that
181// we have a result which can be reported to the user if needed.
182BenchmarkSuite.prototype.NotifyResult = function() {
183  var mean = BenchmarkSuite.GeometricMean(this.results);
184  var score = this.reference / mean;
185  BenchmarkSuite.scores.push(score);
186  if (this.runner.NotifyResult) {
187    var formatted = BenchmarkSuite.FormatScore(100 * score);
188    this.runner.NotifyResult(this.name, formatted);
189  }
190}
191
192
193// Notifies the runner that running a benchmark resulted in an error.
194BenchmarkSuite.prototype.NotifyError = function(error) {
195  if (this.runner.NotifyError) {
196    this.runner.NotifyError(this.name, error);
197  }
198  if (this.runner.NotifyStep) {
199    this.runner.NotifyStep(this.name);
200  }
201}
202
203
204// Runs a single benchmark for at least a second and computes the
205// average time it takes to run a single iteration.
206BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) {
207  function Measure(data) {
208    var elapsed = 0;
209    var start = new Date();
210    for (var n = 0; elapsed < 1000; n++) {
211      benchmark.run();
212      elapsed = new Date() - start;
213    }
214    if (data != null) {
215      data.runs += n;
216      data.elapsed += elapsed;
217    }
218  }
219
220  if (data == null) {
221    // Measure the benchmark once for warm up and throw the result
222    // away. Return a fresh data object.
223    Measure(null);
224    return { runs: 0, elapsed: 0 };
225  } else {
226    Measure(data);
227    // If we've run too few iterations, we continue for another second.
228    if (data.runs < benchmark.minIterations) return data;
229    var usec = (data.elapsed * 1000) / data.runs;
230    this.NotifyStep(new BenchmarkResult(benchmark, usec));
231    return null;
232  }
233}
234
235
236// This function starts running a suite, but stops between each
237// individual benchmark in the suite and returns a continuation
238// function which can be invoked to run the next benchmark. Once the
239// last benchmark has been executed, null is returned.
240BenchmarkSuite.prototype.RunStep = function(runner) {
241  this.results = [];
242  this.runner = runner;
243  var length = this.benchmarks.length;
244  var index = 0;
245  var suite = this;
246  var data;
247
248  // Run the setup, the actual benchmark, and the tear down in three
249  // separate steps to allow the framework to yield between any of the
250  // steps.
251
252  function RunNextSetup() {
253    if (index < length) {
254      try {
255        suite.benchmarks[index].Setup();
256      } catch (e) {
257        suite.NotifyError(e);
258        return null;
259      }
260      return RunNextBenchmark;
261    }
262    suite.NotifyResult();
263    return null;
264  }
265
266  function RunNextBenchmark() {
267    try {
268      data = suite.RunSingleBenchmark(suite.benchmarks[index], data);
269    } catch (e) {
270      suite.NotifyError(e);
271      return null;
272    }
273    // If data is null, we're done with this benchmark.
274    return (data == null) ? RunNextTearDown : RunNextBenchmark();
275  }
276
277  function RunNextTearDown() {
278    try {
279      suite.benchmarks[index++].TearDown();
280    } catch (e) {
281      suite.NotifyError(e);
282      return null;
283    }
284    return RunNextSetup;
285  }
286
287  // Start out running the setup.
288  return RunNextSetup();
289}
290