1#!/usr/bin/env python
2# Copyright 2014 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from collections import namedtuple
7import coverage
8import json
9from mock import DEFAULT
10from mock import MagicMock
11import os
12from os import path, sys
13import platform
14import shutil
15import subprocess
16import tempfile
17import unittest
18
19# Requires python-coverage and python-mock. Native python coverage
20# version >= 3.7.1 should be installed to get the best speed.
21
22TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf")
23
24V8_JSON = {
25  "path": ["."],
26  "binary": "d7",
27  "flags": ["--flag"],
28  "main": "run.js",
29  "run_count": 1,
30  "results_regexp": "^%s: (.+)$",
31  "tests": [
32    {"name": "Richards"},
33    {"name": "DeltaBlue"},
34  ]
35}
36
37V8_NESTED_SUITES_JSON = {
38  "path": ["."],
39  "flags": ["--flag"],
40  "run_count": 1,
41  "units": "score",
42  "tests": [
43    {"name": "Richards",
44     "path": ["richards"],
45     "binary": "d7",
46     "main": "run.js",
47     "resources": ["file1.js", "file2.js"],
48     "run_count": 2,
49     "results_regexp": "^Richards: (.+)$"},
50    {"name": "Sub",
51     "path": ["sub"],
52     "tests": [
53       {"name": "Leaf",
54        "path": ["leaf"],
55        "run_count_x64": 3,
56        "units": "ms",
57        "main": "run.js",
58        "results_regexp": "^Simple: (.+) ms.$"},
59     ]
60    },
61    {"name": "DeltaBlue",
62     "path": ["delta_blue"],
63     "main": "run.js",
64     "flags": ["--flag2"],
65     "results_regexp": "^DeltaBlue: (.+)$"},
66    {"name": "ShouldntRun",
67     "path": ["."],
68     "archs": ["arm"],
69     "main": "run.js"},
70  ]
71}
72
73V8_GENERIC_JSON = {
74  "path": ["."],
75  "binary": "cc",
76  "flags": ["--flag"],
77  "generic": True,
78  "run_count": 1,
79  "units": "ms",
80}
81
82Output = namedtuple("Output", "stdout, stderr, timed_out")
83
84class PerfTest(unittest.TestCase):
85  @classmethod
86  def setUpClass(cls):
87    cls.base = path.dirname(path.dirname(path.abspath(__file__)))
88    sys.path.append(cls.base)
89    cls._cov = coverage.coverage(
90        include=([os.path.join(cls.base, "run_perf.py")]))
91    cls._cov.start()
92    import run_perf
93    from testrunner.local import commands
94    global commands
95    global run_perf
96
97  @classmethod
98  def tearDownClass(cls):
99    cls._cov.stop()
100    print ""
101    print cls._cov.report()
102
103  def setUp(self):
104    self.maxDiff = None
105    if path.exists(TEST_WORKSPACE):
106      shutil.rmtree(TEST_WORKSPACE)
107    os.makedirs(TEST_WORKSPACE)
108
109  def tearDown(self):
110    if path.exists(TEST_WORKSPACE):
111      shutil.rmtree(TEST_WORKSPACE)
112
113  def _WriteTestInput(self, json_content):
114    self._test_input = path.join(TEST_WORKSPACE, "test.json")
115    with open(self._test_input, "w") as f:
116      f.write(json.dumps(json_content))
117
118  def _MockCommand(self, *args, **kwargs):
119    # Fake output for each test run.
120    test_outputs = [Output(stdout=arg,
121                           stderr=None,
122                           timed_out=kwargs.get("timed_out", False))
123                    for arg in args[1]]
124    def execute(*args, **kwargs):
125      return test_outputs.pop()
126    commands.Execute = MagicMock(side_effect=execute)
127
128    # Check that d8 is called from the correct cwd for each test run.
129    dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]]
130    def chdir(*args, **kwargs):
131      self.assertEquals(dirs.pop(), args[0])
132    os.chdir = MagicMock(side_effect=chdir)
133
134    subprocess.check_call = MagicMock()
135    platform.system = MagicMock(return_value='Linux')
136
137  def _CallMain(self, *args):
138    self._test_output = path.join(TEST_WORKSPACE, "results.json")
139    all_args=[
140      "--json-test-results",
141      self._test_output,
142      self._test_input,
143    ]
144    all_args += args
145    return run_perf.Main(all_args)
146
147  def _LoadResults(self, file_name=None):
148    with open(file_name or self._test_output) as f:
149      return json.load(f)
150
151  def _VerifyResults(self, suite, units, traces, file_name=None):
152    self.assertEquals([
153      {"units": units,
154       "graphs": [suite, trace["name"]],
155       "results": trace["results"],
156       "stddev": trace["stddev"]} for trace in traces],
157      self._LoadResults(file_name)["traces"])
158
159  def _VerifyErrors(self, errors):
160    self.assertEquals(errors, self._LoadResults()["errors"])
161
162  def _VerifyMock(self, binary, *args, **kwargs):
163    arg = [path.join(path.dirname(self.base), binary)]
164    arg += args
165    commands.Execute.assert_called_with(
166        arg, timeout=kwargs.get("timeout", 60))
167
168  def _VerifyMockMultiple(self, *args, **kwargs):
169    expected = []
170    for arg in args:
171      a = [path.join(path.dirname(self.base), arg[0])]
172      a += arg[1:]
173      expected.append(((a,), {"timeout": kwargs.get("timeout", 60)}))
174    self.assertEquals(expected, commands.Execute.call_args_list)
175
176  def testOneRun(self):
177    self._WriteTestInput(V8_JSON)
178    self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"])
179    self.assertEquals(0, self._CallMain())
180    self._VerifyResults("test", "score", [
181      {"name": "Richards", "results": ["1.234"], "stddev": ""},
182      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
183    ])
184    self._VerifyErrors([])
185    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
186
187  def testOneRunWithTestFlags(self):
188    test_input = dict(V8_JSON)
189    test_input["test_flags"] = ["2", "test_name"]
190    self._WriteTestInput(test_input)
191    self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567"])
192    self.assertEquals(0, self._CallMain())
193    self._VerifyResults("test", "score", [
194      {"name": "Richards", "results": ["1.234"], "stddev": ""},
195      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
196    ])
197    self._VerifyErrors([])
198    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js",
199                     "--", "2", "test_name")
200
201  def testTwoRuns_Units_SuiteName(self):
202    test_input = dict(V8_JSON)
203    test_input["run_count"] = 2
204    test_input["name"] = "v8"
205    test_input["units"] = "ms"
206    self._WriteTestInput(test_input)
207    self._MockCommand([".", "."],
208                      ["Richards: 100\nDeltaBlue: 200\n",
209                       "Richards: 50\nDeltaBlue: 300\n"])
210    self.assertEquals(0, self._CallMain())
211    self._VerifyResults("v8", "ms", [
212      {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
213      {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
214    ])
215    self._VerifyErrors([])
216    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
217
218  def testTwoRuns_SubRegexp(self):
219    test_input = dict(V8_JSON)
220    test_input["run_count"] = 2
221    del test_input["results_regexp"]
222    test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$"
223    test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$"
224    self._WriteTestInput(test_input)
225    self._MockCommand([".", "."],
226                      ["Richards: 100\nDeltaBlue: 200\n",
227                       "Richards: 50\nDeltaBlue: 300\n"])
228    self.assertEquals(0, self._CallMain())
229    self._VerifyResults("test", "score", [
230      {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
231      {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
232    ])
233    self._VerifyErrors([])
234    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
235
236  def testNestedSuite(self):
237    self._WriteTestInput(V8_NESTED_SUITES_JSON)
238    self._MockCommand(["delta_blue", "sub/leaf", "richards"],
239                      ["DeltaBlue: 200\n",
240                       "Simple: 1 ms.\n",
241                       "Simple: 2 ms.\n",
242                       "Simple: 3 ms.\n",
243                       "Richards: 100\n",
244                       "Richards: 50\n"])
245    self.assertEquals(0, self._CallMain())
246    self.assertEquals([
247      {"units": "score",
248       "graphs": ["test", "Richards"],
249       "results": ["50.0", "100.0"],
250       "stddev": ""},
251      {"units": "ms",
252       "graphs": ["test", "Sub", "Leaf"],
253       "results": ["3.0", "2.0", "1.0"],
254       "stddev": ""},
255      {"units": "score",
256       "graphs": ["test", "DeltaBlue"],
257       "results": ["200.0"],
258       "stddev": ""},
259      ], self._LoadResults()["traces"])
260    self._VerifyErrors([])
261    self._VerifyMockMultiple(
262        (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
263        (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
264        (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
265        (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
266        (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
267        (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js"))
268
269  def testOneRunStdDevRegExp(self):
270    test_input = dict(V8_JSON)
271    test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
272    self._WriteTestInput(test_input)
273    self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n"
274                              "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"])
275    self.assertEquals(0, self._CallMain())
276    self._VerifyResults("test", "score", [
277      {"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
278      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": "106"},
279    ])
280    self._VerifyErrors([])
281    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
282
283  def testTwoRunsStdDevRegExp(self):
284    test_input = dict(V8_JSON)
285    test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
286    test_input["run_count"] = 2
287    self._WriteTestInput(test_input)
288    self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n"
289                              "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n",
290                              "Richards: 2\nRichards-stddev: 0.5\n"
291                              "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
292    self.assertEquals(1, self._CallMain())
293    self._VerifyResults("test", "score", [
294      {"name": "Richards", "results": ["2.0", "3.0"], "stddev": "0.7"},
295      {"name": "DeltaBlue", "results": ["5.0", "6.0"], "stddev": "0.8"},
296    ])
297    self._VerifyErrors(
298        ["Test Richards should only run once since a stddev is provided "
299         "by the test.",
300         "Test DeltaBlue should only run once since a stddev is provided "
301         "by the test.",
302         "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test "
303         "DeltaBlue."])
304    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
305
306  def testBuildbot(self):
307    self._WriteTestInput(V8_JSON)
308    self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
309    self.assertEquals(0, self._CallMain("--buildbot"))
310    self._VerifyResults("test", "score", [
311      {"name": "Richards", "results": ["1.234"], "stddev": ""},
312      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
313    ])
314    self._VerifyErrors([])
315    self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
316
317  def testBuildbotWithTotal(self):
318    test_input = dict(V8_JSON)
319    test_input["total"] = True
320    self._WriteTestInput(test_input)
321    self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
322    self.assertEquals(0, self._CallMain("--buildbot"))
323    self._VerifyResults("test", "score", [
324      {"name": "Richards", "results": ["1.234"], "stddev": ""},
325      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
326      {"name": "Total", "results": ["3626.49109719"], "stddev": ""},
327    ])
328    self._VerifyErrors([])
329    self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
330
331  def testBuildbotWithTotalAndErrors(self):
332    test_input = dict(V8_JSON)
333    test_input["total"] = True
334    self._WriteTestInput(test_input)
335    self._MockCommand(["."], ["x\nRichards: bla\nDeltaBlue: 10657567\ny\n"])
336    self.assertEquals(1, self._CallMain("--buildbot"))
337    self._VerifyResults("test", "score", [
338      {"name": "Richards", "results": [], "stddev": ""},
339      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
340    ])
341    self._VerifyErrors(
342        ["Regexp \"^Richards: (.+)$\" "
343         "returned a non-numeric for test Richards.",
344         "Not all traces have the same number of results."])
345    self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
346
347  def testRegexpNoMatch(self):
348    self._WriteTestInput(V8_JSON)
349    self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
350    self.assertEquals(1, self._CallMain())
351    self._VerifyResults("test", "score", [
352      {"name": "Richards", "results": [], "stddev": ""},
353      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
354    ])
355    self._VerifyErrors(
356        ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."])
357    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
358
359  def testOneRunGeneric(self):
360    test_input = dict(V8_GENERIC_JSON)
361    self._WriteTestInput(test_input)
362    self._MockCommand(["."], [
363      "RESULT Infra: Constant1= 11 count\n"
364      "RESULT Infra: Constant2= [10,5,10,15] count\n"
365      "RESULT Infra: Constant3= {12,1.2} count\n"
366      "RESULT Infra: Constant4= [10,5,error,15] count\n"])
367    self.assertEquals(1, self._CallMain())
368    self.assertEquals([
369      {"units": "count",
370       "graphs": ["test", "Infra", "Constant1"],
371       "results": ["11.0"],
372       "stddev": ""},
373      {"units": "count",
374       "graphs": ["test", "Infra", "Constant2"],
375       "results": ["10.0", "5.0", "10.0", "15.0"],
376       "stddev": ""},
377      {"units": "count",
378       "graphs": ["test", "Infra", "Constant3"],
379       "results": ["12.0"],
380       "stddev": "1.2"},
381      {"units": "count",
382       "graphs": ["test", "Infra", "Constant4"],
383       "results": [],
384       "stddev": ""},
385      ], self._LoadResults()["traces"])
386    self._VerifyErrors(["Found non-numeric in test/Infra/Constant4"])
387    self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
388
389  def testOneRunTimingOut(self):
390    test_input = dict(V8_JSON)
391    test_input["timeout"] = 70
392    self._WriteTestInput(test_input)
393    self._MockCommand(["."], [""], timed_out=True)
394    self.assertEquals(1, self._CallMain())
395    self._VerifyResults("test", "score", [
396      {"name": "Richards", "results": [], "stddev": ""},
397      {"name": "DeltaBlue", "results": [], "stddev": ""},
398    ])
399    self._VerifyErrors([
400      "Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
401      "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.",
402    ])
403    self._VerifyMock(
404        path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70)
405
406  # Simple test that mocks out the android platform. Testing the platform would
407  # require lots of complicated mocks for the android tools.
408  def testAndroid(self):
409    self._WriteTestInput(V8_JSON)
410    # FIXME(machenbach): This is not test-local!
411    platform = run_perf.AndroidPlatform
412    platform.PreExecution = MagicMock(return_value=None)
413    platform.PostExecution = MagicMock(return_value=None)
414    platform.PreTests = MagicMock(return_value=None)
415    platform.Run = MagicMock(
416        return_value=("Richards: 1.234\nDeltaBlue: 10657567\n", None))
417    run_perf.AndroidPlatform = MagicMock(return_value=platform)
418    self.assertEquals(
419        0, self._CallMain("--android-build-tools", "/some/dir",
420                          "--arch", "arm"))
421    self._VerifyResults("test", "score", [
422      {"name": "Richards", "results": ["1.234"], "stddev": ""},
423      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
424    ])
425
426  def testTwoRuns_Trybot(self):
427    test_input = dict(V8_JSON)
428    test_input["run_count"] = 2
429    self._WriteTestInput(test_input)
430    self._MockCommand([".", ".", ".", "."],
431                      ["Richards: 100\nDeltaBlue: 200\n",
432                       "Richards: 200\nDeltaBlue: 20\n",
433                       "Richards: 50\nDeltaBlue: 200\n",
434                       "Richards: 100\nDeltaBlue: 20\n"])
435    test_output_no_patch = path.join(TEST_WORKSPACE, "results_no_patch.json")
436    self.assertEquals(0, self._CallMain(
437        "--outdir-no-patch", "out-no-patch",
438        "--json-test-results-no-patch", test_output_no_patch,
439    ))
440    self._VerifyResults("test", "score", [
441      {"name": "Richards", "results": ["100.0", "200.0"], "stddev": ""},
442      {"name": "DeltaBlue", "results": ["20.0", "20.0"], "stddev": ""},
443    ])
444    self._VerifyResults("test", "score", [
445      {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
446      {"name": "DeltaBlue", "results": ["200.0", "200.0"], "stddev": ""},
447    ], test_output_no_patch)
448    self._VerifyErrors([])
449    self._VerifyMockMultiple(
450        (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
451        (path.join("out-no-patch", "x64.release", "d7"), "--flag", "run.js"),
452        (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
453        (path.join("out-no-patch", "x64.release", "d7"), "--flag", "run.js"),
454    )
455
456  def testWrongBinaryWithProf(self):
457    test_input = dict(V8_JSON)
458    self._WriteTestInput(test_input)
459    self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"])
460    self.assertEquals(0, self._CallMain("--extra-flags=--prof"))
461    self._VerifyResults("test", "score", [
462      {"name": "Richards", "results": ["1.234"], "stddev": ""},
463      {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
464    ])
465    self._VerifyErrors([])
466    self._VerifyMock(path.join("out", "x64.release", "d7"),
467                     "--flag", "--prof", "run.js")
468
469  def testUnzip(self):
470    def Gen():
471      for i in [1, 2, 3]:
472        yield i, i + 1
473    l, r = run_perf.Unzip(Gen())
474    self.assertEquals([1, 2, 3], list(l()))
475    self.assertEquals([2, 3, 4], list(r()))
476