Lines Matching +full:test +full:- +full:results
3 # Use of this source code is governed by a BSD-style license that can be
18 # Requires python-coverage and python-mock. Native python coverage
25 TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf")
31 "flags": ["--flag"],
44 "flags": ["--flag"],
69 "flags": ["--flag2"],
82 "flags": ["--flag"],
120 self._test_input = path.join(TEST_WORKSPACE, "test.json")
125 # Fake output for each test run.
139 # Check that d8 is called from the correct cwd for each test run.
149 self._test_output = path.join(TEST_WORKSPACE, "results.json")
151 "--json-test-results",
166 "results": trace["results"],
196 self._VerifyResults("test", "score", [
197 {"name": "Richards", "results": ["1.234"], "stddev": ""},
198 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
201 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
209 self._VerifyResults("test", "score", [
210 {"name": "Richards", "results": ["1.234"], "stddev": ""},
211 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
214 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js",
215 "--", "2", "test_name")
228 {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
229 {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
232 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
245 self._VerifyResults("test", "score", [
246 {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
247 {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""},
250 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
264 "graphs": ["test", "Richards"],
265 "results": ["50.0", "100.0"],
268 "graphs": ["test", "Sub", "Leaf"],
269 "results": ["3.0", "2.0", "1.0"],
272 "graphs": ["test", "DeltaBlue"],
273 "results": ["200.0"],
278 (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
279 (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
280 (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
281 (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
282 (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
283 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js"))
287 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
289 self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n"
290 "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"])
292 self._VerifyResults("test", "score", [
293 {"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
294 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": "106"},
297 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
301 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
304 self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n"
305 "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n",
306 "Richards: 2\nRichards-stddev: 0.5\n"
307 "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
309 self._VerifyResults("test", "score", [
310 {"name": "Richards", "results": ["2.0", "3.0"], "stddev": "0.7"},
311 {"name": "DeltaBlue", "results": ["5.0", "6.0"], "stddev": "0.8"},
314 ["Test test/Richards should only run once since a stddev is provided "
315 "by the test.",
316 "Test test/DeltaBlue should only run once since a stddev is provided "
317 "by the test.",
318 "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test "
319 "test/DeltaBlue."])
320 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
325 self.assertEquals(0, self._CallMain("--buildbot"))
326 self._VerifyResults("test", "score", [
327 {"name": "Richards", "results": ["1.234"], "stddev": ""},
328 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
331 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
338 self.assertEquals(0, self._CallMain("--buildbot"))
339 self._VerifyResults("test", "score", [
340 {"name": "Richards", "results": ["1.234"], "stddev": ""},
341 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
342 {"name": "Total", "results": ["3626.49109719"], "stddev": ""},
345 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
352 self.assertEquals(1, self._CallMain("--buildbot"))
353 self._VerifyResults("test", "score", [
354 {"name": "Richards", "results": [], "stddev": ""},
355 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
359 "returned a non-numeric for test test/Richards.",
360 "Not all traces have the same number of results."])
361 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
367 self._VerifyResults("test", "score", [
368 {"name": "Richards", "results": [], "stddev": ""},
369 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
372 ["Regexp \"^Richards: (.+)$\" didn't match for test test/Richards."])
373 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
386 "graphs": ["test", "Infra", "Constant1"],
387 "results": ["11.0"],
390 "graphs": ["test", "Infra", "Constant2"],
391 "results": ["10.0", "5.0", "10.0", "15.0"],
394 "graphs": ["test", "Infra", "Constant3"],
395 "results": ["12.0"],
398 "graphs": ["test", "Infra", "Constant4"],
399 "results": [],
402 self._VerifyErrors(["Found non-numeric in test/Infra/Constant4"])
403 self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
411 self._VerifyResults("test", "score", [
412 {"name": "Richards", "results": [], "stddev": ""},
413 {"name": "DeltaBlue", "results": [], "stddev": ""},
416 "Regexp \"^Richards: (.+)$\" didn't match for test test/Richards.",
417 "Regexp \"^DeltaBlue: (.+)$\" didn't match for test test/DeltaBlue.",
420 path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70)
422 # Simple test that mocks out the android platform. Testing the platform would
426 # FIXME(machenbach): This is not test-local!
436 self.assertEquals(0, self._CallMain("--arch", "arm"))
437 self._VerifyResults("test", "score", [
438 {"name": "Richards", "results": ["1.234"], "stddev": ""},
439 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
453 "--outdir-secondary", "out-secondary",
454 "--json-test-results-secondary", test_output_secondary,
456 self._VerifyResults("test", "score", [
457 {"name": "Richards", "results": ["100.0", "200.0"], "stddev": ""},
458 {"name": "DeltaBlue", "results": ["20.0", "20.0"], "stddev": ""},
460 self._VerifyResults("test", "score", [
461 {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""},
462 {"name": "DeltaBlue", "results": ["200.0", "200.0"], "stddev": ""},
466 (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
467 (path.join("out-secondary", "x64.release", "d7"), "--flag", "run.js"),
468 (path.join("out", "x64.release", "d7"), "--flag", "run.js"),
469 (path.join("out-secondary", "x64.release", "d7"), "--flag", "run.js"),
476 self.assertEquals(0, self._CallMain("--extra-flags=--prof"))
477 self._VerifyResults("test", "score", [
478 {"name": "Richards", "results": ["1.234"], "stddev": ""},
479 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""},
483 "--flag", "--prof", "run.js")
500 "--binary-override-path", os.path.join(TEST_DATA, mocked_d8),
501 "--json-test-results", output_json,
508 results = self._RunPerf("d8_mocked1.py", "test1.json")
509 self.assertEquals([], results['errors'])
514 'results': [u'1.2', u'1.2'],
520 'results': [u'2.1', u'2.1'],
523 ], results['traces'])
526 results = self._RunPerf("d8_mocked2.py", "test2.json")
527 self.assertEquals([], results['errors'])
532 'results': [u'1.2', u'1.2'],
538 'results': [u'2.1', u'2.1'],
541 ], results['traces'])
544 results = self._RunPerf("d8_mocked2.py", "test3.json")
545 self.assertEquals([], results['errors'])
550 'results': [u'1.2'],
556 'results': [u'2.1'],
559 ], results['traces'])