Lines Matching full:suite
11 The suite json format is expected to be:
14 "name": <optional suite name, file name is default>,
15 "archs": [<architecture name for which this suite is run>, ...],
19 "run_count": <how often will this suite run (optional)>,
20 "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
36 The tests field can also nest other suites in arbitrary depth. A suite
37 with a "main" file is a leaf suite that can contain one more level of
40 A suite's results_regexp is expected to have one string place holder
41 "%s" for the trace name. A trace's results_regexp overwrites suite
44 A suite's results_processor may point to an optional python script. If
46 relatve to the suite level's path):
47 <results_processor file> <same flags as for d8> <suite level name> <output>
52 A suite without "tests" is considered a performance test itself.
54 Full example (suite with one runner):
73 Full example (suite with several runners):
93 Path pieces are concatenated. D8 is always run with the suite's path as cwd.
295 suite_units: Measurement default units as defined by the benchmark suite.
343 """Represents a node in the suite tree structure."""
370 """Represents a suite definition.
374 def __init__(self, suite, parent, arch): argument
376 self._suite = suite
378 assert isinstance(suite.get("path", []), list)
379 assert isinstance(suite["name"], basestring)
380 assert isinstance(suite.get("flags", []), list)
381 assert isinstance(suite.get("test_flags", []), list)
382 assert isinstance(suite.get("resources", []), list)
385 self.path = parent.path[:] + suite.get("path", [])
386 self.graphs = parent.graphs[:] + [suite["name"]]
387 self.flags = parent.flags[:] + suite.get("flags", [])
388 self.test_flags = parent.test_flags[:] + suite.get("test_flags", [])
391 self.resources = suite.get("resources", [])
394 self.binary = suite.get("binary", parent.binary)
395 self.run_count = suite.get("run_count", parent.run_count)
396 self.run_count = suite.get("run_count_%s" % arch, self.run_count)
397 self.timeout = suite.get("timeout", parent.timeout)
398 self.timeout = suite.get("timeout_%s" % arch, self.timeout)
399 self.units = suite.get("units", parent.units)
400 self.total = suite.get("total", parent.total)
403 # regexp and the current suite has none, a string place holder for the
404 # suite name is expected.
408 regexp_default = parent.results_regexp % re.escape(suite["name"])
411 self.results_regexp = suite.get("results_regexp", regexp_default)
415 stddev_default = parent.stddev_regexp % re.escape(suite["name"])
418 self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
422 """Represents a leaf in the suite tree structure."""
423 def __init__(self, suite, parent, arch): argument
424 super(TraceConfig, self).__init__(suite, parent, arch)
441 """Represents a runnable suite definition (i.e. has a main file).
450 The tests are supposed to be relative to the suite configuration.
492 """Represents a runnable suite definition that is a leaf."""
493 def __init__(self, suite, parent, arch): argument
494 super(RunnableTraceConfig, self).__init__(suite, parent, arch)
510 """Represents a runnable suite definition with generic traces."""
511 def __init__(self, suite, parent, arch): argument
512 super(RunnableGenericConfig, self).__init__(suite, parent, arch)
522 def MakeGraphConfig(suite, arch, parent): argument
526 return TraceConfig(suite, parent, arch)
527 elif suite.get("main") is not None:
529 if suite.get("tests"):
531 return RunnableConfig(suite, parent, arch)
534 return RunnableTraceConfig(suite, parent, arch)
535 elif suite.get("generic"):
536 # This is a generic suite definition. It is either a runnable executable
538 return RunnableGenericConfig(suite, parent, arch)
539 elif suite.get("tests"):
541 return GraphConfig(suite, parent, arch)
543 raise Exception("Invalid suite configuration.")
546 def BuildGraphConfigs(suite, arch, parent=None): argument
547 """Builds a tree structure of graph objects that corresponds to the suite
553 if arch not in suite.get("archs", SUPPORTED_ARCHS):
556 graph = MakeGraphConfig(suite, arch, parent)
557 for subsuite in suite.get("tests", []):
575 raise Exception("Invalid suite configuration.")
866 suite = json.loads(f.read())
869 suite.setdefault("name", os.path.splitext(os.path.basename(path))[0])
871 # Setup things common to one test suite.
875 root = BuildGraphConfigs(suite, options.arch)
883 print ">>> Running suite: %s" % "/".join(runnable.graphs)