1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import optparse
6
7from telemetry import decorators
8from telemetry.internal import story_runner
9from telemetry.internal.util import command_line
10from telemetry.page import page_test
11from telemetry.web_perf import timeline_based_measurement
12
13Disabled = decorators.Disabled
14Enabled = decorators.Enabled
15
16
17class InvalidOptionsError(Exception):
18  """Raised for invalid benchmark options."""
19  pass
20
21
22class BenchmarkMetadata(object):
23  def __init__(self, name, description='', rerun_options=None):
24    self._name = name
25    self._description = description
26    self._rerun_options = rerun_options
27
28  @property
29  def name(self):
30    return self._name
31
32  @property
33  def description(self):
34    return self._description
35
36  @property
37  def rerun_options(self):
38    return self._rerun_options
39
40  def AsDict(self):
41    return {
42      'type': 'telemetry_benchmark',
43      'name': self._name,
44      'description': self._description,
45      'rerun_options': self._rerun_options,
46    }
47
48
49class Benchmark(command_line.Command):
50  """Base class for a Telemetry benchmark.
51
52  A benchmark packages a measurement and a PageSet together.
53  Benchmarks default to using TBM unless you override the value of
54  Benchmark.test, or override the CreatePageTest method.
55
56  New benchmarks should override CreateStorySet.
57  """
58  options = {}
59  page_set = None
60  test = timeline_based_measurement.TimelineBasedMeasurement
61
62  def __init__(self, max_failures=None):
63    """Creates a new Benchmark.
64
65    Args:
66      max_failures: The number of story run's failures before bailing
67          from executing subsequent page runs. If None, we never bail.
68    """
69    self._max_failures = max_failures
70    self._has_original_tbm_options = (
71        self.CreateTimelineBasedMeasurementOptions.__func__ ==
72        Benchmark.CreateTimelineBasedMeasurementOptions.__func__)
73    has_original_create_page_test = (
74        self.CreatePageTest.__func__ == Benchmark.CreatePageTest.__func__)
75    assert self._has_original_tbm_options or has_original_create_page_test, (
76        'Cannot override both CreatePageTest and '
77        'CreateTimelineBasedMeasurementOptions.')
78
79  # pylint: disable=unused-argument
80  @classmethod
81  def ShouldDisable(cls, possible_browser):
82    """Override this method to disable a benchmark under specific conditions.
83
84     Supports logic too complex for simple Enabled and Disabled decorators.
85     Decorators are still respected in cases where this function returns False.
86     """
87    return False
88
89  def Run(self, finder_options):
90    """Do not override this method."""
91    return story_runner.RunBenchmark(self, finder_options)
92
93  @property
94  def max_failures(self):
95    return self._max_failures
96
97  @classmethod
98  def Name(cls):
99    return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
100
101  @classmethod
102  def ShouldTearDownStateAfterEachStoryRun(cls):
103    """Override this method to tear down state after each story run.
104
105    Tearing down all states after each story run, e.g., clearing profiles,
106    stopping the browser, stopping local server, etc. So the browser will not be
107    reused among multiple stories. This is particularly useful to get the
108    startup part of launching the browser in each story.
109
110    This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
111    not by PageTest based benchmarks.
112    """
113    return False
114
115  @classmethod
116  def AddCommandLineArgs(cls, parser):
117    group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
118    cls.AddBenchmarkCommandLineArgs(group)
119
120    if cls.HasTraceRerunDebugOption():
121      group.add_option(
122          '--rerun-with-debug-trace',
123          action='store_true',
124          help='Rerun option that enables more extensive tracing.')
125
126    if group.option_list:
127      parser.add_option_group(group)
128
129  @classmethod
130  def AddBenchmarkCommandLineArgs(cls, group):
131    del group  # unused
132
133  @classmethod
134  def HasTraceRerunDebugOption(cls):
135    return False
136
137  def GetTraceRerunCommands(self):
138    if self.HasTraceRerunDebugOption():
139      return [['Debug Trace', '--rerun-with-debug-trace']]
140    return []
141
142  def SetupTraceRerunOptions(self, browser_options, tbm_options):
143    if self.HasTraceRerunDebugOption():
144      if browser_options.rerun_with_debug_trace:
145        self.SetupBenchmarkDebugTraceRerunOptions(tbm_options)
146      else:
147        self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options)
148
149  def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
150    """Setup tracing categories associated with default trace option."""
151
152  def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
153    """Setup tracing categories associated with debug trace option."""
154
155  @classmethod
156  def SetArgumentDefaults(cls, parser):
157    default_values = parser.get_default_values()
158    invalid_options = [
159        o for o in cls.options if not hasattr(default_values, o)]
160    if invalid_options:
161      raise InvalidOptionsError('Invalid benchmark options: %s',
162                                ', '.join(invalid_options))
163    parser.set_defaults(**cls.options)
164
165  @classmethod
166  def ProcessCommandLineArgs(cls, parser, args):
167    pass
168
169  # pylint: disable=unused-argument
170  @classmethod
171  def ValueCanBeAddedPredicate(cls, value, is_first_result):
172    """Returns whether |value| can be added to the test results.
173
174    Override this method to customize the logic of adding values to test
175    results.
176
177    Args:
178      value: a value.Value instance (except failure.FailureValue,
179        skip.SkipValue or trace.TraceValue which will always be added).
180      is_first_result: True if |value| is the first result for its
181          corresponding story.
182
183    Returns:
184      True if |value| should be added to the test results.
185      Otherwise, it returns False.
186    """
187    return True
188
189  def CustomizeBrowserOptions(self, options):
190    """Add browser options that are required by this benchmark."""
191
192  def GetMetadata(self):
193    return BenchmarkMetadata(
194        self.Name(), self.__doc__, self.GetTraceRerunCommands())
195
196  def CreateTimelineBasedMeasurementOptions(self):
197    """Return the TimelineBasedMeasurementOptions for this Benchmark.
198
199    Override this method to configure a TimelineBasedMeasurement benchmark.
200    Otherwise, override CreatePageTest for PageTest tests. Do not override
201    both methods.
202    """
203    return timeline_based_measurement.Options()
204
205  def CreatePageTest(self, options):  # pylint: disable=unused-argument
206    """Return the PageTest for this Benchmark.
207
208    Override this method for PageTest tests.
209    Override, override CreateTimelineBasedMeasurementOptions to configure
210    TimelineBasedMeasurement tests. Do not override both methods.
211
212    Args:
213      options: a browser_options.BrowserFinderOptions instance
214    Returns:
215      |test()| if |test| is a PageTest class.
216      Otherwise, a TimelineBasedMeasurement instance.
217    """
218    is_page_test = issubclass(self.test, page_test.PageTest)
219    is_tbm = self.test == timeline_based_measurement.TimelineBasedMeasurement
220    if not is_page_test and not is_tbm:
221      raise TypeError('"%s" is not a PageTest or a TimelineBasedMeasurement.' %
222                      self.test.__name__)
223    if is_page_test:
224      assert self._has_original_tbm_options, (
225          'Cannot override CreateTimelineBasedMeasurementOptions '
226          'with a PageTest.')
227      return self.test()  # pylint: disable=no-value-for-parameter
228
229    opts = self.CreateTimelineBasedMeasurementOptions()
230    self.SetupTraceRerunOptions(options, opts)
231    return timeline_based_measurement.TimelineBasedMeasurement(opts)
232
233  def CreateStorySet(self, options):
234    """Creates the instance of StorySet used to run the benchmark.
235
236    Can be overridden by subclasses.
237    """
238    del options  # unused
239    # TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with
240    # story_set.
241    if not self.page_set:
242      raise NotImplementedError('This test has no "page_set" attribute.')
243    return self.page_set()  # pylint: disable=not-callable
244
245
246def AddCommandLineArgs(parser):
247  story_runner.AddCommandLineArgs(parser)
248
249
250def ProcessCommandLineArgs(parser, args):
251  story_runner.ProcessCommandLineArgs(parser, args)
252