1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import optparse 6 7from telemetry import decorators 8from telemetry.internal import story_runner 9from telemetry.internal.util import command_line 10from telemetry.page import page_test 11from telemetry.web_perf import timeline_based_measurement 12 13Disabled = decorators.Disabled 14Enabled = decorators.Enabled 15 16 17class InvalidOptionsError(Exception): 18 """Raised for invalid benchmark options.""" 19 pass 20 21 22class BenchmarkMetadata(object): 23 def __init__(self, name, description='', rerun_options=None): 24 self._name = name 25 self._description = description 26 self._rerun_options = rerun_options 27 28 @property 29 def name(self): 30 return self._name 31 32 @property 33 def description(self): 34 return self._description 35 36 @property 37 def rerun_options(self): 38 return self._rerun_options 39 40 def AsDict(self): 41 return { 42 'type': 'telemetry_benchmark', 43 'name': self._name, 44 'description': self._description, 45 'rerun_options': self._rerun_options, 46 } 47 48 49class Benchmark(command_line.Command): 50 """Base class for a Telemetry benchmark. 51 52 A benchmark packages a measurement and a PageSet together. 53 Benchmarks default to using TBM unless you override the value of 54 Benchmark.test, or override the CreatePageTest method. 55 56 New benchmarks should override CreateStorySet. 57 """ 58 options = {} 59 page_set = None 60 test = timeline_based_measurement.TimelineBasedMeasurement 61 62 def __init__(self, max_failures=None): 63 """Creates a new Benchmark. 64 65 Args: 66 max_failures: The number of story run's failures before bailing 67 from executing subsequent page runs. If None, we never bail. 68 """ 69 self._max_failures = max_failures 70 self._has_original_tbm_options = ( 71 self.CreateTimelineBasedMeasurementOptions.__func__ == 72 Benchmark.CreateTimelineBasedMeasurementOptions.__func__) 73 has_original_create_page_test = ( 74 self.CreatePageTest.__func__ == Benchmark.CreatePageTest.__func__) 75 assert self._has_original_tbm_options or has_original_create_page_test, ( 76 'Cannot override both CreatePageTest and ' 77 'CreateTimelineBasedMeasurementOptions.') 78 79 # pylint: disable=unused-argument 80 @classmethod 81 def ShouldDisable(cls, possible_browser): 82 """Override this method to disable a benchmark under specific conditions. 83 84 Supports logic too complex for simple Enabled and Disabled decorators. 85 Decorators are still respected in cases where this function returns False. 86 """ 87 return False 88 89 def Run(self, finder_options): 90 """Do not override this method.""" 91 return story_runner.RunBenchmark(self, finder_options) 92 93 @property 94 def max_failures(self): 95 return self._max_failures 96 97 @classmethod 98 def Name(cls): 99 return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__) 100 101 @classmethod 102 def ShouldTearDownStateAfterEachStoryRun(cls): 103 """Override this method to tear down state after each story run. 104 105 Tearing down all states after each story run, e.g., clearing profiles, 106 stopping the browser, stopping local server, etc. So the browser will not be 107 reused among multiple stories. This is particularly useful to get the 108 startup part of launching the browser in each story. 109 110 This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but 111 not by PageTest based benchmarks. 112 """ 113 return False 114 115 @classmethod 116 def AddCommandLineArgs(cls, parser): 117 group = optparse.OptionGroup(parser, '%s test options' % cls.Name()) 118 cls.AddBenchmarkCommandLineArgs(group) 119 120 if cls.HasTraceRerunDebugOption(): 121 group.add_option( 122 '--rerun-with-debug-trace', 123 action='store_true', 124 help='Rerun option that enables more extensive tracing.') 125 126 if group.option_list: 127 parser.add_option_group(group) 128 129 @classmethod 130 def AddBenchmarkCommandLineArgs(cls, group): 131 del group # unused 132 133 @classmethod 134 def HasTraceRerunDebugOption(cls): 135 return False 136 137 def GetTraceRerunCommands(self): 138 if self.HasTraceRerunDebugOption(): 139 return [['Debug Trace', '--rerun-with-debug-trace']] 140 return [] 141 142 def SetupTraceRerunOptions(self, browser_options, tbm_options): 143 if self.HasTraceRerunDebugOption(): 144 if browser_options.rerun_with_debug_trace: 145 self.SetupBenchmarkDebugTraceRerunOptions(tbm_options) 146 else: 147 self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options) 148 149 def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options): 150 """Setup tracing categories associated with default trace option.""" 151 152 def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options): 153 """Setup tracing categories associated with debug trace option.""" 154 155 @classmethod 156 def SetArgumentDefaults(cls, parser): 157 default_values = parser.get_default_values() 158 invalid_options = [ 159 o for o in cls.options if not hasattr(default_values, o)] 160 if invalid_options: 161 raise InvalidOptionsError('Invalid benchmark options: %s', 162 ', '.join(invalid_options)) 163 parser.set_defaults(**cls.options) 164 165 @classmethod 166 def ProcessCommandLineArgs(cls, parser, args): 167 pass 168 169 # pylint: disable=unused-argument 170 @classmethod 171 def ValueCanBeAddedPredicate(cls, value, is_first_result): 172 """Returns whether |value| can be added to the test results. 173 174 Override this method to customize the logic of adding values to test 175 results. 176 177 Args: 178 value: a value.Value instance (except failure.FailureValue, 179 skip.SkipValue or trace.TraceValue which will always be added). 180 is_first_result: True if |value| is the first result for its 181 corresponding story. 182 183 Returns: 184 True if |value| should be added to the test results. 185 Otherwise, it returns False. 186 """ 187 return True 188 189 def CustomizeBrowserOptions(self, options): 190 """Add browser options that are required by this benchmark.""" 191 192 def GetMetadata(self): 193 return BenchmarkMetadata( 194 self.Name(), self.__doc__, self.GetTraceRerunCommands()) 195 196 def CreateTimelineBasedMeasurementOptions(self): 197 """Return the TimelineBasedMeasurementOptions for this Benchmark. 198 199 Override this method to configure a TimelineBasedMeasurement benchmark. 200 Otherwise, override CreatePageTest for PageTest tests. Do not override 201 both methods. 202 """ 203 return timeline_based_measurement.Options() 204 205 def CreatePageTest(self, options): # pylint: disable=unused-argument 206 """Return the PageTest for this Benchmark. 207 208 Override this method for PageTest tests. 209 Override, override CreateTimelineBasedMeasurementOptions to configure 210 TimelineBasedMeasurement tests. Do not override both methods. 211 212 Args: 213 options: a browser_options.BrowserFinderOptions instance 214 Returns: 215 |test()| if |test| is a PageTest class. 216 Otherwise, a TimelineBasedMeasurement instance. 217 """ 218 is_page_test = issubclass(self.test, page_test.PageTest) 219 is_tbm = self.test == timeline_based_measurement.TimelineBasedMeasurement 220 if not is_page_test and not is_tbm: 221 raise TypeError('"%s" is not a PageTest or a TimelineBasedMeasurement.' % 222 self.test.__name__) 223 if is_page_test: 224 assert self._has_original_tbm_options, ( 225 'Cannot override CreateTimelineBasedMeasurementOptions ' 226 'with a PageTest.') 227 return self.test() # pylint: disable=no-value-for-parameter 228 229 opts = self.CreateTimelineBasedMeasurementOptions() 230 self.SetupTraceRerunOptions(options, opts) 231 return timeline_based_measurement.TimelineBasedMeasurement(opts) 232 233 def CreateStorySet(self, options): 234 """Creates the instance of StorySet used to run the benchmark. 235 236 Can be overridden by subclasses. 237 """ 238 del options # unused 239 # TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with 240 # story_set. 241 if not self.page_set: 242 raise NotImplementedError('This test has no "page_set" attribute.') 243 return self.page_set() # pylint: disable=not-callable 244 245 246def AddCommandLineArgs(parser): 247 story_runner.AddCommandLineArgs(parser) 248 249 250def ProcessCommandLineArgs(parser, args): 251 story_runner.ProcessCommandLineArgs(parser, args) 252