1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import collections
5import logging
6from collections import defaultdict
7
8from tracing.metrics import metric_runner
9
10from telemetry.timeline import model as model_module
11from telemetry.timeline import tracing_category_filter
12from telemetry.timeline import tracing_config
13from telemetry.value import trace
14from telemetry.value import translate_common_values
15from telemetry.web_perf.metrics import timeline_based_metric
16from telemetry.web_perf.metrics import blob_timeline
17from telemetry.web_perf.metrics import jitter_timeline
18from telemetry.web_perf.metrics import webrtc_rendering_timeline
19from telemetry.web_perf.metrics import gpu_timeline
20from telemetry.web_perf.metrics import indexeddb_timeline
21from telemetry.web_perf.metrics import layout
22from telemetry.web_perf.metrics import memory_timeline
23from telemetry.web_perf.metrics import responsiveness_metric
24from telemetry.web_perf.metrics import smoothness
25from telemetry.web_perf.metrics import text_selection
26from telemetry.web_perf import smooth_gesture_util
27from telemetry.web_perf import story_test
28from telemetry.web_perf import timeline_interaction_record as tir_module
29
30# TimelineBasedMeasurement considers all instrumentation as producing a single
31# timeline. But, depending on the amount of instrumentation that is enabled,
32# overhead increases. The user of the measurement must therefore chose between
33# a few levels of instrumentation.
34NO_OVERHEAD_LEVEL = 'no-overhead'
35MINIMAL_OVERHEAD_LEVEL = 'minimal-overhead'
36DEBUG_OVERHEAD_LEVEL = 'debug-overhead'
37
38ALL_OVERHEAD_LEVELS = [
39  NO_OVERHEAD_LEVEL,
40  MINIMAL_OVERHEAD_LEVEL,
41  DEBUG_OVERHEAD_LEVEL
42]
43
44
45def _GetAllLegacyTimelineBasedMetrics():
46  # TODO(nednguyen): use discovery pattern to return all the instances of
47  # all TimelineBasedMetrics class in web_perf/metrics/ folder.
48  # This cannot be done until crbug.com/460208 is fixed.
49  return (smoothness.SmoothnessMetric(),
50          responsiveness_metric.ResponsivenessMetric(),
51          layout.LayoutMetric(),
52          gpu_timeline.GPUTimelineMetric(),
53          blob_timeline.BlobTimelineMetric(),
54          jitter_timeline.JitterTimelineMetric(),
55          memory_timeline.MemoryTimelineMetric(),
56          text_selection.TextSelectionMetric(),
57          indexeddb_timeline.IndexedDBTimelineMetric(),
58          webrtc_rendering_timeline.WebRtcRenderingTimelineMetric())
59
60
61class InvalidInteractions(Exception):
62  pass
63
64
65# TODO(nednguyen): Get rid of this results wrapper hack after we add interaction
66# record to telemetry value system (crbug.com/453109)
67class ResultsWrapperInterface(object):
68  def __init__(self):
69    self._tir_label = None
70    self._results = None
71
72  def SetResults(self, results):
73    self._results = results
74
75  def SetTirLabel(self, tir_label):
76    self._tir_label = tir_label
77
78  @property
79  def current_page(self):
80    return self._results.current_page
81
82  def AddValue(self, value):
83    raise NotImplementedError
84
85
86class _TBMResultWrapper(ResultsWrapperInterface):
87  def AddValue(self, value):
88    assert self._tir_label
89    if value.tir_label:
90      assert value.tir_label == self._tir_label
91    else:
92      logging.warning(
93          'TimelineBasedMetric should create the interaction record label '
94          'for %r values.' % value.name)
95      value.tir_label = self._tir_label
96    self._results.AddValue(value)
97
98
99def _GetRendererThreadsToInteractionRecordsMap(model):
100  threads_to_records_map = defaultdict(list)
101  interaction_labels_of_previous_threads = set()
102  for curr_thread in model.GetAllThreads():
103    for event in curr_thread.async_slices:
104      # TODO(nduca): Add support for page-load interaction record.
105      if tir_module.IsTimelineInteractionRecord(event.name):
106        interaction = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
107        # Adjust the interaction record to match the synthetic gesture
108        # controller if needed.
109        interaction = (
110            smooth_gesture_util.GetAdjustedInteractionIfContainGesture(
111                model, interaction))
112        threads_to_records_map[curr_thread].append(interaction)
113        if interaction.label in interaction_labels_of_previous_threads:
114          raise InvalidInteractions(
115            'Interaction record label %s is duplicated on different '
116            'threads' % interaction.label)
117    if curr_thread in threads_to_records_map:
118      interaction_labels_of_previous_threads.update(
119        r.label for r in threads_to_records_map[curr_thread])
120
121  return threads_to_records_map
122
123
124class _TimelineBasedMetrics(object):
125  def __init__(self, model, renderer_thread, interaction_records,
126               results_wrapper, metrics):
127    self._model = model
128    self._renderer_thread = renderer_thread
129    self._interaction_records = interaction_records
130    self._results_wrapper = results_wrapper
131    self._all_metrics = metrics
132
133  def AddResults(self, results):
134    interactions_by_label = defaultdict(list)
135    for i in self._interaction_records:
136      interactions_by_label[i.label].append(i)
137
138    for label, interactions in interactions_by_label.iteritems():
139      are_repeatable = [i.repeatable for i in interactions]
140      if not all(are_repeatable) and len(interactions) > 1:
141        raise InvalidInteractions('Duplicate unrepeatable interaction records '
142                                  'on the page')
143      self._results_wrapper.SetResults(results)
144      self._results_wrapper.SetTirLabel(label)
145      self.UpdateResultsByMetric(interactions, self._results_wrapper)
146
147  def UpdateResultsByMetric(self, interactions, wrapped_results):
148    if not interactions:
149      return
150
151    for metric in self._all_metrics:
152      metric.AddResults(self._model, self._renderer_thread,
153                        interactions, wrapped_results)
154
155
156class Options(object):
157  """A class to be used to configure TimelineBasedMeasurement.
158
159  This is created and returned by
160  Benchmark.CreateTimelineBasedMeasurementOptions.
161
162  By default, all the timeline based metrics in telemetry/web_perf/metrics are
163  used (see _GetAllLegacyTimelineBasedMetrics above).
164  To customize your metric needs, use SetTimelineBasedMetric().
165  """
166
167  def __init__(self, overhead_level=NO_OVERHEAD_LEVEL):
168    """As the amount of instrumentation increases, so does the overhead.
169    The user of the measurement chooses the overhead level that is appropriate,
170    and the tracing is filtered accordingly.
171
172    overhead_level: Can either be a custom TracingCategoryFilter object or
173        one of NO_OVERHEAD_LEVEL, MINIMAL_OVERHEAD_LEVEL or
174        DEBUG_OVERHEAD_LEVEL.
175    """
176    self._config = tracing_config.TracingConfig()
177    self._config.enable_chrome_trace = True
178    self._config.enable_platform_display_trace = True
179
180    if isinstance(overhead_level,
181                  tracing_category_filter.TracingCategoryFilter):
182      self._config.SetTracingCategoryFilter(overhead_level)
183    elif overhead_level in ALL_OVERHEAD_LEVELS:
184      if overhead_level == NO_OVERHEAD_LEVEL:
185        self._config.SetNoOverheadFilter()
186      elif overhead_level == MINIMAL_OVERHEAD_LEVEL:
187        self._config.SetMinimalOverheadFilter()
188      else:
189        self._config.SetDebugOverheadFilter()
190    else:
191      raise Exception("Overhead level must be a TracingCategoryFilter object"
192                      " or valid overhead level string."
193                      " Given overhead level: %s" % overhead_level)
194
195    self._timeline_based_metric = None
196    self._legacy_timeline_based_metrics = _GetAllLegacyTimelineBasedMetrics()
197
198
199  def ExtendTraceCategoryFilter(self, filters):
200    for new_category_filter in filters:
201      self._config.tracing_category_filter.AddIncludedCategory(
202          new_category_filter)
203
204  @property
205  def category_filter(self):
206    return self._config.tracing_category_filter
207
208  @property
209  def config(self):
210    return self._config
211
212  def SetTimelineBasedMetric(self, metric):
213    """Sets the new-style (TBMv2) metric to run.
214
215    Metrics are assumed to live in //tracing/tracing/metrics, so the path
216    should be relative to that. For example, to specify sample_metric.html,
217    you would pass 'sample_metric.html'.
218
219    Args:
220      metric: A string metric path under //tracing/tracing/metrics.
221    """
222    assert isinstance(metric, basestring)
223    self._legacy_timeline_based_metrics = None
224    self._timeline_based_metric = metric
225
226  def GetTimelineBasedMetric(self):
227    return self._timeline_based_metric
228
229  def SetLegacyTimelineBasedMetrics(self, metrics):
230    assert self._timeline_based_metric == None
231    assert isinstance(metrics, collections.Iterable)
232    for m in metrics:
233      assert isinstance(m, timeline_based_metric.TimelineBasedMetric)
234    self._legacy_timeline_based_metrics = metrics
235
236  def GetLegacyTimelineBasedMetrics(self):
237    return self._legacy_timeline_based_metrics
238
239
240class TimelineBasedMeasurement(story_test.StoryTest):
241  """Collects multiple metrics based on their interaction records.
242
243  A timeline based measurement shifts the burden of what metrics to collect onto
244  the story under test. Instead of the measurement
245  having a fixed set of values it collects, the story being tested
246  issues (via javascript) an Interaction record into the user timing API that
247  describing what is happening at that time, as well as a standardized set
248  of flags describing the semantics of the work being done. The
249  TimelineBasedMeasurement object collects a trace that includes both these
250  interaction records, and a user-chosen amount of performance data using
251  Telemetry's various timeline-producing APIs, tracing especially.
252
253  It then passes the recorded timeline to different TimelineBasedMetrics based
254  on those flags. As an example, this allows a single story run to produce
255  load timing data, smoothness data, critical jank information and overall cpu
256  usage information.
257
258  For information on how to mark up a page to work with
259  TimelineBasedMeasurement, refer to the
260  perf.metrics.timeline_interaction_record module.
261
262  Args:
263      options: an instance of timeline_based_measurement.Options.
264      results_wrapper: A class that has the __init__ method takes in
265        the page_test_results object and the interaction record label. This
266        class follows the ResultsWrapperInterface. Note: this class is not
267        supported long term and to be removed when crbug.com/453109 is resolved.
268  """
269  def __init__(self, options, results_wrapper=None):
270    self._tbm_options = options
271    self._results_wrapper = results_wrapper or _TBMResultWrapper()
272
273  def WillRunStory(self, platform):
274    """Configure and start tracing."""
275    if not platform.tracing_controller.IsChromeTracingSupported():
276      raise Exception('Not supported')
277    platform.tracing_controller.StartTracing(self._tbm_options.config)
278
279  def Measure(self, platform, results):
280    """Collect all possible metrics and added them to results."""
281    trace_result = platform.tracing_controller.StopTracing()
282    trace_value = trace.TraceValue(results.current_page, trace_result)
283    results.AddValue(trace_value)
284
285    if self._tbm_options.GetTimelineBasedMetric():
286      self._ComputeTimelineBasedMetric(results, trace_value)
287    else:
288      assert self._tbm_options.GetLegacyTimelineBasedMetrics()
289      self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
290
291
292  def DidRunStory(self, platform):
293    """Clean up after running the story."""
294    if platform.tracing_controller.is_tracing_running:
295      platform.tracing_controller.StopTracing()
296
297  def _ComputeTimelineBasedMetric(self, results, trace_value):
298    metric = self._tbm_options.GetTimelineBasedMetric()
299    extra_import_options = {
300      'trackDetailedModelStats': True
301    }
302
303    mre_result = metric_runner.RunMetric(
304        trace_value.filename, metric, extra_import_options)
305    page = results.current_page
306
307    failure_dicts = mre_result.failures
308    for d in failure_dicts:
309      results.AddValue(
310          translate_common_values.TranslateMreFailure(d, page))
311
312    value_dicts = mre_result.pairs.get('values', [])
313    for d in value_dicts:
314      results.AddValue(
315          translate_common_values.TranslateScalarValue(d, page))
316
317  def _ComputeLegacyTimelineBasedMetrics(self, results, trace_result):
318    model = model_module.TimelineModel(trace_result)
319    threads_to_records_map = _GetRendererThreadsToInteractionRecordsMap(model)
320    if (len(threads_to_records_map.values()) == 0 and
321        self._tbm_options.config.enable_chrome_trace):
322      logging.warning(
323          'No timeline interaction records were recorded in the trace. '
324          'This could be caused by console.time() & console.timeEnd() execution'
325          ' failure or the tracing category specified doesn\'t include '
326          'blink.console categories.')
327
328    all_metrics = self._tbm_options.GetLegacyTimelineBasedMetrics()
329
330    for renderer_thread, interaction_records in (
331        threads_to_records_map.iteritems()):
332      meta_metrics = _TimelineBasedMetrics(
333          model, renderer_thread, interaction_records, self._results_wrapper,
334          all_metrics)
335      meta_metrics.AddResults(results)
336
337    for metric in all_metrics:
338      metric.AddWholeTraceResults(model, results)
339