1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import collections
5import logging
6from collections import defaultdict
7
8from tracing.metrics import metric_runner
9
10from telemetry.timeline import chrome_trace_category_filter
11from telemetry.timeline import model as model_module
12from telemetry.timeline import tracing_config
13from telemetry.value import trace
14from telemetry.value import common_value_helpers
15from telemetry.web_perf.metrics import timeline_based_metric
16from telemetry.web_perf.metrics import blob_timeline
17from telemetry.web_perf.metrics import jitter_timeline
18from telemetry.web_perf.metrics import webrtc_rendering_timeline
19from telemetry.web_perf.metrics import gpu_timeline
20from telemetry.web_perf.metrics import indexeddb_timeline
21from telemetry.web_perf.metrics import layout
22from telemetry.web_perf.metrics import smoothness
23from telemetry.web_perf.metrics import text_selection
24from telemetry.web_perf import smooth_gesture_util
25from telemetry.web_perf import story_test
26from telemetry.web_perf import timeline_interaction_record as tir_module
27
28# TimelineBasedMeasurement considers all instrumentation as producing a single
29# timeline. But, depending on the amount of instrumentation that is enabled,
30# overhead increases. The user of the measurement must therefore chose between
31# a few levels of instrumentation.
32LOW_OVERHEAD_LEVEL = 'low-overhead'
33DEFAULT_OVERHEAD_LEVEL = 'default-overhead'
34DEBUG_OVERHEAD_LEVEL = 'debug-overhead'
35
36ALL_OVERHEAD_LEVELS = [
37  LOW_OVERHEAD_LEVEL,
38  DEFAULT_OVERHEAD_LEVEL,
39  DEBUG_OVERHEAD_LEVEL,
40]
41
42
43def _GetAllLegacyTimelineBasedMetrics():
44  # TODO(nednguyen): use discovery pattern to return all the instances of
45  # all TimelineBasedMetrics class in web_perf/metrics/ folder.
46  # This cannot be done until crbug.com/460208 is fixed.
47  return (smoothness.SmoothnessMetric(),
48          layout.LayoutMetric(),
49          gpu_timeline.GPUTimelineMetric(),
50          blob_timeline.BlobTimelineMetric(),
51          jitter_timeline.JitterTimelineMetric(),
52          text_selection.TextSelectionMetric(),
53          indexeddb_timeline.IndexedDBTimelineMetric(),
54          webrtc_rendering_timeline.WebRtcRenderingTimelineMetric())
55
56
57class InvalidInteractions(Exception):
58  pass
59
60
61# TODO(nednguyen): Get rid of this results wrapper hack after we add interaction
62# record to telemetry value system (crbug.com/453109)
63class ResultsWrapperInterface(object):
64  def __init__(self):
65    self._tir_label = None
66    self._results = None
67
68  def SetResults(self, results):
69    self._results = results
70
71  def SetTirLabel(self, tir_label):
72    self._tir_label = tir_label
73
74  @property
75  def current_page(self):
76    return self._results.current_page
77
78  def AddValue(self, value):
79    raise NotImplementedError
80
81
82class _TBMResultWrapper(ResultsWrapperInterface):
83  def AddValue(self, value):
84    assert self._tir_label
85    if value.tir_label:
86      assert value.tir_label == self._tir_label
87    else:
88      value.tir_label = self._tir_label
89    self._results.AddValue(value)
90
91
92def _GetRendererThreadsToInteractionRecordsMap(model):
93  threads_to_records_map = defaultdict(list)
94  interaction_labels_of_previous_threads = set()
95  for curr_thread in model.GetAllThreads():
96    for event in curr_thread.async_slices:
97      # TODO(nduca): Add support for page-load interaction record.
98      if tir_module.IsTimelineInteractionRecord(event.name):
99        interaction = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
100        # Adjust the interaction record to match the synthetic gesture
101        # controller if needed.
102        interaction = (
103            smooth_gesture_util.GetAdjustedInteractionIfContainGesture(
104                model, interaction))
105        threads_to_records_map[curr_thread].append(interaction)
106        if interaction.label in interaction_labels_of_previous_threads:
107          raise InvalidInteractions(
108            'Interaction record label %s is duplicated on different '
109            'threads' % interaction.label)
110    if curr_thread in threads_to_records_map:
111      interaction_labels_of_previous_threads.update(
112        r.label for r in threads_to_records_map[curr_thread])
113
114  return threads_to_records_map
115
116
117class _TimelineBasedMetrics(object):
118  def __init__(self, model, renderer_thread, interaction_records,
119               results_wrapper, metrics):
120    self._model = model
121    self._renderer_thread = renderer_thread
122    self._interaction_records = interaction_records
123    self._results_wrapper = results_wrapper
124    self._all_metrics = metrics
125
126  def AddResults(self, results):
127    interactions_by_label = defaultdict(list)
128    for i in self._interaction_records:
129      interactions_by_label[i.label].append(i)
130
131    for label, interactions in interactions_by_label.iteritems():
132      are_repeatable = [i.repeatable for i in interactions]
133      if not all(are_repeatable) and len(interactions) > 1:
134        raise InvalidInteractions('Duplicate unrepeatable interaction records '
135                                  'on the page')
136      self._results_wrapper.SetResults(results)
137      self._results_wrapper.SetTirLabel(label)
138      self.UpdateResultsByMetric(interactions, self._results_wrapper)
139
140  def UpdateResultsByMetric(self, interactions, wrapped_results):
141    if not interactions:
142      return
143
144    for metric in self._all_metrics:
145      metric.AddResults(self._model, self._renderer_thread,
146                        interactions, wrapped_results)
147
148
149class Options(object):
150  """A class to be used to configure TimelineBasedMeasurement.
151
152  This is created and returned by
153  Benchmark.CreateTimelineBasedMeasurementOptions.
154
155  By default, all the timeline based metrics in telemetry/web_perf/metrics are
156  used (see _GetAllLegacyTimelineBasedMetrics above).
157  To customize your metric needs, use SetTimelineBasedMetrics().
158  """
159
160  def __init__(self, overhead_level=LOW_OVERHEAD_LEVEL):
161    """As the amount of instrumentation increases, so does the overhead.
162    The user of the measurement chooses the overhead level that is appropriate,
163    and the tracing is filtered accordingly.
164
165    overhead_level: Can either be a custom ChromeTraceCategoryFilter object or
166        one of LOW_OVERHEAD_LEVEL, DEFAULT_OVERHEAD_LEVEL or
167        DEBUG_OVERHEAD_LEVEL.
168    """
169    self._config = tracing_config.TracingConfig()
170    self._config.enable_chrome_trace = True
171    self._config.enable_platform_display_trace = False
172
173    if isinstance(overhead_level,
174                  chrome_trace_category_filter.ChromeTraceCategoryFilter):
175      self._config.chrome_trace_config.SetCategoryFilter(overhead_level)
176    elif overhead_level in ALL_OVERHEAD_LEVELS:
177      if overhead_level == LOW_OVERHEAD_LEVEL:
178        self._config.chrome_trace_config.SetLowOverheadFilter()
179      elif overhead_level == DEFAULT_OVERHEAD_LEVEL:
180        self._config.chrome_trace_config.SetDefaultOverheadFilter()
181      else:
182        self._config.chrome_trace_config.SetDebugOverheadFilter()
183    else:
184      raise Exception("Overhead level must be a ChromeTraceCategoryFilter "
185                      "object or valid overhead level string. Given overhead "
186                      "level: %s" % overhead_level)
187
188    self._timeline_based_metrics = None
189    self._legacy_timeline_based_metrics = []
190
191
192  def ExtendTraceCategoryFilter(self, filters):
193    category_filter = self._config.chrome_trace_config.category_filter
194    for new_category_filter in filters:
195      category_filter.AddIncludedCategory(new_category_filter)
196
197  @property
198  def category_filter(self):
199    return self._config.chrome_trace_config.category_filter
200
201  @property
202  def config(self):
203    return self._config
204
205  def AddTimelineBasedMetric(self, metric):
206    assert isinstance(metric, basestring)
207    if self._timeline_based_metrics is None:
208      self._timeline_based_metrics = []
209    self._timeline_based_metrics.append(metric)
210
211  def SetTimelineBasedMetrics(self, metrics):
212    """Sets the new-style (TBMv2) metrics to run.
213
214    Metrics are assumed to live in //tracing/tracing/metrics, so the path you
215    pass in should be relative to that. For example, to specify
216    sample_metric.html, you should pass in ['sample_metric.html'].
217
218    Args:
219      metrics: A list of strings giving metric paths under
220          //tracing/tracing/metrics.
221    """
222    assert isinstance(metrics, list)
223    for metric in metrics:
224      assert isinstance(metric, basestring)
225    self._timeline_based_metrics = metrics
226
227  def GetTimelineBasedMetrics(self):
228    return self._timeline_based_metrics
229
230  def SetLegacyTimelineBasedMetrics(self, metrics):
231    assert isinstance(metrics, collections.Iterable)
232    for m in metrics:
233      assert isinstance(m, timeline_based_metric.TimelineBasedMetric)
234    self._legacy_timeline_based_metrics = metrics
235
236  def GetLegacyTimelineBasedMetrics(self):
237    return self._legacy_timeline_based_metrics
238
239
240class TimelineBasedMeasurement(story_test.StoryTest):
241  """Collects multiple metrics based on their interaction records.
242
243  A timeline based measurement shifts the burden of what metrics to collect onto
244  the story under test. Instead of the measurement
245  having a fixed set of values it collects, the story being tested
246  issues (via javascript) an Interaction record into the user timing API that
247  describing what is happening at that time, as well as a standardized set
248  of flags describing the semantics of the work being done. The
249  TimelineBasedMeasurement object collects a trace that includes both these
250  interaction records, and a user-chosen amount of performance data using
251  Telemetry's various timeline-producing APIs, tracing especially.
252
253  It then passes the recorded timeline to different TimelineBasedMetrics based
254  on those flags. As an example, this allows a single story run to produce
255  load timing data, smoothness data, critical jank information and overall cpu
256  usage information.
257
258  For information on how to mark up a page to work with
259  TimelineBasedMeasurement, refer to the
260  perf.metrics.timeline_interaction_record module.
261
262  Args:
263      options: an instance of timeline_based_measurement.Options.
264      results_wrapper: A class that has the __init__ method takes in
265        the page_test_results object and the interaction record label. This
266        class follows the ResultsWrapperInterface. Note: this class is not
267        supported long term and to be removed when crbug.com/453109 is resolved.
268  """
269  def __init__(self, options, results_wrapper=None):
270    self._tbm_options = options
271    self._results_wrapper = results_wrapper or _TBMResultWrapper()
272
273  def WillRunStory(self, platform):
274    """Configure and start tracing."""
275    if not platform.tracing_controller.IsChromeTracingSupported():
276      raise Exception('Not supported')
277    if self._tbm_options.config.enable_chrome_trace:
278      # Always enable 'blink.console' category for:
279      # 1) Backward compat of chrome clock sync (crbug.com/646925)
280      # 2) Allows users to add trace event through javascript.
281      # Note that blink.console is extremely low-overhead, so this doesn't
282      # affect the tracing overhead budget much.
283      chrome_config = self._tbm_options.config.chrome_trace_config
284      chrome_config.category_filter.AddIncludedCategory('blink.console')
285    platform.tracing_controller.StartTracing(self._tbm_options.config)
286
287  def Measure(self, platform, results):
288    """Collect all possible metrics and added them to results."""
289    platform.tracing_controller.telemetry_info = results.telemetry_info
290    trace_result = platform.tracing_controller.StopTracing()
291    trace_value = trace.TraceValue(results.current_page, trace_result)
292    results.AddValue(trace_value)
293
294    try:
295      if self._tbm_options.GetTimelineBasedMetrics():
296        assert not self._tbm_options.GetLegacyTimelineBasedMetrics(), (
297            'Specifying both TBMv1 and TBMv2 metrics is not allowed.')
298        self._ComputeTimelineBasedMetrics(results, trace_value)
299      else:
300        # Run all TBMv1 metrics if no other metric is specified
301        # (legacy behavior)
302        if not self._tbm_options.GetLegacyTimelineBasedMetrics():
303          logging.warn('Please specify the TBMv1 metrics you are interested in '
304                       'explicitly. This implicit functionality will be removed'
305                       ' on July 17, 2016.')
306          self._tbm_options.SetLegacyTimelineBasedMetrics(
307              _GetAllLegacyTimelineBasedMetrics())
308        self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
309    finally:
310      trace_result.CleanUpAllTraces()
311
312  def DidRunStory(self, platform):
313    """Clean up after running the story."""
314    if platform.tracing_controller.is_tracing_running:
315      platform.tracing_controller.StopTracing()
316
317  def _ComputeTimelineBasedMetrics(self, results, trace_value):
318    metrics = self._tbm_options.GetTimelineBasedMetrics()
319    extra_import_options = {
320      'trackDetailedModelStats': True
321    }
322
323    mre_result = metric_runner.RunMetric(
324        trace_value.filename, metrics, extra_import_options)
325    page = results.current_page
326
327    failure_dicts = mre_result.failures
328    for d in failure_dicts:
329      results.AddValue(
330          common_value_helpers.TranslateMreFailure(d, page))
331
332    results.value_set.extend(mre_result.pairs.get('histograms', []))
333
334    for d in mre_result.pairs.get('scalars', []):
335      results.AddValue(common_value_helpers.TranslateScalarValue(d, page))
336
337  def _ComputeLegacyTimelineBasedMetrics(self, results, trace_result):
338    model = model_module.TimelineModel(trace_result)
339    threads_to_records_map = _GetRendererThreadsToInteractionRecordsMap(model)
340    if (len(threads_to_records_map.values()) == 0 and
341        self._tbm_options.config.enable_chrome_trace):
342      logging.warning(
343          'No timeline interaction records were recorded in the trace. '
344          'This could be caused by console.time() & console.timeEnd() execution'
345          ' failure or the tracing category specified doesn\'t include '
346          'blink.console categories.')
347
348    all_metrics = self._tbm_options.GetLegacyTimelineBasedMetrics()
349
350    for renderer_thread, interaction_records in (
351        threads_to_records_map.iteritems()):
352      meta_metrics = _TimelineBasedMetrics(
353          model, renderer_thread, interaction_records, self._results_wrapper,
354          all_metrics)
355      meta_metrics.AddResults(results)
356
357    for metric in all_metrics:
358      metric.AddWholeTraceResults(model, results)
359