1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6
7from telemetry.util import perf_tests_helper
8from telemetry.util import statistics
9from telemetry.value import improvement_direction
10from telemetry.value import list_of_scalar_values
11from telemetry.value import scalar
12from telemetry.web_perf.metrics import rendering_stats
13from telemetry.web_perf.metrics import timeline_based_metric
14
15
16NOT_ENOUGH_FRAMES_MESSAGE = (
17  'Not enough frames for smoothness metrics (at least two are required).\n'
18  'Issues that have caused this in the past:\n'
19  '- Browser bugs that prevents the page from redrawing\n'
20  '- Bugs in the synthetic gesture code\n'
21  '- Page and benchmark out of sync (e.g. clicked element was renamed)\n'
22  '- Pages that render extremely slow\n'
23  '- Pages that can\'t be scrolled')
24
25
26class SmoothnessMetric(timeline_based_metric.TimelineBasedMetric):
27  """Computes metrics that measure smoothness of animations over given ranges.
28
29  Animations are typically considered smooth if the frame rates are close to
30  60 frames per second (fps) and uniformly distributed over the sequence. To
31  determine if a timeline range contains a smooth animation, we update the
32  results object with several representative metrics:
33
34    frame_times: A list of raw frame times
35    mean_frame_time: The arithmetic mean of frame times
36    percentage_smooth: Percentage of frames that were hitting 60 FPS.
37    frame_time_discrepancy: The absolute discrepancy of frame timestamps
38    mean_pixels_approximated: The mean percentage of pixels approximated
39    queueing_durations: The queueing delay between compositor & main threads
40
41  Note that if any of the interaction records provided to AddResults have less
42  than 2 frames, we will return telemetry values with None values for each of
43  the smoothness metrics. Similarly, older browsers without support for
44  tracking the BeginMainFrame events will report a ListOfScalarValues with a
45  None value for the queueing duration metric.
46  """
47
48  def __init__(self):
49    super(SmoothnessMetric, self).__init__()
50
51  def AddResults(self, model, renderer_thread, interaction_records, results):
52    self.VerifyNonOverlappedRecords(interaction_records)
53    renderer_process = renderer_thread.parent
54    stats = rendering_stats.RenderingStats(
55      renderer_process, model.browser_process, model.surface_flinger_process,
56      [r.GetBounds() for r in interaction_records])
57    has_surface_flinger_stats = model.surface_flinger_process is not None
58    self._PopulateResultsFromStats(results, stats, has_surface_flinger_stats)
59
60  def _PopulateResultsFromStats(self, results, stats,
61                                has_surface_flinger_stats):
62    page = results.current_page
63    values = [
64        self._ComputeQueueingDuration(page, stats),
65        self._ComputeFrameTimeDiscrepancy(page, stats),
66        self._ComputeMeanPixelsApproximated(page, stats),
67        self._ComputeMeanPixelsCheckerboarded(page, stats)
68    ]
69    values += self._ComputeLatencyMetric(page, stats, 'input_event_latency',
70                                         stats.input_event_latency)
71    values += self._ComputeLatencyMetric(page, stats,
72                                         'main_thread_scroll_latency',
73                                         stats.main_thread_scroll_latency)
74    values.append(self._ComputeFirstGestureScrollUpdateLatencies(page, stats))
75    values += self._ComputeFrameTimeMetric(page, stats)
76    if has_surface_flinger_stats:
77      values += self._ComputeSurfaceFlingerMetric(page, stats)
78
79    for v in values:
80      results.AddValue(v)
81
82  def _HasEnoughFrames(self, list_of_frame_timestamp_lists):
83    """Whether we have collected at least two frames in every timestamp list."""
84    return all(len(s) >= 2 for s in list_of_frame_timestamp_lists)
85
86  @staticmethod
87  def _GetNormalizedDeltas(data, refresh_period, min_normalized_delta=None):
88    deltas = [t2 - t1 for t1, t2 in zip(data, data[1:])]
89    if min_normalized_delta != None:
90      deltas = [d for d in deltas
91                if d / refresh_period >= min_normalized_delta]
92    return (deltas, [delta / refresh_period for delta in deltas])
93
94  @staticmethod
95  def _JoinTimestampRanges(frame_timestamps):
96    """Joins ranges of timestamps, adjusting timestamps to remove deltas
97    between the start of a range and the end of the prior range.
98    """
99    timestamps = []
100    for timestamp_range in frame_timestamps:
101      if len(timestamps) == 0:
102        timestamps.extend(timestamp_range)
103      else:
104        for i in range(1, len(timestamp_range)):
105          timestamps.append(timestamps[-1] +
106              timestamp_range[i] - timestamp_range[i-1])
107    return timestamps
108
109  def _ComputeSurfaceFlingerMetric(self, page, stats):
110    jank_count = None
111    avg_surface_fps = None
112    max_frame_delay = None
113    frame_lengths = None
114    none_value_reason = None
115    if self._HasEnoughFrames(stats.frame_timestamps):
116      timestamps = self._JoinTimestampRanges(stats.frame_timestamps)
117      frame_count = len(timestamps)
118      milliseconds = timestamps[-1] - timestamps[0]
119      min_normalized_frame_length = 0.5
120
121      frame_lengths, normalized_frame_lengths = \
122          self._GetNormalizedDeltas(timestamps, stats.refresh_period,
123                                    min_normalized_frame_length)
124      if len(frame_lengths) < frame_count - 1:
125        logging.warning('Skipping frame lengths that are too short.')
126        frame_count = len(frame_lengths) + 1
127      if len(frame_lengths) == 0:
128        raise Exception('No valid frames lengths found.')
129      _, normalized_changes = \
130          self._GetNormalizedDeltas(frame_lengths, stats.refresh_period)
131      jankiness = [max(0, round(change)) for change in normalized_changes]
132      pause_threshold = 20
133      jank_count = sum(1 for change in jankiness
134                       if change > 0 and change < pause_threshold)
135      avg_surface_fps = int(round((frame_count - 1) * 1000.0 / milliseconds))
136      max_frame_delay = round(max(normalized_frame_lengths))
137      frame_lengths = normalized_frame_lengths
138    else:
139      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
140
141    return (
142        scalar.ScalarValue(
143            page, 'avg_surface_fps', 'fps', avg_surface_fps,
144            description='Average frames per second as measured by the '
145                        'platform\'s SurfaceFlinger.',
146            none_value_reason=none_value_reason,
147            improvement_direction=improvement_direction.UP),
148        scalar.ScalarValue(
149            page, 'jank_count', 'janks', jank_count,
150            description='Number of changes in frame rate as measured by the '
151                        'platform\'s SurfaceFlinger.',
152            none_value_reason=none_value_reason,
153            improvement_direction=improvement_direction.DOWN),
154        scalar.ScalarValue(
155            page, 'max_frame_delay', 'vsyncs', max_frame_delay,
156            description='Largest frame time as measured by the platform\'s '
157                        'SurfaceFlinger.',
158            none_value_reason=none_value_reason,
159            improvement_direction=improvement_direction.DOWN),
160        list_of_scalar_values.ListOfScalarValues(
161            page, 'frame_lengths', 'vsyncs', frame_lengths,
162            description='Frame time in vsyncs as measured by the platform\'s '
163                        'SurfaceFlinger.',
164            none_value_reason=none_value_reason,
165            improvement_direction=improvement_direction.DOWN)
166    )
167
168  def _ComputeLatencyMetric(self, page, stats, name, list_of_latency_lists):
169    """Returns Values for the mean and discrepancy for given latency stats."""
170    mean_latency = None
171    latency_discrepancy = None
172    none_value_reason = None
173    if self._HasEnoughFrames(stats.frame_timestamps):
174      latency_list = perf_tests_helper.FlattenList(list_of_latency_lists)
175      if len(latency_list) == 0:
176        return ()
177      mean_latency = round(statistics.ArithmeticMean(latency_list), 3)
178      latency_discrepancy = (
179          round(statistics.DurationsDiscrepancy(latency_list), 4))
180    else:
181      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
182    return (
183      scalar.ScalarValue(
184          page, 'mean_%s' % name, 'ms', mean_latency,
185          description='Arithmetic mean of the raw %s values' % name,
186          none_value_reason=none_value_reason,
187          improvement_direction=improvement_direction.DOWN),
188      scalar.ScalarValue(
189          page, '%s_discrepancy' % name, 'ms', latency_discrepancy,
190          description='Discrepancy of the raw %s values' % name,
191          none_value_reason=none_value_reason,
192          improvement_direction=improvement_direction.DOWN)
193    )
194
195  def _ComputeFirstGestureScrollUpdateLatencies(self, page, stats):
196    """Returns a ListOfScalarValuesValues of gesture scroll update latencies.
197
198    Returns a Value for the first gesture scroll update latency for each
199    interaction record in |stats|.
200    """
201    none_value_reason = None
202    first_gesture_scroll_update_latencies = [round(latencies[0], 4)
203        for latencies in stats.gesture_scroll_update_latency
204        if len(latencies)]
205    if (not self._HasEnoughFrames(stats.frame_timestamps) or
206        not first_gesture_scroll_update_latencies):
207      first_gesture_scroll_update_latencies = None
208      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
209    return list_of_scalar_values.ListOfScalarValues(
210        page, 'first_gesture_scroll_update_latency', 'ms',
211        first_gesture_scroll_update_latencies,
212        description='First gesture scroll update latency measures the time it '
213                    'takes to process the very first gesture scroll update '
214                    'input event. The first scroll gesture can often get '
215                    'delayed by work related to page loading.',
216        none_value_reason=none_value_reason,
217        improvement_direction=improvement_direction.DOWN)
218
219  def _ComputeQueueingDuration(self, page, stats):
220    """Returns a Value for the frame queueing durations."""
221    queueing_durations = None
222    none_value_reason = None
223    if 'frame_queueing_durations' in stats.errors:
224      none_value_reason = stats.errors['frame_queueing_durations']
225    elif self._HasEnoughFrames(stats.frame_timestamps):
226      queueing_durations = perf_tests_helper.FlattenList(
227          stats.frame_queueing_durations)
228      if len(queueing_durations) == 0:
229        queueing_durations = None
230        none_value_reason = 'No frame queueing durations recorded.'
231    else:
232      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
233    return list_of_scalar_values.ListOfScalarValues(
234        page, 'queueing_durations', 'ms', queueing_durations,
235        description='The frame queueing duration quantifies how out of sync '
236                    'the compositor and renderer threads are. It is the amount '
237                    'of wall time that elapses between a '
238                    'ScheduledActionSendBeginMainFrame event in the compositor '
239                    'thread and the corresponding BeginMainFrame event in the '
240                    'main thread.',
241        none_value_reason=none_value_reason,
242        improvement_direction=improvement_direction.DOWN)
243
244  def _ComputeFrameTimeMetric(self, page, stats):
245    """Returns Values for the frame time metrics.
246
247    This includes the raw and mean frame times, as well as the percentage of
248    frames that were hitting 60 fps.
249    """
250    frame_times = None
251    mean_frame_time = None
252    percentage_smooth = None
253    none_value_reason = None
254    if self._HasEnoughFrames(stats.frame_timestamps):
255      frame_times = perf_tests_helper.FlattenList(stats.frame_times)
256      mean_frame_time = round(statistics.ArithmeticMean(frame_times), 3)
257      # We use 17ms as a somewhat looser threshold, instead of 1000.0/60.0.
258      smooth_threshold = 17.0
259      smooth_count = sum(1 for t in frame_times if t < smooth_threshold)
260      percentage_smooth = float(smooth_count) / len(frame_times) * 100.0
261    else:
262      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
263    return (
264        list_of_scalar_values.ListOfScalarValues(
265            page, 'frame_times', 'ms', frame_times,
266            description='List of raw frame times, helpful to understand the '
267                        'other metrics.',
268            none_value_reason=none_value_reason,
269            improvement_direction=improvement_direction.DOWN),
270        scalar.ScalarValue(
271            page, 'mean_frame_time', 'ms', mean_frame_time,
272            description='Arithmetic mean of frame times.',
273            none_value_reason=none_value_reason,
274            improvement_direction=improvement_direction.DOWN),
275        scalar.ScalarValue(
276            page, 'percentage_smooth', 'score', percentage_smooth,
277            description='Percentage of frames that were hitting 60 fps.',
278            none_value_reason=none_value_reason,
279            improvement_direction=improvement_direction.UP)
280    )
281
282  def _ComputeFrameTimeDiscrepancy(self, page, stats):
283    """Returns a Value for the absolute discrepancy of frame time stamps."""
284
285    frame_discrepancy = None
286    none_value_reason = None
287    if self._HasEnoughFrames(stats.frame_timestamps):
288      frame_discrepancy = round(statistics.TimestampsDiscrepancy(
289          stats.frame_timestamps), 4)
290    else:
291      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
292    return scalar.ScalarValue(
293        page, 'frame_time_discrepancy', 'ms', frame_discrepancy,
294        description='Absolute discrepancy of frame time stamps, where '
295                    'discrepancy is a measure of irregularity. It quantifies '
296                    'the worst jank. For a single pause, discrepancy '
297                    'corresponds to the length of this pause in milliseconds. '
298                    'Consecutive pauses increase the discrepancy. This metric '
299                    'is important because even if the mean and 95th '
300                    'percentile are good, one long pause in the middle of an '
301                    'interaction is still bad.',
302        none_value_reason=none_value_reason,
303        improvement_direction=improvement_direction.DOWN)
304
305  def _ComputeMeanPixelsApproximated(self, page, stats):
306    """Add the mean percentage of pixels approximated.
307
308    This looks at tiles which are missing or of low or non-ideal resolution.
309    """
310    mean_pixels_approximated = None
311    none_value_reason = None
312    if self._HasEnoughFrames(stats.frame_timestamps):
313      mean_pixels_approximated = round(statistics.ArithmeticMean(
314          perf_tests_helper.FlattenList(
315              stats.approximated_pixel_percentages)), 3)
316    else:
317      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
318    return scalar.ScalarValue(
319        page, 'mean_pixels_approximated', 'percent', mean_pixels_approximated,
320        description='Percentage of pixels that were approximated '
321                    '(checkerboarding, low-resolution tiles, etc.).',
322        none_value_reason=none_value_reason,
323        improvement_direction=improvement_direction.DOWN)
324
325  def _ComputeMeanPixelsCheckerboarded(self, page, stats):
326    """Add the mean percentage of pixels checkerboarded.
327
328    This looks at tiles which are only missing.
329    It does not take into consideration tiles which are of low or
330    non-ideal resolution.
331    """
332    mean_pixels_checkerboarded = None
333    none_value_reason = None
334    if self._HasEnoughFrames(stats.frame_timestamps):
335      if rendering_stats.CHECKERBOARDED_PIXEL_ERROR in stats.errors:
336        none_value_reason = stats.errors[
337            rendering_stats.CHECKERBOARDED_PIXEL_ERROR]
338      else:
339        mean_pixels_checkerboarded = round(statistics.ArithmeticMean(
340            perf_tests_helper.FlattenList(
341                stats.checkerboarded_pixel_percentages)), 3)
342    else:
343      none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
344    return scalar.ScalarValue(
345        page, 'mean_pixels_checkerboarded', 'percent',
346        mean_pixels_checkerboarded,
347        description='Percentage of pixels that were checkerboarded.',
348        none_value_reason=none_value_reason,
349        improvement_direction=improvement_direction.DOWN)
350