1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import copy
7import datetime
8import json
9import logging
10import os
11import random
12import sys
13import tempfile
14import traceback
15
16from py_utils import cloud_storage  # pylint: disable=import-error
17
18from telemetry import value as value_module
19from telemetry.internal.results import chart_json_output_formatter
20from telemetry.internal.results import json_output_formatter
21from telemetry.internal.results import progress_reporter as reporter_module
22from telemetry.internal.results import story_run
23from telemetry.value import failure
24from telemetry.value import skip
25from telemetry.value import trace
26
27from tracing.value import convert_chart_json
28
29class TelemetryInfo(object):
30  def __init__(self):
31    self._benchmark_name = None
32    self._benchmark_start_ms = None
33    self._label = None
34    self._story_display_name = ''
35    self._story_grouping_keys = {}
36    self._storyset_repeat_counter = 0
37
38  @property
39  def benchmark_name(self):
40    return self._benchmark_name
41
42  @benchmark_name.setter
43  def benchmark_name(self, benchmark_name):
44    assert self.benchmark_name is None, (
45      'benchmark_name must be set exactly once')
46    self._benchmark_name = benchmark_name
47
48  @property
49  def benchmark_start_ms(self):
50    return self._benchmark_start_ms
51
52  @benchmark_start_ms.setter
53  def benchmark_start_ms(self, benchmark_start_ms):
54    assert self.benchmark_start_ms is None, (
55      'benchmark_start_ms must be set exactly once')
56    self._benchmark_start_ms = benchmark_start_ms
57
58  @property
59  def label(self):
60    return self._label
61
62  @label.setter
63  def label(self, label):
64    assert self.label is None, 'label cannot be set more than once'
65    self._label = label
66
67  @property
68  def story_display_name(self):
69    return self._story_display_name
70
71  @property
72  def story_grouping_keys(self):
73    return self._story_grouping_keys
74
75  @property
76  def storyset_repeat_counter(self):
77    return self._storyset_repeat_counter
78
79  def WillRunStory(self, story, storyset_repeat_counter):
80    self._story_display_name = story.display_name
81    if story.grouping_keys:
82      self._story_grouping_keys = story.grouping_keys
83    self._storyset_repeat_counter = storyset_repeat_counter
84
85  def AsDict(self):
86    assert self.benchmark_name is not None, (
87        'benchmark_name must be set exactly once')
88    assert self.benchmark_start_ms is not None, (
89        'benchmark_start_ms must be set exactly once')
90    d = {}
91    d['benchmarkName'] = self.benchmark_name
92    d['benchmarkStartMs'] = self.benchmark_start_ms
93    if self.label:
94      d['label'] = self.label
95    d['storyDisplayName'] = self.story_display_name
96    d['storyGroupingKeys'] = self.story_grouping_keys
97    d['storysetRepeatCounter'] = self.storyset_repeat_counter
98    return d
99
100
101class PageTestResults(object):
102  def __init__(self, output_formatters=None,
103               progress_reporter=None, trace_tag='', output_dir=None,
104               value_can_be_added_predicate=lambda v, is_first: True,
105               benchmark_enabled=True):
106    """
107    Args:
108      output_formatters: A list of output formatters. The output
109          formatters are typically used to format the test results, such
110          as CsvPivotTableOutputFormatter, which output the test results as CSV.
111      progress_reporter: An instance of progress_reporter.ProgressReporter,
112          to be used to output test status/results progressively.
113      trace_tag: A string to append to the buildbot trace name. Currently only
114          used for buildbot.
115      output_dir: A string specified the directory where to store the test
116          artifacts, e.g: trace, videos,...
117      value_can_be_added_predicate: A function that takes two arguments:
118          a value.Value instance (except failure.FailureValue, skip.SkipValue
119          or trace.TraceValue) and a boolean (True when the value is part of
120          the first result for the story). It returns True if the value
121          can be added to the test results and False otherwise.
122    """
123    # TODO(chrishenry): Figure out if trace_tag is still necessary.
124
125    super(PageTestResults, self).__init__()
126    self._progress_reporter = (
127        progress_reporter if progress_reporter is not None
128        else reporter_module.ProgressReporter())
129    self._output_formatters = (
130        output_formatters if output_formatters is not None else [])
131    self._trace_tag = trace_tag
132    self._output_dir = output_dir
133    self._value_can_be_added_predicate = value_can_be_added_predicate
134
135    self._current_page_run = None
136    self._all_page_runs = []
137    self._all_stories = set()
138    self._representative_value_for_each_value_name = {}
139    self._all_summary_values = []
140    self._serialized_trace_file_ids_to_paths = {}
141    self._pages_to_profiling_files = collections.defaultdict(list)
142    self._pages_to_profiling_files_cloud_url = collections.defaultdict(list)
143
144    # You'd expect this to be a set(), but Values are dictionaries, which are
145    # unhashable. We could wrap Values with custom __eq/hash__, but we don't
146    # actually need set-ness in python.
147    self._value_set = []
148
149    self._telemetry_info = TelemetryInfo()
150
151    # State of the benchmark this set of results represents.
152    self._benchmark_enabled = benchmark_enabled
153
154  @property
155  def telemetry_info(self):
156    return self._telemetry_info
157
158  @property
159  def value_set(self):
160    return self._value_set
161
162  def AsHistogramDicts(self, benchmark_metadata):
163    if self.value_set:
164      return self.value_set
165    chart_json = chart_json_output_formatter.ResultsAsChartDict(
166        benchmark_metadata, self.all_page_specific_values,
167        self.all_summary_values)
168    info = self.telemetry_info
169    chart_json['label'] = info.label
170    chart_json['benchmarkStartMs'] = info.benchmark_start_ms
171
172    file_descriptor, chart_json_path = tempfile.mkstemp()
173    os.close(file_descriptor)
174    json.dump(chart_json, file(chart_json_path, 'w'))
175
176    vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
177
178    os.remove(chart_json_path)
179
180    if vinn_result.returncode != 0:
181      logging.error('Error converting chart json to Histograms:\n' +
182          vinn_result.stdout)
183      return []
184    return json.loads(vinn_result.stdout)
185
186  def __copy__(self):
187    cls = self.__class__
188    result = cls.__new__(cls)
189    for k, v in self.__dict__.items():
190      if isinstance(v, collections.Container):
191        v = copy.copy(v)
192      setattr(result, k, v)
193    return result
194
195  @property
196  def pages_to_profiling_files(self):
197    return self._pages_to_profiling_files
198
199  @property
200  def serialized_trace_file_ids_to_paths(self):
201    return self._serialized_trace_file_ids_to_paths
202
203  @property
204  def pages_to_profiling_files_cloud_url(self):
205    return self._pages_to_profiling_files_cloud_url
206
207  @property
208  def all_page_specific_values(self):
209    values = []
210    for run in self._all_page_runs:
211      values += run.values
212    if self._current_page_run:
213      values += self._current_page_run.values
214    return values
215
216  @property
217  def all_summary_values(self):
218    return self._all_summary_values
219
220  @property
221  def current_page(self):
222    assert self._current_page_run, 'Not currently running test.'
223    return self._current_page_run.story
224
225  @property
226  def current_page_run(self):
227    assert self._current_page_run, 'Not currently running test.'
228    return self._current_page_run
229
230  @property
231  def all_page_runs(self):
232    return self._all_page_runs
233
234  @property
235  def pages_that_succeeded(self):
236    """Returns the set of pages that succeeded."""
237    pages = set(run.story for run in self.all_page_runs)
238    pages.difference_update(self.pages_that_failed)
239    return pages
240
241  @property
242  def pages_that_failed(self):
243    """Returns the set of failed pages."""
244    failed_pages = set()
245    for run in self.all_page_runs:
246      if run.failed:
247        failed_pages.add(run.story)
248    return failed_pages
249
250  @property
251  def failures(self):
252    values = self.all_page_specific_values
253    return [v for v in values if isinstance(v, failure.FailureValue)]
254
255  @property
256  def skipped_values(self):
257    values = self.all_page_specific_values
258    return [v for v in values if isinstance(v, skip.SkipValue)]
259
260  def _GetStringFromExcInfo(self, err):
261    return ''.join(traceback.format_exception(*err))
262
263  def CleanUp(self):
264    """Clean up any TraceValues contained within this results object."""
265    for run in self._all_page_runs:
266      for v in run.values:
267        if isinstance(v, trace.TraceValue):
268          v.CleanUp()
269          run.values.remove(v)
270
271  def __enter__(self):
272    return self
273
274  def __exit__(self, _, __, ___):
275    self.CleanUp()
276
277  def WillRunPage(self, page, storyset_repeat_counter=0):
278    assert not self._current_page_run, 'Did not call DidRunPage.'
279    self._current_page_run = story_run.StoryRun(page)
280    self._progress_reporter.WillRunPage(self)
281    self.telemetry_info.WillRunStory(
282        page, storyset_repeat_counter)
283
284  def DidRunPage(self, page):  # pylint: disable=unused-argument
285    """
286    Args:
287      page: The current page under test.
288    """
289    assert self._current_page_run, 'Did not call WillRunPage.'
290    self._progress_reporter.DidRunPage(self)
291    self._all_page_runs.append(self._current_page_run)
292    self._all_stories.add(self._current_page_run.story)
293    self._current_page_run = None
294
295  def AddValue(self, value):
296    assert self._current_page_run, 'Not currently running test.'
297    assert self._benchmark_enabled, 'Cannot add value to disabled results'
298    self._ValidateValue(value)
299    is_first_result = (
300      self._current_page_run.story not in self._all_stories)
301
302    story_keys = self._current_page_run.story.grouping_keys
303
304    if story_keys:
305      for k, v in story_keys.iteritems():
306        assert k not in value.grouping_keys, (
307            'Tried to add story grouping key ' + k + ' already defined by ' +
308            'value')
309        value.grouping_keys[k] = v
310
311      # We sort by key name to make building the tir_label deterministic.
312      story_keys_label = '_'.join(v for _, v in sorted(story_keys.iteritems()))
313      if value.tir_label:
314        assert value.tir_label == story_keys_label, (
315            'Value has an explicit tir_label (%s) that does not match the '
316            'one computed from story_keys (%s)' % (value.tir_label, story_keys))
317      else:
318        value.tir_label = story_keys_label
319
320    if not (isinstance(value, skip.SkipValue) or
321            isinstance(value, failure.FailureValue) or
322            isinstance(value, trace.TraceValue) or
323            self._value_can_be_added_predicate(value, is_first_result)):
324      return
325    # TODO(eakuefner/chrishenry): Add only one skip per pagerun assert here
326    self._current_page_run.AddValue(value)
327    self._progress_reporter.DidAddValue(value)
328
329  def AddProfilingFile(self, page, file_handle):
330    self._pages_to_profiling_files[page].append(file_handle)
331
332  def AddSummaryValue(self, value):
333    assert value.page is None
334    self._ValidateValue(value)
335    self._all_summary_values.append(value)
336
337  def _ValidateValue(self, value):
338    assert isinstance(value, value_module.Value)
339    if value.name not in self._representative_value_for_each_value_name:
340      self._representative_value_for_each_value_name[value.name] = value
341    representative_value = self._representative_value_for_each_value_name[
342        value.name]
343    assert value.IsMergableWith(representative_value)
344
345  def PrintSummary(self):
346    if self._benchmark_enabled:
347      self._progress_reporter.DidFinishAllTests(self)
348
349      # Only serialize the trace if output_format is json.
350      if (self._output_dir and
351          any(isinstance(o, json_output_formatter.JsonOutputFormatter)
352              for o in self._output_formatters)):
353        self._SerializeTracesToDirPath(self._output_dir)
354      for output_formatter in self._output_formatters:
355        output_formatter.Format(self)
356        output_formatter.PrintViewResults()
357    else:
358      for output_formatter in self._output_formatters:
359        output_formatter.FormatDisabled()
360
361  def FindValues(self, predicate):
362    """Finds all values matching the specified predicate.
363
364    Args:
365      predicate: A function that takes a Value and returns a bool.
366    Returns:
367      A list of values matching |predicate|.
368    """
369    values = []
370    for value in self.all_page_specific_values:
371      if predicate(value):
372        values.append(value)
373    return values
374
375  def FindPageSpecificValuesForPage(self, page, value_name):
376    return self.FindValues(lambda v: v.page == page and v.name == value_name)
377
378  def FindAllPageSpecificValuesNamed(self, value_name):
379    return self.FindValues(lambda v: v.name == value_name)
380
381  def FindAllPageSpecificValuesFromIRNamed(self, tir_label, value_name):
382    return self.FindValues(lambda v: v.name == value_name
383                           and v.tir_label == tir_label)
384
385  def FindAllTraceValues(self):
386    return self.FindValues(lambda v: isinstance(v, trace.TraceValue))
387
388  def _SerializeTracesToDirPath(self, dir_path):
389    """ Serialize all trace values to files in dir_path and return a list of
390    file handles to those files. """
391    for value in self.FindAllTraceValues():
392      fh = value.Serialize(dir_path)
393      self._serialized_trace_file_ids_to_paths[fh.id] = fh.GetAbsPath()
394
395  def UploadTraceFilesToCloud(self, bucket):
396    for value in self.FindAllTraceValues():
397      value.UploadToCloud(bucket)
398
399  def UploadProfilingFilesToCloud(self, bucket):
400    for page, file_handle_list in self._pages_to_profiling_files.iteritems():
401      for file_handle in file_handle_list:
402        remote_path = ('profiler-file-id_%s-%s%-d%s' % (
403            file_handle.id,
404            datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
405            random.randint(1, 100000),
406            file_handle.extension))
407        try:
408          cloud_url = cloud_storage.Insert(
409              bucket, remote_path, file_handle.GetAbsPath())
410          sys.stderr.write(
411              'View generated profiler files online at %s for page %s\n' %
412              (cloud_url, page.display_name))
413          self._pages_to_profiling_files_cloud_url[page].append(cloud_url)
414        except cloud_storage.PermissionError as e:
415          logging.error('Cannot upload profiling files to cloud storage due to '
416                        ' permission error: %s' % e.message)
417