1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import copy
7import datetime
8import logging
9import random
10import sys
11import traceback
12
13from catapult_base import cloud_storage  # pylint: disable=import-error
14
15from telemetry.internal.results import json_output_formatter
16from telemetry.internal.results import progress_reporter as reporter_module
17from telemetry.internal.results import story_run
18from telemetry import value as value_module
19from telemetry.value import failure
20from telemetry.value import skip
21from telemetry.value import trace
22
23
24class PageTestResults(object):
25  def __init__(self, output_formatters=None,
26               progress_reporter=None, trace_tag='', output_dir=None,
27               value_can_be_added_predicate=lambda v, is_first: True):
28    """
29    Args:
30      output_formatters: A list of output formatters. The output
31          formatters are typically used to format the test results, such
32          as CsvPivotTableOutputFormatter, which output the test results as CSV.
33      progress_reporter: An instance of progress_reporter.ProgressReporter,
34          to be used to output test status/results progressively.
35      trace_tag: A string to append to the buildbot trace name. Currently only
36          used for buildbot.
37      output_dir: A string specified the directory where to store the test
38          artifacts, e.g: trace, videos,...
39      value_can_be_added_predicate: A function that takes two arguments:
40          a value.Value instance (except failure.FailureValue, skip.SkipValue
41          or trace.TraceValue) and a boolean (True when the value is part of
42          the first result for the story). It returns True if the value
43          can be added to the test results and False otherwise.
44    """
45    # TODO(chrishenry): Figure out if trace_tag is still necessary.
46
47    super(PageTestResults, self).__init__()
48    self._progress_reporter = (
49        progress_reporter if progress_reporter is not None
50        else reporter_module.ProgressReporter())
51    self._output_formatters = (
52        output_formatters if output_formatters is not None else [])
53    self._trace_tag = trace_tag
54    self._output_dir = output_dir
55    self._value_can_be_added_predicate = value_can_be_added_predicate
56
57    self._current_page_run = None
58    self._all_page_runs = []
59    self._all_stories = set()
60    self._representative_value_for_each_value_name = {}
61    self._all_summary_values = []
62    self._serialized_trace_file_ids_to_paths = {}
63    self._pages_to_profiling_files = collections.defaultdict(list)
64    self._pages_to_profiling_files_cloud_url = collections.defaultdict(list)
65
66  def __copy__(self):
67    cls = self.__class__
68    result = cls.__new__(cls)
69    for k, v in self.__dict__.items():
70      if isinstance(v, collections.Container):
71        v = copy.copy(v)
72      setattr(result, k, v)
73    return result
74
75  @property
76  def pages_to_profiling_files(self):
77    return self._pages_to_profiling_files
78
79  @property
80  def serialized_trace_file_ids_to_paths(self):
81    return self._serialized_trace_file_ids_to_paths
82
83  @property
84  def pages_to_profiling_files_cloud_url(self):
85    return self._pages_to_profiling_files_cloud_url
86
87  @property
88  def all_page_specific_values(self):
89    values = []
90    for run in self._all_page_runs:
91      values += run.values
92    if self._current_page_run:
93      values += self._current_page_run.values
94    return values
95
96  @property
97  def all_summary_values(self):
98    return self._all_summary_values
99
100  @property
101  def current_page(self):
102    assert self._current_page_run, 'Not currently running test.'
103    return self._current_page_run.story
104
105  @property
106  def current_page_run(self):
107    assert self._current_page_run, 'Not currently running test.'
108    return self._current_page_run
109
110  @property
111  def all_page_runs(self):
112    return self._all_page_runs
113
114  @property
115  def pages_that_succeeded(self):
116    """Returns the set of pages that succeeded."""
117    pages = set(run.story for run in self.all_page_runs)
118    pages.difference_update(self.pages_that_failed)
119    return pages
120
121  @property
122  def pages_that_failed(self):
123    """Returns the set of failed pages."""
124    failed_pages = set()
125    for run in self.all_page_runs:
126      if run.failed:
127        failed_pages.add(run.story)
128    return failed_pages
129
130  @property
131  def failures(self):
132    values = self.all_page_specific_values
133    return [v for v in values if isinstance(v, failure.FailureValue)]
134
135  @property
136  def skipped_values(self):
137    values = self.all_page_specific_values
138    return [v for v in values if isinstance(v, skip.SkipValue)]
139
140  def _GetStringFromExcInfo(self, err):
141    return ''.join(traceback.format_exception(*err))
142
143  def CleanUp(self):
144    """Clean up any TraceValues contained within this results object."""
145    for run in self._all_page_runs:
146      for v in run.values:
147        if isinstance(v, trace.TraceValue):
148          v.CleanUp()
149          run.values.remove(v)
150
151  def __enter__(self):
152    return self
153
154  def __exit__(self, _, __, ___):
155    self.CleanUp()
156
157  def WillRunPage(self, page):
158    assert not self._current_page_run, 'Did not call DidRunPage.'
159    self._current_page_run = story_run.StoryRun(page)
160    self._progress_reporter.WillRunPage(self)
161
162  def DidRunPage(self, page):  # pylint: disable=unused-argument
163    """
164    Args:
165      page: The current page under test.
166    """
167    assert self._current_page_run, 'Did not call WillRunPage.'
168    self._progress_reporter.DidRunPage(self)
169    self._all_page_runs.append(self._current_page_run)
170    self._all_stories.add(self._current_page_run.story)
171    self._current_page_run = None
172
173  def AddValue(self, value):
174    assert self._current_page_run, 'Not currently running test.'
175    self._ValidateValue(value)
176    is_first_result = (
177      self._current_page_run.story not in self._all_stories)
178    if not (isinstance(value, skip.SkipValue) or
179            isinstance(value, failure.FailureValue) or
180            isinstance(value, trace.TraceValue) or
181            self._value_can_be_added_predicate(value, is_first_result)):
182      return
183    # TODO(eakuefner/chrishenry): Add only one skip per pagerun assert here
184    self._current_page_run.AddValue(value)
185    self._progress_reporter.DidAddValue(value)
186
187  def AddProfilingFile(self, page, file_handle):
188    self._pages_to_profiling_files[page].append(file_handle)
189
190  def AddSummaryValue(self, value):
191    assert value.page is None
192    self._ValidateValue(value)
193    self._all_summary_values.append(value)
194
195  def _ValidateValue(self, value):
196    assert isinstance(value, value_module.Value)
197    if value.name not in self._representative_value_for_each_value_name:
198      self._representative_value_for_each_value_name[value.name] = value
199    representative_value = self._representative_value_for_each_value_name[
200        value.name]
201    assert value.IsMergableWith(representative_value)
202
203  def PrintSummary(self):
204    self._progress_reporter.DidFinishAllTests(self)
205
206    # Only serialize the trace if output_format is json.
207    if (self._output_dir and
208        any(isinstance(o, json_output_formatter.JsonOutputFormatter)
209            for o in self._output_formatters)):
210      self._SerializeTracesToDirPath(self._output_dir)
211    for output_formatter in self._output_formatters:
212      output_formatter.Format(self)
213
214  def FindValues(self, predicate):
215    """Finds all values matching the specified predicate.
216
217    Args:
218      predicate: A function that takes a Value and returns a bool.
219    Returns:
220      A list of values matching |predicate|.
221    """
222    values = []
223    for value in self.all_page_specific_values:
224      if predicate(value):
225        values.append(value)
226    return values
227
228  def FindPageSpecificValuesForPage(self, page, value_name):
229    return self.FindValues(lambda v: v.page == page and v.name == value_name)
230
231  def FindAllPageSpecificValuesNamed(self, value_name):
232    return self.FindValues(lambda v: v.name == value_name)
233
234  def FindAllPageSpecificValuesFromIRNamed(self, tir_label, value_name):
235    return self.FindValues(lambda v: v.name == value_name
236                           and v.tir_label == tir_label)
237
238  def FindAllTraceValues(self):
239    return self.FindValues(lambda v: isinstance(v, trace.TraceValue))
240
241  def _SerializeTracesToDirPath(self, dir_path):
242    """ Serialize all trace values to files in dir_path and return a list of
243    file handles to those files. """
244    for value in self.FindAllTraceValues():
245      fh = value.Serialize(dir_path)
246      self._serialized_trace_file_ids_to_paths[fh.id] = fh.GetAbsPath()
247
248  def UploadTraceFilesToCloud(self, bucket):
249    for value in self.FindAllTraceValues():
250      value.UploadToCloud(bucket)
251
252  def UploadProfilingFilesToCloud(self, bucket):
253    for page, file_handle_list in self._pages_to_profiling_files.iteritems():
254      for file_handle in file_handle_list:
255        remote_path = ('profiler-file-id_%s-%s%-d%s' % (
256            file_handle.id,
257            datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
258            random.randint(1, 100000),
259            file_handle.extension))
260        try:
261          cloud_url = cloud_storage.Insert(
262              bucket, remote_path, file_handle.GetAbsPath())
263          sys.stderr.write(
264              'View generated profiler files online at %s for page %s\n' %
265              (cloud_url, page.display_name))
266          self._pages_to_profiling_files_cloud_url[page].append(cloud_url)
267        except cloud_storage.PermissionError as e:
268          logging.error('Cannot upload profiling files to cloud storage due to '
269                        ' permission error: %s' % e.message)
270