1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Uploads performance data to the performance dashboard.
6
7Performance tests may output data that needs to be displayed on the performance
8dashboard.  The autotest TKO parser invokes this module with each test
9associated with a job.  If a test has performance data associated with it, it
10is uploaded to the performance dashboard.  The performance dashboard is owned
11by Chrome team and is available here: https://chromeperf.appspot.com/.  Users
12must be logged in with an @google.com account to view chromeOS perf data there.
13
14"""
15
16import httplib, json, math, os, re, urllib, urllib2
17
18import common
19from autotest_lib.client.cros import constants
20from autotest_lib.tko import utils as tko_utils
21
22_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
23_PRESENTATION_CONFIG_FILE = os.path.join(
24        _ROOT_DIR, 'perf_dashboard_config.json')
25_PRESENTATION_SHADOW_CONFIG_FILE = os.path.join(
26        _ROOT_DIR, 'perf_dashboard_shadow_config.json')
27_DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'
28
29# Format for Chrome and Chrome OS version strings.
30VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$'
31
32class PerfUploadingError(Exception):
33    """Exception raised in perf_uploader"""
34    pass
35
36
37def _aggregate_iterations(perf_values):
38    """Aggregate same measurements from multiple iterations.
39
40    Each perf measurement may exist multiple times across multiple iterations
41    of a test.  Here, the results for each unique measured perf metric are
42    aggregated across multiple iterations.
43
44    @param perf_values: A list of tko.models.perf_value_iteration objects.
45
46    @return A dictionary mapping each unique measured perf value (keyed by
47        tuple of its description and graph name) to information about that
48        perf value (in particular, the value is a list of values
49        for each iteration).
50
51    """
52    perf_data = {}
53    for perf_iteration in perf_values:
54        for perf_dict in perf_iteration.perf_measurements:
55            key = (perf_dict['description'], perf_dict['graph'])
56            if key not in perf_data:
57                perf_data[key] = {
58                    'units': perf_dict['units'],
59                    'higher_is_better': perf_dict['higher_is_better'],
60                    'graph': perf_dict['graph'],
61                    'value': [perf_dict['value']],   # Note: a list of values.
62                    'stddev': perf_dict['stddev']
63                }
64            else:
65                perf_data[key]['value'].append(perf_dict['value'])
66                # Note: the stddev will be recomputed later when the results
67                # from each of the multiple iterations are averaged together.
68    return perf_data
69
70
71def _mean_and_stddev(data, precision=4):
72    """Computes mean and standard deviation from a list of numbers.
73
74    Assumes that the list contains at least 2 numbers.
75
76    @param data: A list of numeric values.
77    @param precision: The integer number of decimal places to which to
78        round the results.
79
80    @return A 2-tuple (mean, standard_deviation), in which each value is
81        rounded to |precision| decimal places.
82
83    """
84    n = len(data)
85    mean = float(sum(data)) / n
86    # Divide by n-1 to compute "sample standard deviation".
87    variance = sum([(elem - mean) ** 2 for elem in data]) / (n - 1)
88    return round(mean, precision), round(math.sqrt(variance), precision)
89
90
91def _compute_avg_stddev(perf_data):
92    """Compute average and standard deviations as needed for perf measurements.
93
94    For any perf measurement that exists in multiple iterations (has more than
95    one measured value), compute the average and standard deviation for it and
96    then store the updated information in the dictionary.
97
98    @param perf_data: A dictionary of measured perf data as computed by
99        _aggregate_iterations(), except each value is now a single value, not a
100        list of values.
101
102    """
103    for perf_dict in perf_data.itervalues():
104        if len(perf_dict['value']) > 1:
105            perf_dict['value'], perf_dict['stddev'] = (
106                    _mean_and_stddev(map(float, perf_dict['value'])))
107        else:
108            perf_dict['value'] = perf_dict['value'][0]  # Take out of list.
109
110
111def _parse_config_file(config_file):
112    """Parses a presentation config file and stores the info into a dict.
113
114    The config file contains information about how to present the perf data
115    on the perf dashboard.  This is required if the default presentation
116    settings aren't desired for certain tests.
117
118    @param config_file: Path to the configuration file to be parsed.
119
120    @returns A dictionary mapping each unique autotest name to a dictionary
121        of presentation config information.
122
123    @raises PerfUploadingError if config data or master name for the test
124        is missing from the config file.
125
126    """
127    json_obj = []
128    if os.path.exists(config_file):
129        with open(config_file, 'r') as fp:
130            json_obj = json.load(fp)
131    config_dict = {}
132    for entry in json_obj:
133        config_dict[entry['autotest_name']] = entry
134    return config_dict
135
136
137def _gather_presentation_info(config_data, test_name):
138    """Gathers presentation info from config data for the given test name.
139
140    @param config_data: A dictionary of dashboard presentation info for all
141        tests, as returned by _parse_config_file().  Info is keyed by autotest
142        name.
143    @param test_name: The name of an autotest.
144
145    @return A dictionary containing presentation information extracted from
146        |config_data| for the given autotest name.
147
148    @raises PerfUploadingError if some required data is missing.
149    """
150    if not test_name in config_data:
151        raise PerfUploadingError(
152                'No config data is specified for test %s in %s.' %
153                (test_name, _PRESENTATION_CONFIG_FILE))
154
155    presentation_dict = config_data[test_name]
156    try:
157        master_name = presentation_dict['master_name']
158    except KeyError:
159        raise PerfUploadingError(
160                'No master name is specified for test %s in %s.' %
161                (test_name, _PRESENTATION_CONFIG_FILE))
162    if 'dashboard_test_name' in presentation_dict:
163        test_name = presentation_dict['dashboard_test_name']
164    return {'master_name': master_name, 'test_name': test_name}
165
166
167def _format_for_upload(platform_name, cros_version, chrome_version,
168                       hardware_id, variant_name, hardware_hostname,
169                       perf_data, presentation_info):
170    """Formats perf data suitably to upload to the perf dashboard.
171
172    The perf dashboard expects perf data to be uploaded as a
173    specially-formatted JSON string.  In particular, the JSON object must be a
174    dictionary with key "data", and value being a list of dictionaries where
175    each dictionary contains all the information associated with a single
176    measured perf value: master name, bot name, test name, perf value, error
177    value, units, and build version numbers.
178
179    @param platform_name: The string name of the platform.
180    @param cros_version: The string chromeOS version number.
181    @param chrome_version: The string chrome version number.
182    @param hardware_id: String that identifies the type of hardware the test was
183        executed on.
184    @param variant_name: String that identifies the variant name of the board.
185    @param hardware_hostname: String that identifies the name of the device the
186        test was executed on.
187    @param perf_data: A dictionary of measured perf data as computed by
188        _compute_avg_stddev().
189    @param presentation_info: A dictionary of dashboard presentation info for
190        the given test, as identified by _gather_presentation_info().
191
192    @return A dictionary containing the formatted information ready to upload
193        to the performance dashboard.
194
195    """
196    dash_entries = []
197    if variant_name:
198        platform_name += '-' + variant_name
199    for (desc, graph), data in perf_data.iteritems():
200        # Each perf metric is named by a path that encodes the test name,
201        # a graph name (if specified), and a description.  This must be defined
202        # according to rules set by the Chrome team, as implemented in:
203        # chromium/tools/build/scripts/slave/results_dashboard.py.
204        if desc.endswith('_ref'):
205            desc = 'ref'
206        desc = desc.replace('_by_url', '')
207        desc = desc.replace('/', '_')
208        if data['graph']:
209            test_path = '%s/%s/%s' % (presentation_info['test_name'],
210                                      data['graph'], desc)
211        else:
212            test_path = '%s/%s' % (presentation_info['test_name'], desc)
213
214        new_dash_entry = {
215            'master': presentation_info['master_name'],
216            'bot': 'cros-' + platform_name,  # Prefix to clarify it's chromeOS.
217            'test': test_path,
218            'value': data['value'],
219            'error': data['stddev'],
220            'units': data['units'],
221            'higher_is_better': data['higher_is_better'],
222            'revision': _get_id_from_version(chrome_version, cros_version),
223            'supplemental_columns': {
224                'r_cros_version': cros_version,
225                'r_chrome_version': chrome_version,
226                'a_default_rev': 'r_chrome_version',
227                'a_hardware_identifier': hardware_id,
228                'a_hardware_hostname': hardware_hostname,
229            }
230        }
231
232        dash_entries.append(new_dash_entry)
233
234    json_string = json.dumps(dash_entries)
235    return {'data': json_string}
236
237
238def _get_version_numbers(test_attributes):
239    """Gets the version numbers from the test attributes and validates them.
240
241    @param test_attributes: The attributes property (which is a dict) of an
242        autotest tko.models.test object.
243
244    @return A pair of strings (Chrome OS version, Chrome version).
245
246    @raises PerfUploadingError if a version isn't formatted as expected.
247    """
248    chrome_version = test_attributes.get('CHROME_VERSION', '')
249    cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '')
250    # Prefix the ChromeOS version number with the Chrome milestone.
251    cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version
252    if not re.match(VERSION_REGEXP, cros_version):
253        raise PerfUploadingError('CrOS version "%s" does not match expected '
254                                 'format.' % cros_version)
255    if not re.match(VERSION_REGEXP, chrome_version):
256        raise PerfUploadingError('Chrome version "%s" does not match expected '
257                                 'format.' % chrome_version)
258    return (cros_version, chrome_version)
259
260
261def _get_id_from_version(chrome_version, cros_version):
262    """Computes the point ID to use, from Chrome and ChromeOS version numbers.
263
264    For ChromeOS row data, data values are associated with both a Chrome
265    version number and a ChromeOS version number (unlike for Chrome row data
266    that is associated with a single revision number).  This function takes
267    both version numbers as input, then computes a single, unique integer ID
268    from them, which serves as a 'fake' revision number that can uniquely
269    identify each ChromeOS data point, and which will allow ChromeOS data points
270    to be sorted by Chrome version number, with ties broken by ChromeOS version
271    number.
272
273    To compute the integer ID, we take the portions of each version number that
274    serve as the shortest unambiguous names for each (as described here:
275    http://www.chromium.org/developers/version-numbers).  We then force each
276    component of each portion to be a fixed width (padded by zeros if needed),
277    concatenate all digits together (with those coming from the Chrome version
278    number first), and convert the entire string of digits into an integer.
279    We ensure that the total number of digits does not exceed that which is
280    allowed by AppEngine NDB for an integer (64-bit signed value).
281
282    For example:
283      Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2)
284      ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0)
285      concatenated together with padding for fixed-width columns:
286          ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000'
287      Final integer ID: 14520020390600000
288
289    @param chrome_ver: The Chrome version number as a string.
290    @param cros_ver: The ChromeOS version number as a string.
291
292    @return A unique integer ID associated with the two given version numbers.
293
294    """
295
296    # Number of digits to use from each part of the version string for Chrome
297    # and Chrome OS versions when building a point ID out of these two versions.
298    chrome_version_col_widths = [0, 0, 5, 3]
299    cros_version_col_widths = [0, 5, 3, 2]
300
301    def get_digits_from_version(version_num, column_widths):
302        if re.match(VERSION_REGEXP, version_num):
303            computed_string = ''
304            version_parts = version_num.split('.')
305            for i, version_part in enumerate(version_parts):
306                if column_widths[i]:
307                    computed_string += version_part.zfill(column_widths[i])
308            return computed_string
309        else:
310            return None
311
312    chrome_digits = get_digits_from_version(
313            chrome_version, chrome_version_col_widths)
314    cros_digits = get_digits_from_version(
315            cros_version, cros_version_col_widths)
316    if not chrome_digits or not cros_digits:
317        return None
318    result_digits = chrome_digits + cros_digits
319    max_digits = sum(chrome_version_col_widths + cros_version_col_widths)
320    if len(result_digits) > max_digits:
321        return None
322    return int(result_digits)
323
324
325def _send_to_dashboard(data_obj):
326    """Sends formatted perf data to the perf dashboard.
327
328    @param data_obj: A formatted data object as returned by
329        _format_for_upload().
330
331    @raises PerfUploadingError if an exception was raised when uploading.
332
333    """
334    encoded = urllib.urlencode(data_obj)
335    req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded)
336    try:
337        urllib2.urlopen(req)
338    except urllib2.HTTPError as e:
339        raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % (
340                e.code, e.msg, data_obj['data']))
341    except urllib2.URLError as e:
342        raise PerfUploadingError(
343                'URLError: %s for JSON %s\n' %
344                (str(e.reason), data_obj['data']))
345    except httplib.HTTPException:
346        raise PerfUploadingError(
347                'HTTPException for JSON %s\n' % data_obj['data'])
348
349
350def upload_test(job, test):
351    """Uploads any perf data associated with a test to the perf dashboard.
352
353    @param job: An autotest tko.models.job object that is associated with the
354        given |test|.
355    @param test: An autotest tko.models.test object that may or may not be
356        associated with measured perf data.
357
358    """
359    if not test.perf_values:
360        return
361
362    # Aggregate values from multiple iterations together.
363    perf_data = _aggregate_iterations(test.perf_values)
364
365    # Compute averages and standard deviations as needed for measured perf
366    # values that exist in multiple iterations.  Ultimately, we only upload a
367    # single measurement (with standard deviation) for every unique measured
368    # perf metric.
369    _compute_avg_stddev(perf_data)
370
371    # Format the perf data for the upload, then upload it.
372    test_name = test.testname
373    platform_name = job.machine_group
374    hardware_id = test.attributes.get('hwid', '')
375    hardware_hostname = test.machine
376    variant_name = test.attributes.get(constants.VARIANT_KEY, None)
377    config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE)
378    try:
379        shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE)
380        config_data.update(shadow_config_data)
381    except ValueError as e:
382        tko_utils.dprint('Failed to parse config file %s: %s.' %
383                         (_PRESENTATION_SHADOW_CONFIG_FILE, e))
384    try:
385        cros_version, chrome_version = _get_version_numbers(test.attributes)
386        presentation_info = _gather_presentation_info(config_data, test_name)
387        formatted_data = _format_for_upload(
388                platform_name, cros_version, chrome_version, hardware_id,
389                variant_name, hardware_hostname, perf_data, presentation_info)
390        _send_to_dashboard(formatted_data)
391    except PerfUploadingError as e:
392        tko_utils.dprint('Error when uploading perf data to the perf '
393                         'dashboard for test %s: %s' % (test_name, e))
394    else:
395        tko_utils.dprint('Successfully uploaded perf data to the perf '
396                         'dashboard for test %s.' % test_name)
397