1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Uploads performance data to the performance dashboard.
6
7Performance tests may output data that needs to be displayed on the performance
8dashboard.  The autotest TKO parser invokes this module with each test
9associated with a job.  If a test has performance data associated with it, it
10is uploaded to the performance dashboard.  The performance dashboard is owned
11by Chrome team and is available here: https://chromeperf.appspot.com/.  Users
12must be logged in with an @google.com account to view chromeOS perf data there.
13
14"""
15
16import httplib
17import json
18import os
19import re
20import urllib
21import urllib2
22
23import common
24from autotest_lib.client.cros import constants
25from autotest_lib.tko import utils as tko_utils
26
27_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
28_PRESENTATION_CONFIG_FILE = os.path.join(
29        _ROOT_DIR, 'perf_dashboard_config.json')
30_PRESENTATION_SHADOW_CONFIG_FILE = os.path.join(
31        _ROOT_DIR, 'perf_dashboard_shadow_config.json')
32_DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'
33
34# Format for Chrome and Chrome OS version strings.
35VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$'
36
37class PerfUploadingError(Exception):
38    """Exception raised in perf_uploader"""
39    pass
40
41
42def _parse_config_file(config_file):
43    """Parses a presentation config file and stores the info into a dict.
44
45    The config file contains information about how to present the perf data
46    on the perf dashboard.  This is required if the default presentation
47    settings aren't desired for certain tests.
48
49    @param config_file: Path to the configuration file to be parsed.
50
51    @returns A dictionary mapping each unique autotest name to a dictionary
52        of presentation config information.
53
54    @raises PerfUploadingError if config data or master name for the test
55        is missing from the config file.
56
57    """
58    json_obj = []
59    if os.path.exists(config_file):
60        with open(config_file, 'r') as fp:
61            json_obj = json.load(fp)
62    config_dict = {}
63    for entry in json_obj:
64        config_dict[entry['autotest_name']] = entry
65    return config_dict
66
67
68def _gather_presentation_info(config_data, test_name):
69    """Gathers presentation info from config data for the given test name.
70
71    @param config_data: A dictionary of dashboard presentation info for all
72        tests, as returned by _parse_config_file().  Info is keyed by autotest
73        name.
74    @param test_name: The name of an autotest.
75
76    @return A dictionary containing presentation information extracted from
77        |config_data| for the given autotest name.
78
79    @raises PerfUploadingError if some required data is missing.
80    """
81    if not test_name in config_data:
82        raise PerfUploadingError(
83                'No config data is specified for test %s in %s.' %
84                (test_name, _PRESENTATION_CONFIG_FILE))
85
86    presentation_dict = config_data[test_name]
87    try:
88        master_name = presentation_dict['master_name']
89    except KeyError:
90        raise PerfUploadingError(
91                'No master name is specified for test %s in %s.' %
92                (test_name, _PRESENTATION_CONFIG_FILE))
93    if 'dashboard_test_name' in presentation_dict:
94        test_name = presentation_dict['dashboard_test_name']
95    return {'master_name': master_name, 'test_name': test_name}
96
97
98def _format_for_upload(platform_name, cros_version, chrome_version,
99                       hardware_id, variant_name, hardware_hostname,
100                       perf_data, presentation_info, jobname):
101    """Formats perf data suitable to upload to the perf dashboard.
102
103    The perf dashboard expects perf data to be uploaded as a
104    specially-formatted JSON string.  In particular, the JSON object must be a
105    dictionary with key "data", and value being a list of dictionaries where
106    each dictionary contains all the information associated with a single
107    measured perf value: master name, bot name, test name, perf value, error
108    value, units, and build version numbers.
109
110    @param platform_name: The string name of the platform.
111    @param cros_version: The string chromeOS version number.
112    @param chrome_version: The string chrome version number.
113    @param hardware_id: String that identifies the type of hardware the test was
114            executed on.
115    @param variant_name: String that identifies the variant name of the board.
116    @param hardware_hostname: String that identifies the name of the device the
117            test was executed on.
118    @param perf_data: A dictionary of measured perf data as computed by
119            _compute_avg_stddev().
120    @param presentation_info: A dictionary of dashboard presentation info for
121            the given test, as identified by _gather_presentation_info().
122    @param jobname: A string uniquely identifying the test run, this enables
123            linking back from a test result to the logs of the test run.
124
125    @return A dictionary containing the formatted information ready to upload
126        to the performance dashboard.
127
128    """
129    if variant_name:
130        platform_name += '-' + variant_name
131
132    perf_values = perf_data
133    # Client side case - server side comes with its own charts data section.
134    if 'charts' not in perf_values:
135        perf_values = {
136          'format_version': '1.0',
137          'benchmark_name': presentation_info['test_name'],
138          'charts': perf_data,
139        }
140
141    dash_entry = {
142        'master': presentation_info['master_name'],
143        'bot': 'cros-' + platform_name,  # Prefix to clarify it's ChromeOS.
144        'point_id': _get_id_from_version(chrome_version, cros_version),
145        'versions': {
146            'cros_version': cros_version,
147            'chrome_version': chrome_version,
148        },
149        'supplemental': {
150            'default_rev': 'r_cros_version',
151            'hardware_identifier': hardware_id,
152            'hardware_hostname': hardware_hostname,
153            'variant_name': variant_name,
154            'jobname': jobname,
155        },
156        'chart_data': perf_values,
157    }
158    return {'data': json.dumps(dash_entry)}
159
160
161def _get_version_numbers(test_attributes):
162    """Gets the version numbers from the test attributes and validates them.
163
164    @param test_attributes: The attributes property (which is a dict) of an
165        autotest tko.models.test object.
166
167    @return A pair of strings (Chrome OS version, Chrome version).
168
169    @raises PerfUploadingError if a version isn't formatted as expected.
170    """
171    chrome_version = test_attributes.get('CHROME_VERSION', '')
172    cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '')
173    cros_milestone = test_attributes.get('CHROMEOS_RELEASE_CHROME_MILESTONE')
174    # Use the release milestone as the milestone if present, othewise prefix the
175    # cros version with the with the Chrome browser milestone.
176    if cros_milestone:
177      cros_version = "%s.%s" % (cros_milestone, cros_version)
178    else:
179      cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version
180    if not re.match(VERSION_REGEXP, cros_version):
181        raise PerfUploadingError('CrOS version "%s" does not match expected '
182                                 'format.' % cros_version)
183    if not re.match(VERSION_REGEXP, chrome_version):
184        raise PerfUploadingError('Chrome version "%s" does not match expected '
185                                 'format.' % chrome_version)
186    return (cros_version, chrome_version)
187
188
189def _get_id_from_version(chrome_version, cros_version):
190    """Computes the point ID to use, from Chrome and ChromeOS version numbers.
191
192    For ChromeOS row data, data values are associated with both a Chrome
193    version number and a ChromeOS version number (unlike for Chrome row data
194    that is associated with a single revision number).  This function takes
195    both version numbers as input, then computes a single, unique integer ID
196    from them, which serves as a 'fake' revision number that can uniquely
197    identify each ChromeOS data point, and which will allow ChromeOS data points
198    to be sorted by Chrome version number, with ties broken by ChromeOS version
199    number.
200
201    To compute the integer ID, we take the portions of each version number that
202    serve as the shortest unambiguous names for each (as described here:
203    http://www.chromium.org/developers/version-numbers).  We then force each
204    component of each portion to be a fixed width (padded by zeros if needed),
205    concatenate all digits together (with those coming from the Chrome version
206    number first), and convert the entire string of digits into an integer.
207    We ensure that the total number of digits does not exceed that which is
208    allowed by AppEngine NDB for an integer (64-bit signed value).
209
210    For example:
211      Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2)
212      ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0)
213      concatenated together with padding for fixed-width columns:
214          ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000'
215      Final integer ID: 14520020390600000
216
217    @param chrome_ver: The Chrome version number as a string.
218    @param cros_ver: The ChromeOS version number as a string.
219
220    @return A unique integer ID associated with the two given version numbers.
221
222    """
223
224    # Number of digits to use from each part of the version string for Chrome
225    # and Chrome OS versions when building a point ID out of these two versions.
226    chrome_version_col_widths = [0, 0, 5, 3]
227    cros_version_col_widths = [0, 5, 3, 2]
228
229    def get_digits_from_version(version_num, column_widths):
230        if re.match(VERSION_REGEXP, version_num):
231            computed_string = ''
232            version_parts = version_num.split('.')
233            for i, version_part in enumerate(version_parts):
234                if column_widths[i]:
235                    computed_string += version_part.zfill(column_widths[i])
236            return computed_string
237        else:
238            return None
239
240    chrome_digits = get_digits_from_version(
241            chrome_version, chrome_version_col_widths)
242    cros_digits = get_digits_from_version(
243            cros_version, cros_version_col_widths)
244    if not chrome_digits or not cros_digits:
245        return None
246    result_digits = chrome_digits + cros_digits
247    max_digits = sum(chrome_version_col_widths + cros_version_col_widths)
248    if len(result_digits) > max_digits:
249        return None
250    return int(result_digits)
251
252
253def _send_to_dashboard(data_obj):
254    """Sends formatted perf data to the perf dashboard.
255
256    @param data_obj: A formatted data object as returned by
257        _format_for_upload().
258
259    @raises PerfUploadingError if an exception was raised when uploading.
260
261    """
262    encoded = urllib.urlencode(data_obj)
263    req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded)
264    try:
265        urllib2.urlopen(req)
266    except urllib2.HTTPError as e:
267        raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % (
268                e.code, e.msg, data_obj['data']))
269    except urllib2.URLError as e:
270        raise PerfUploadingError(
271                'URLError: %s for JSON %s\n' %
272                (str(e.reason), data_obj['data']))
273    except httplib.HTTPException:
274        raise PerfUploadingError(
275                'HTTPException for JSON %s\n' % data_obj['data'])
276
277
278def upload_test(job, test, jobname):
279    """Uploads any perf data associated with a test to the perf dashboard.
280
281    @param job: An autotest tko.models.job object that is associated with the
282        given |test|.
283    @param test: An autotest tko.models.test object that may or may not be
284        associated with measured perf data.
285    @param jobname: A string uniquely identifying the test run, this enables
286            linking back from a test result to the logs of the test run.
287
288    """
289
290    # Format the perf data for the upload, then upload it.
291    test_name = test.testname
292    platform_name = job.machine_group
293    # Append the platform name with '.arc' if the suffix of the control
294    # filename is '.arc'.
295    if job.label and re.match('.*\.arc$', job.label):
296        platform_name += '.arc'
297    hardware_id = test.attributes.get('hwid', '')
298    hardware_hostname = test.machine
299    variant_name = test.attributes.get(constants.VARIANT_KEY, None)
300    config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE)
301    try:
302        shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE)
303        config_data.update(shadow_config_data)
304    except ValueError as e:
305        tko_utils.dprint('Failed to parse config file %s: %s.' %
306                         (_PRESENTATION_SHADOW_CONFIG_FILE, e))
307    try:
308        cros_version, chrome_version = _get_version_numbers(test.attributes)
309        presentation_info = _gather_presentation_info(config_data, test_name)
310        formatted_data = _format_for_upload(
311                platform_name, cros_version, chrome_version, hardware_id,
312                variant_name, hardware_hostname, test.perf_values,
313                presentation_info, jobname)
314        _send_to_dashboard(formatted_data)
315    except PerfUploadingError as e:
316        tko_utils.dprint('Error when uploading perf data to the perf '
317                         'dashboard for test %s: %s' % (test_name, e))
318    else:
319        tko_utils.dprint('Successfully uploaded perf data to the perf '
320                         'dashboard for test %s.' % test_name)
321
322