1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Uploads performance data to the performance dashboard.
6
7Performance tests may output data that needs to be displayed on the performance
8dashboard.  The autotest TKO parser invokes this module with each test
9associated with a job.  If a test has performance data associated with it, it
10is uploaded to the performance dashboard.  The performance dashboard is owned
11by Chrome team and is available here: https://chromeperf.appspot.com/.  Users
12must be logged in with an @google.com account to view chromeOS perf data there.
13
14"""
15
16import httplib
17import json
18import os
19import re
20import urllib
21import urllib2
22
23import common
24from autotest_lib.client.cros import constants
25from autotest_lib.tko import utils as tko_utils
26
27_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
28_PRESENTATION_CONFIG_FILE = os.path.join(
29        _ROOT_DIR, 'perf_dashboard_config.json')
30_PRESENTATION_SHADOW_CONFIG_FILE = os.path.join(
31        _ROOT_DIR, 'perf_dashboard_shadow_config.json')
32_DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'
33
34# Format for Chrome and Chrome OS version strings.
35VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$'
36
37class PerfUploadingError(Exception):
38    """Exception raised in perf_uploader"""
39    pass
40
41
42def _parse_config_file(config_file):
43    """Parses a presentation config file and stores the info into a dict.
44
45    The config file contains information about how to present the perf data
46    on the perf dashboard.  This is required if the default presentation
47    settings aren't desired for certain tests.
48
49    @param config_file: Path to the configuration file to be parsed.
50
51    @returns A dictionary mapping each unique autotest name to a dictionary
52        of presentation config information.
53
54    @raises PerfUploadingError if config data or master name for the test
55        is missing from the config file.
56
57    """
58    json_obj = []
59    if os.path.exists(config_file):
60        with open(config_file, 'r') as fp:
61            json_obj = json.load(fp)
62    config_dict = {}
63    for entry in json_obj:
64        config_dict[entry['autotest_name']] = entry
65    return config_dict
66
67
68def _gather_presentation_info(config_data, test_name):
69    """Gathers presentation info from config data for the given test name.
70
71    @param config_data: A dictionary of dashboard presentation info for all
72        tests, as returned by _parse_config_file().  Info is keyed by autotest
73        name.
74    @param test_name: The name of an autotest.
75
76    @return A dictionary containing presentation information extracted from
77        |config_data| for the given autotest name.
78
79    @raises PerfUploadingError if some required data is missing.
80    """
81    presentation_dict = None
82    for regex in config_data:
83        match = re.match(regex, test_name)
84        if match:
85            presentation_dict = config_data[regex]
86            break
87
88    if not presentation_dict:
89        raise PerfUploadingError(
90                'No config data is specified for test %s in %s.' %
91                (test_name, _PRESENTATION_CONFIG_FILE))
92    try:
93        master_name = presentation_dict['master_name']
94    except KeyError:
95        raise PerfUploadingError(
96                'No master name is specified for test %s in %s.' %
97                (test_name, _PRESENTATION_CONFIG_FILE))
98    if 'dashboard_test_name' in presentation_dict:
99        test_name = presentation_dict['dashboard_test_name']
100    return {'master_name': master_name, 'test_name': test_name}
101
102
103def _format_for_upload(platform_name, cros_version, chrome_version,
104                       hardware_id, variant_name, hardware_hostname,
105                       perf_data, presentation_info, jobname):
106    """Formats perf data suitable to upload to the perf dashboard.
107
108    The perf dashboard expects perf data to be uploaded as a
109    specially-formatted JSON string.  In particular, the JSON object must be a
110    dictionary with key "data", and value being a list of dictionaries where
111    each dictionary contains all the information associated with a single
112    measured perf value: master name, bot name, test name, perf value, error
113    value, units, and build version numbers.
114
115    @param platform_name: The string name of the platform.
116    @param cros_version: The string chromeOS version number.
117    @param chrome_version: The string chrome version number.
118    @param hardware_id: String that identifies the type of hardware the test was
119            executed on.
120    @param variant_name: String that identifies the variant name of the board.
121    @param hardware_hostname: String that identifies the name of the device the
122            test was executed on.
123    @param perf_data: A dictionary of measured perf data as computed by
124            _compute_avg_stddev().
125    @param presentation_info: A dictionary of dashboard presentation info for
126            the given test, as identified by _gather_presentation_info().
127    @param jobname: A string uniquely identifying the test run, this enables
128            linking back from a test result to the logs of the test run.
129
130    @return A dictionary containing the formatted information ready to upload
131        to the performance dashboard.
132
133    """
134    if variant_name:
135        platform_name += '-' + variant_name
136
137    perf_values = perf_data
138    # Client side case - server side comes with its own charts data section.
139    if 'charts' not in perf_values:
140        perf_values = {
141          'format_version': '1.0',
142          'benchmark_name': presentation_info['test_name'],
143          'charts': perf_data,
144        }
145
146    dash_entry = {
147        'master': presentation_info['master_name'],
148        'bot': 'cros-' + platform_name,  # Prefix to clarify it's ChromeOS.
149        'point_id': _get_id_from_version(chrome_version, cros_version),
150        'versions': {
151            'cros_version': cros_version,
152            'chrome_version': chrome_version,
153        },
154        'supplemental': {
155            'default_rev': 'r_cros_version',
156            'hardware_identifier': hardware_id,
157            'hardware_hostname': hardware_hostname,
158            'variant_name': variant_name,
159            'jobname': jobname,
160        },
161        'chart_data': perf_values,
162    }
163    return {'data': json.dumps(dash_entry)}
164
165
166def _get_version_numbers(test_attributes):
167    """Gets the version numbers from the test attributes and validates them.
168
169    @param test_attributes: The attributes property (which is a dict) of an
170        autotest tko.models.test object.
171
172    @return A pair of strings (Chrome OS version, Chrome version).
173
174    @raises PerfUploadingError if a version isn't formatted as expected.
175    """
176    chrome_version = test_attributes.get('CHROME_VERSION', '')
177    cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '')
178    cros_milestone = test_attributes.get('CHROMEOS_RELEASE_CHROME_MILESTONE')
179    # Use the release milestone as the milestone if present, othewise prefix the
180    # cros version with the with the Chrome browser milestone.
181    if cros_milestone:
182      cros_version = "%s.%s" % (cros_milestone, cros_version)
183    else:
184      cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version
185    if not re.match(VERSION_REGEXP, cros_version):
186        raise PerfUploadingError('CrOS version "%s" does not match expected '
187                                 'format.' % cros_version)
188    if not re.match(VERSION_REGEXP, chrome_version):
189        raise PerfUploadingError('Chrome version "%s" does not match expected '
190                                 'format.' % chrome_version)
191    return (cros_version, chrome_version)
192
193
194def _get_id_from_version(chrome_version, cros_version):
195    """Computes the point ID to use, from Chrome and ChromeOS version numbers.
196
197    For ChromeOS row data, data values are associated with both a Chrome
198    version number and a ChromeOS version number (unlike for Chrome row data
199    that is associated with a single revision number).  This function takes
200    both version numbers as input, then computes a single, unique integer ID
201    from them, which serves as a 'fake' revision number that can uniquely
202    identify each ChromeOS data point, and which will allow ChromeOS data points
203    to be sorted by Chrome version number, with ties broken by ChromeOS version
204    number.
205
206    To compute the integer ID, we take the portions of each version number that
207    serve as the shortest unambiguous names for each (as described here:
208    http://www.chromium.org/developers/version-numbers).  We then force each
209    component of each portion to be a fixed width (padded by zeros if needed),
210    concatenate all digits together (with those coming from the Chrome version
211    number first), and convert the entire string of digits into an integer.
212    We ensure that the total number of digits does not exceed that which is
213    allowed by AppEngine NDB for an integer (64-bit signed value).
214
215    For example:
216      Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2)
217      ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0)
218      concatenated together with padding for fixed-width columns:
219          ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000'
220      Final integer ID: 14520020390600000
221
222    @param chrome_ver: The Chrome version number as a string.
223    @param cros_ver: The ChromeOS version number as a string.
224
225    @return A unique integer ID associated with the two given version numbers.
226
227    """
228
229    # Number of digits to use from each part of the version string for Chrome
230    # and Chrome OS versions when building a point ID out of these two versions.
231    chrome_version_col_widths = [0, 0, 5, 3]
232    cros_version_col_widths = [0, 5, 3, 2]
233
234    def get_digits_from_version(version_num, column_widths):
235        if re.match(VERSION_REGEXP, version_num):
236            computed_string = ''
237            version_parts = version_num.split('.')
238            for i, version_part in enumerate(version_parts):
239                if column_widths[i]:
240                    computed_string += version_part.zfill(column_widths[i])
241            return computed_string
242        else:
243            return None
244
245    chrome_digits = get_digits_from_version(
246            chrome_version, chrome_version_col_widths)
247    cros_digits = get_digits_from_version(
248            cros_version, cros_version_col_widths)
249    if not chrome_digits or not cros_digits:
250        return None
251    result_digits = chrome_digits + cros_digits
252    max_digits = sum(chrome_version_col_widths + cros_version_col_widths)
253    if len(result_digits) > max_digits:
254        return None
255    return int(result_digits)
256
257
258def _send_to_dashboard(data_obj):
259    """Sends formatted perf data to the perf dashboard.
260
261    @param data_obj: A formatted data object as returned by
262        _format_for_upload().
263
264    @raises PerfUploadingError if an exception was raised when uploading.
265
266    """
267    encoded = urllib.urlencode(data_obj)
268    req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded)
269    try:
270        urllib2.urlopen(req)
271    except urllib2.HTTPError as e:
272        raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % (
273                e.code, e.msg, data_obj['data']))
274    except urllib2.URLError as e:
275        raise PerfUploadingError(
276                'URLError: %s for JSON %s\n' %
277                (str(e.reason), data_obj['data']))
278    except httplib.HTTPException:
279        raise PerfUploadingError(
280                'HTTPException for JSON %s\n' % data_obj['data'])
281
282
283def upload_test(job, test, jobname):
284    """Uploads any perf data associated with a test to the perf dashboard.
285
286    @param job: An autotest tko.models.job object that is associated with the
287        given |test|.
288    @param test: An autotest tko.models.test object that may or may not be
289        associated with measured perf data.
290    @param jobname: A string uniquely identifying the test run, this enables
291            linking back from a test result to the logs of the test run.
292
293    """
294
295    # Format the perf data for the upload, then upload it.
296    test_name = test.testname
297    platform_name = job.machine_group
298    # Append the platform name with '.arc' if the suffix of the control
299    # filename is '.arc'.
300    if job.label and re.match('.*\.arc$', job.label):
301        platform_name += '.arc'
302    hardware_id = test.attributes.get('hwid', '')
303    hardware_hostname = test.machine
304    variant_name = test.attributes.get(constants.VARIANT_KEY, None)
305    config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE)
306    try:
307        shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE)
308        config_data.update(shadow_config_data)
309    except ValueError as e:
310        tko_utils.dprint('Failed to parse config file %s: %s.' %
311                         (_PRESENTATION_SHADOW_CONFIG_FILE, e))
312    try:
313        cros_version, chrome_version = _get_version_numbers(test.attributes)
314        presentation_info = _gather_presentation_info(config_data, test_name)
315        formatted_data = _format_for_upload(
316                platform_name, cros_version, chrome_version, hardware_id,
317                variant_name, hardware_hostname, test.perf_values,
318                presentation_info, jobname)
319        _send_to_dashboard(formatted_data)
320    except PerfUploadingError as e:
321        tko_utils.dprint('Error when uploading perf data to the perf '
322                         'dashboard for test %s: %s' % (test_name, e))
323    else:
324        tko_utils.dprint('Successfully uploaded perf data to the perf '
325                         'dashboard for test %s.' % test_name)
326
327