1import copy
2import json
3import logging
4import re
5
6import common
7
8from autotest_lib.client.common_lib import autotemp
9from autotest_lib.client.common_lib import global_config
10
11
12# Try importing the essential bug reporting libraries. Chromite and gdata_lib
13# are useless unless they can import gdata too.
14try:
15    __import__('chromite')
16    __import__('gdata')
17except ImportError, e:
18    fundamental_libs = False
19    logging.debug('Will not be able to generate link '
20                  'to the buildbot page when filing bugs. %s', e)
21else:
22    from chromite.lib import cros_build_lib, gs
23    fundamental_libs = True
24
25
26# Number of times to retry if a gs command fails. Defaults to 10,
27# which is far too long given that we already wait on these files
28# before starting HWTests.
29_GS_RETRIES = 1
30
31
32_HTTP_ERROR_THRESHOLD = 400
33BUG_CONFIG_SECTION = 'BUG_REPORTING'
34
35# global configurations needed for build artifacts
36_gs_domain = global_config.global_config.get_config_value(
37    BUG_CONFIG_SECTION, 'gs_domain', default='')
38_chromeos_image_archive = global_config.global_config.get_config_value(
39    BUG_CONFIG_SECTION, 'chromeos_image_archive', default='')
40_arg_prefix = global_config.global_config.get_config_value(
41    BUG_CONFIG_SECTION, 'arg_prefix', default='')
42
43
44# global configurations needed for results log
45_retrieve_logs_cgi = global_config.global_config.get_config_value(
46    BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='')
47_generic_results_bin = global_config.global_config.get_config_value(
48    BUG_CONFIG_SECTION, 'generic_results_bin', default='')
49_debug_dir = global_config.global_config.get_config_value(
50    BUG_CONFIG_SECTION, 'debug_dir', default='')
51
52
53# Template for the url used to generate the link to the job
54_job_view = global_config.global_config.get_config_value(
55    BUG_CONFIG_SECTION, 'job_view', default='')
56
57
58# gs prefix to perform file like operations (gs://)
59_gs_file_prefix = global_config.global_config.get_config_value(
60    BUG_CONFIG_SECTION, 'gs_file_prefix', default='')
61
62
63# global configurations needed for buildbot stages link
64_buildbot_builders = global_config.global_config.get_config_value(
65    BUG_CONFIG_SECTION, 'buildbot_builders', default='')
66_build_prefix = global_config.global_config.get_config_value(
67    BUG_CONFIG_SECTION, 'build_prefix', default='')
68
69
70WMATRIX_RETRY_URL = global_config.global_config.get_config_value(
71    BUG_CONFIG_SECTION, 'wmatrix_retry_url')
72
73
74class InvalidBugTemplateException(Exception):
75    """Exception raised when a bug template is not valid, e.g., missing value
76    for essential attributes.
77    """
78    pass
79
80
81class BugTemplate(object):
82    """Wrapper class to merge a suite and test bug templates, and do validation.
83    """
84
85    # Names of expected attributes.
86    EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title',
87                                        'cc', 'summary']
88    LIST_ATTRIBUTES = ['cc', 'labels']
89    EMAIL_ATTRIBUTES = ['owner', 'cc']
90
91    EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+')
92
93
94    def __init__(self, bug_template):
95        """Initialize a BugTemplate object.
96
97        @param bug_template: initial bug template, e.g., bug template from suite
98                             control file.
99        """
100        self.bug_template = self.cleanup_bug_template(bug_template)
101
102
103    @classmethod
104    def validate_bug_template(cls, bug_template):
105        """Verify if a bug template has value for all essential attributes.
106
107        @param bug_template: bug template to be verified.
108        @raise InvalidBugTemplateException: raised when a bug template
109                is invalid, e.g., has missing essential attribute, or any given
110                template is not a dictionary.
111        """
112        if not type(bug_template) is dict:
113            raise InvalidBugTemplateException('Bug template must be a '
114                                              'dictionary.')
115
116        unexpected_keys = []
117        for key, value in bug_template.iteritems():
118            if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES:
119                raise InvalidBugTemplateException('Key %s is not expected in '
120                                                  'bug template.' % key)
121            if (key in cls.LIST_ATTRIBUTES and
122                not isinstance(value, list)):
123                raise InvalidBugTemplateException('Value for %s must be a list.'
124                                                  % key)
125            if key in cls.EMAIL_ATTRIBUTES:
126                emails = value if isinstance(value, list) else [value]
127                for email in emails:
128                    if not email or not cls.EMAIL_REGEX.match(email):
129                        raise InvalidBugTemplateException(
130                                'Invalid email address: %s.' % email)
131
132
133    @classmethod
134    def cleanup_bug_template(cls, bug_template):
135        """Remove empty entries in given bug template.
136
137        @param bug_template: bug template to be verified.
138
139        @return: A cleaned up bug template.
140        @raise InvalidBugTemplateException: raised when a bug template
141                is not a dictionary.
142        """
143        if not type(bug_template) is dict:
144            raise InvalidBugTemplateException('Bug template must be a '
145                                              'dictionary.')
146        template = copy.deepcopy(bug_template)
147        # If owner or cc is set but the value is empty or None, remove it from
148        # the template.
149        for email_attribute in cls.EMAIL_ATTRIBUTES:
150            if email_attribute in template:
151                value = template[email_attribute]
152                if isinstance(value, list):
153                    template[email_attribute] = [email for email in value
154                                                 if email]
155                if not template[email_attribute]:
156                    del(template[email_attribute])
157        return template
158
159
160    def finalize_bug_template(self, test_template):
161        """Merge test and suite bug templates.
162
163        @param test_template: Bug template from test control file.
164        @return: Merged bug template.
165
166        @raise InvalidBugTemplateException: raised when the merged template is
167                invalid, e.g., has missing essential attribute, or any given
168                template is not a dictionary.
169        """
170        test_template = self.cleanup_bug_template(test_template)
171        self.validate_bug_template(self.bug_template)
172        self.validate_bug_template(test_template)
173
174        merged_template = test_template
175        merged_template.update((k, v) for k, v in self.bug_template.iteritems()
176                               if k not in merged_template)
177
178        # test_template wins for common keys, unless values are list that can be
179        # merged.
180        for key in set(merged_template.keys()).intersection(
181                                                    self.bug_template.keys()):
182            if (type(merged_template[key]) is list and
183                type(self.bug_template[key]) is list):
184                merged_template[key] = (merged_template[key] +
185                                        self.bug_template[key])
186            elif not merged_template[key]:
187                merged_template[key] = self.bug_template[key]
188        self.validate_bug_template(merged_template)
189        return merged_template
190
191
192def link_build_artifacts(build):
193    """Returns a url to build artifacts on google storage.
194
195    @param build: A string, e.g. stout32-release/R30-4433.0.0
196
197    @returns: A url to build artifacts on google storage.
198
199    """
200    return (_gs_domain + _arg_prefix +
201            _chromeos_image_archive + build)
202
203
204def link_job(job_id, instance_server=None):
205    """Returns an url to the job on cautotest.
206
207    @param job_id: A string, representing the job id.
208    @param instance_server: The instance server.
209        Eg: cautotest, cautotest-cq, localhost.
210
211    @returns: An url to the job on cautotest.
212
213    """
214    if not job_id:
215        return 'Job did not run, or was aborted prematurely'
216    if not instance_server:
217        instance_server = global_config.global_config.get_config_value(
218            'SERVER', 'hostname', default='localhost')
219    if 'cautotest' in instance_server:
220        instance_server += '.corp.google.com'
221    return _job_view % (instance_server, job_id)
222
223
224def _base_results_log(job_id, result_owner, hostname):
225    """Returns the base url of the job's results.
226
227    @param job_id: A string, representing the job id.
228    @param result_owner: A string, representing the onwer of the job.
229    @param hostname: A string, representing the host on which
230                     the job has run.
231
232    @returns: The base url of the job's results.
233
234    """
235    if job_id and result_owner and hostname:
236        path_to_object = '%s-%s/%s' % (job_id, result_owner,
237                                       hostname)
238        return (_retrieve_logs_cgi + _generic_results_bin +
239                path_to_object)
240
241
242def link_result_logs(job_id, result_owner, hostname):
243    """Returns a url to test logs on google storage.
244
245    @param job_id: A string, representing the job id.
246    @param result_owner: A string, representing the owner of the job.
247    @param hostname: A string, representing the host on which the
248                     jot has run.
249
250    @returns: A url to test logs on google storage.
251
252    """
253    base_results = _base_results_log(job_id, result_owner, hostname)
254    if base_results:
255        return '%s/%s' % (base_results, _debug_dir)
256    return ('Could not generate results log: the job with id %s, '
257            'scheduled by: %s on host: %s did not run' %
258            (job_id, result_owner, hostname))
259
260
261def link_status_log(job_id, result_owner, hostname):
262    """Returns an url to status log of the job.
263
264    @param job_id: A string, representing the job id.
265    @param result_owner: A string, representing the owner of the job.
266    @param hostname: A string, representing the host on which the
267                     jot has run.
268
269    @returns: A url to status log of the job.
270
271    """
272    base_results = _base_results_log(job_id, result_owner, hostname)
273    if base_results:
274        return '%s/%s' % (base_results, 'status.log')
275    return 'NA'
276
277
278def _get_metadata_dict(build):
279    """
280    Get a dictionary of metadata related to this failure.
281
282    Metadata.json is created in the HWTest Archiving stage, if this file
283    isn't found the call to Cat will timeout after the number of retries
284    specified in the GSContext object. If metadata.json exists we parse
285    a json string of it's contents into a dictionary, which we return.
286
287    @param build: A string, e.g. stout32-release/R30-4433.0.0
288
289    @returns: A dictionary with the contents of metadata.json.
290
291    """
292    if not fundamental_libs:
293        return
294    try:
295        tempdir = autotemp.tempdir()
296        gs_context = gs.GSContext(retries=_GS_RETRIES,
297                                  cache_dir=tempdir.name)
298        gs_cmd = '%s%s%s/metadata.json' % (_gs_file_prefix,
299                                           _chromeos_image_archive,
300                                           build)
301        return json.loads(gs_context.Cat(gs_cmd))
302    except (cros_build_lib.RunCommandError, gs.GSContextException) as e:
303        logging.debug(e)
304    finally:
305        tempdir.clean()
306
307
308def link_buildbot_stages(build):
309    """
310    Link to the buildbot page associated with this run of HWTests.
311
312    @param build: A string, e.g. stout32-release/R30-4433.0.0
313
314    @return: A link to the buildbot stages page, or 'NA' if we cannot glean
315             enough information from metadata.json (or it doesn't exist).
316    """
317    metadata = _get_metadata_dict(build)
318    if (metadata and
319        metadata.get('builder-name') and
320        metadata.get('build-number')):
321
322        return ('%s%s/builds/%s' %
323                    (_buildbot_builders,
324                     metadata.get('builder-name'),
325                     metadata.get('build-number'))).replace(' ', '%20')
326    return 'NA'
327
328
329def link_retry_url(test_name):
330    """Link to the wmatrix retry stats page for this test.
331
332    @param test_name: Test we want to search the retry stats page for.
333
334    @return: A link to the wmatrix retry stats dashboard for this test.
335    """
336    return WMATRIX_RETRY_URL % test_name