1# Lint as: python2, python3
2from __future__ import absolute_import
3from __future__ import division
4from __future__ import print_function
5
6import copy
7import datetime
8import re
9import six
10
11import common
12
13from autotest_lib.client.common_lib import global_config
14from autotest_lib.frontend.afe import rpc_client_lib
15
16
17# Number of times to retry if a gs command fails. Defaults to 10,
18# which is far too long given that we already wait on these files
19# before starting HWTests.
20_GS_RETRIES = 1
21
22
23_HTTP_ERROR_THRESHOLD = 400
24BUG_CONFIG_SECTION = 'BUG_REPORTING'
25
26# global configurations needed for build artifacts
27_gs_domain = global_config.global_config.get_config_value(
28    BUG_CONFIG_SECTION, 'gs_domain', default='')
29_chromeos_image_archive = global_config.global_config.get_config_value(
30    BUG_CONFIG_SECTION, 'chromeos_image_archive', default='')
31_arg_prefix = global_config.global_config.get_config_value(
32    BUG_CONFIG_SECTION, 'arg_prefix', default='')
33
34
35# global configurations needed for results log
36_retrieve_logs_cgi = global_config.global_config.get_config_value(
37    BUG_CONFIG_SECTION, 'retrieve_logs_cgi', default='')
38_generic_results_bin = global_config.global_config.get_config_value(
39    BUG_CONFIG_SECTION, 'generic_results_bin', default='')
40_debug_dir = global_config.global_config.get_config_value(
41    BUG_CONFIG_SECTION, 'debug_dir', default='')
42
43
44# Template for the url used to generate the link to the job
45_job_view = global_config.global_config.get_config_value(
46    BUG_CONFIG_SECTION, 'job_view', default='')
47
48
49# gs prefix to perform file like operations (gs://)
50_gs_file_prefix = global_config.global_config.get_config_value(
51    BUG_CONFIG_SECTION, 'gs_file_prefix', default='')
52
53
54_CRBUG_URL = global_config.global_config.get_config_value(
55    BUG_CONFIG_SECTION, 'crbug_url')
56
57
58WMATRIX_RETRY_URL = global_config.global_config.get_config_value(
59    BUG_CONFIG_SECTION, 'wmatrix_retry_url', default='')
60WMATRIX_TEST_HISTORY_URL = global_config.global_config.get_config_value(
61    BUG_CONFIG_SECTION, 'wmatrix_test_history_url', default='')
62STAINLESS_RETRY_URL = global_config.global_config.get_config_value(
63    BUG_CONFIG_SECTION, 'stainless_retry_url', default='')
64STAINLESS_TEST_HISTORY_URL = global_config.global_config.get_config_value(
65    BUG_CONFIG_SECTION, 'stainless_test_history_url', default='')
66
67
68class InvalidBugTemplateException(Exception):
69    """Exception raised when a bug template is not valid, e.g., missing value
70    for essential attributes.
71    """
72    pass
73
74
75class BugTemplate(object):
76    """Wrapper class to merge a suite and test bug templates, and do validation.
77    """
78
79    # Names of expected attributes.
80    EXPECTED_BUG_TEMPLATE_ATTRIBUTES = ['owner', 'labels', 'status', 'title',
81                                        'cc', 'summary', 'components']
82    LIST_ATTRIBUTES = ['cc', 'labels']
83    EMAIL_ATTRIBUTES = ['owner', 'cc']
84
85    EMAIL_REGEX = re.compile(r'[^@]+@[^@]+\.[^@]+')
86
87
88    def __init__(self, bug_template):
89        """Initialize a BugTemplate object.
90
91        @param bug_template: initial bug template, e.g., bug template from suite
92                             control file.
93        """
94        self.bug_template = self.cleanup_bug_template(bug_template)
95
96
97    @classmethod
98    def validate_bug_template(cls, bug_template):
99        """Verify if a bug template has value for all essential attributes.
100
101        @param bug_template: bug template to be verified.
102        @raise InvalidBugTemplateException: raised when a bug template
103                is invalid, e.g., has missing essential attribute, or any given
104                template is not a dictionary.
105        """
106        if not type(bug_template) is dict:
107            raise InvalidBugTemplateException('Bug template must be a '
108                                              'dictionary.')
109
110        unexpected_keys = []
111        for key, value in six.iteritems(bug_template):
112            if not key in cls.EXPECTED_BUG_TEMPLATE_ATTRIBUTES:
113                raise InvalidBugTemplateException('Key %s is not expected in '
114                                                  'bug template.' % key)
115            if (key in cls.LIST_ATTRIBUTES and
116                not isinstance(value, list)):
117                raise InvalidBugTemplateException('Value for %s must be a list.'
118                                                  % key)
119            if key in cls.EMAIL_ATTRIBUTES:
120                emails = value if isinstance(value, list) else [value]
121                for email in emails:
122                    if not email or not cls.EMAIL_REGEX.match(email):
123                        raise InvalidBugTemplateException(
124                                'Invalid email address: %s.' % email)
125
126
127    @classmethod
128    def cleanup_bug_template(cls, bug_template):
129        """Remove empty entries in given bug template.
130
131        @param bug_template: bug template to be verified.
132
133        @return: A cleaned up bug template.
134        @raise InvalidBugTemplateException: raised when a bug template
135                is not a dictionary.
136        """
137        if not type(bug_template) is dict:
138            raise InvalidBugTemplateException('Bug template must be a '
139                                              'dictionary.')
140        template = copy.deepcopy(bug_template)
141        # If owner or cc is set but the value is empty or None, remove it from
142        # the template.
143        for email_attribute in cls.EMAIL_ATTRIBUTES:
144            if email_attribute in template:
145                value = template[email_attribute]
146                if isinstance(value, list):
147                    template[email_attribute] = [email for email in value
148                                                 if email]
149                if not template[email_attribute]:
150                    del(template[email_attribute])
151        return template
152
153
154    def finalize_bug_template(self, test_template):
155        """Merge test and suite bug templates.
156
157        @param test_template: Bug template from test control file.
158        @return: Merged bug template.
159
160        @raise InvalidBugTemplateException: raised when the merged template is
161                invalid, e.g., has missing essential attribute, or any given
162                template is not a dictionary.
163        """
164        test_template = self.cleanup_bug_template(test_template)
165        self.validate_bug_template(self.bug_template)
166        self.validate_bug_template(test_template)
167
168        merged_template = test_template
169        merged_template.update((k, v) for k, v in six.iteritems(self.bug_template)
170                               if k not in merged_template)
171
172        # test_template wins for common keys, unless values are list that can be
173        # merged.
174        for key in set(merged_template.keys()).intersection(
175                                                    list(self.bug_template.keys())):
176            if (type(merged_template[key]) is list and
177                type(self.bug_template[key]) is list):
178                merged_template[key] = (merged_template[key] +
179                                        self.bug_template[key])
180            elif not merged_template[key]:
181                merged_template[key] = self.bug_template[key]
182        self.validate_bug_template(merged_template)
183        return merged_template
184
185
186def link_build_artifacts(build):
187    """Returns a url to build artifacts on google storage.
188
189    @param build: A string, e.g. stout32-release/R30-4433.0.0
190
191    @returns: A url to build artifacts on google storage.
192
193    """
194    return (_gs_domain + _arg_prefix +
195            _chromeos_image_archive + build)
196
197
198def link_job(job_id, instance_server=None):
199    """Returns an url to the job on cautotest.
200
201    @param job_id: A string, representing the job id.
202    @param instance_server: The instance server.
203        Eg: cautotest, cautotest-cq, localhost.
204
205    @returns: An url to the job on cautotest.
206
207    """
208    if not job_id:
209        return 'Job did not run, or was aborted prematurely'
210    if not instance_server:
211        instance_server = global_config.global_config.get_config_value(
212            'SERVER', 'hostname', default='localhost')
213
214    instance_server = rpc_client_lib.add_protocol(instance_server)
215    return _job_view % (instance_server, job_id)
216
217
218def _base_results_log(job_id, result_owner, hostname):
219    """Returns the base url of the job's results.
220
221    @param job_id: A string, representing the job id.
222    @param result_owner: A string, representing the onwer of the job.
223    @param hostname: A string, representing the host on which
224                     the job has run.
225
226    @returns: The base url of the job's results.
227
228    """
229    if job_id and result_owner and hostname:
230        path_to_object = '%s-%s/%s' % (job_id, result_owner,
231                                       hostname)
232        return (_retrieve_logs_cgi + _generic_results_bin +
233                path_to_object)
234
235
236def link_result_logs(job_id, result_owner, hostname):
237    """Returns a url to test logs on google storage.
238
239    @param job_id: A string, representing the job id.
240    @param result_owner: A string, representing the owner of the job.
241    @param hostname: A string, representing the host on which the
242                     jot has run.
243
244    @returns: A url to test logs on google storage.
245
246    """
247    base_results = _base_results_log(job_id, result_owner, hostname)
248    if base_results:
249        return '%s/%s' % (base_results, _debug_dir)
250    return ('Could not generate results log: the job with id %s, '
251            'scheduled by: %s on host: %s did not run' %
252            (job_id, result_owner, hostname))
253
254
255def link_status_log(job_id, result_owner, hostname):
256    """Returns an url to status log of the job.
257
258    @param job_id: A string, representing the job id.
259    @param result_owner: A string, representing the owner of the job.
260    @param hostname: A string, representing the host on which the
261                     jot has run.
262
263    @returns: A url to status log of the job.
264
265    """
266    base_results = _base_results_log(job_id, result_owner, hostname)
267    if base_results:
268        return '%s/%s' % (base_results, 'status.log')
269    return 'NA'
270
271
272def link_wmatrix_retry_url(test_name):
273    """Link to the wmatrix retry stats page for this test.
274
275    @param test_name: Test we want to search the retry stats page for.
276
277    @return: A link to the wmatrix retry stats dashboard for this test.
278    """
279    return WMATRIX_RETRY_URL % test_name
280
281
282def link_retry_url(test_name):
283    """Link to the retry stats page for this test.
284
285    @param test_name: Test we want to search the retry stats page for.
286
287    @return: A link to the retry stats dashboard for this test.
288    """
289    if STAINLESS_RETRY_URL:
290        args_dict = {
291            'test_name_re': '^%s$' % re.escape(test_name),
292        }
293        return STAINLESS_RETRY_URL % args_dict
294    return WMATRIX_RETRY_URL % test_name
295
296
297def link_test_history(test_name):
298    """Link to the test history page for this test.
299
300    @param test_name: Test we want to search the test history for.
301
302    @return: A link to the test history page for this test.
303    """
304    date_format = '%Y-%m-%d'
305    now = datetime.datetime.utcnow()
306    last_date = now.strftime(date_format)
307    first_date = (now - datetime.timedelta(days=28)).strftime(date_format)
308    # Please note that stainless url doesn't work for tests whose test name is
309    # different from its job name. E.g. for moblab_RunSuite/control.dummyServer
310    #     Its job name (NAME in control file) is moblab_DummyServerSuite.
311    #     Its test name is moblab_RunSuite.
312    # Stainless use 'moblab_DummyServerSuite' as the test name, however,
313    # TKO uses 'moblab_RunSuite' as the test name.
314    return STAINLESS_TEST_HISTORY_URL % (
315        '^%s$' % re.escape(test_name), first_date, last_date)
316
317
318def link_crbug(bug_id):
319    """Generate a bug link for the given bug_id.
320
321    @param bug_id: The id of the bug.
322    @return: A link, eg: https://crbug.com/<bug_id>.
323    """
324    return _CRBUG_URL % (bug_id,)
325