1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6import random
7import re
8
9import common
10
11from autotest_lib.client.common_lib import global_config
12
13
14_CONFIG = global_config.global_config
15
16
17def image_url_pattern():
18    """Returns image_url_pattern from global_config."""
19    return _CONFIG.get_config_value('CROS', 'image_url_pattern', type=str)
20
21
22def firmware_url_pattern():
23    """Returns firmware_url_pattern from global_config."""
24    return _CONFIG.get_config_value('CROS', 'firmware_url_pattern', type=str)
25
26
27def factory_image_url_pattern():
28    """Returns path to factory image after it's been staged."""
29    return _CONFIG.get_config_value('CROS', 'factory_image_url_pattern',
30                                    type=str)
31
32
33def sharding_factor():
34    """Returns sharding_factor from global_config."""
35    return _CONFIG.get_config_value('CROS', 'sharding_factor', type=int)
36
37
38def infrastructure_user():
39    """Returns infrastructure_user from global_config."""
40    return _CONFIG.get_config_value('CROS', 'infrastructure_user', type=str)
41
42
43def package_url_pattern():
44    """Returns package_url_pattern from global_config."""
45    return _CONFIG.get_config_value('CROS', 'package_url_pattern', type=str)
46
47
48def try_job_timeout_mins():
49    """Returns try_job_timeout_mins from global_config."""
50    return _CONFIG.get_config_value('SCHEDULER', 'try_job_timeout_mins',
51                                    type=int, default=4*60)
52
53
54def get_package_url(devserver_url, build):
55    """Returns the package url from the |devserver_url| and |build|.
56
57    @param devserver_url: a string specifying the host to contact e.g.
58        http://my_host:9090.
59    @param build: the build/image string to use e.g. mario-release/R19-123.0.1.
60    @return the url where you can find the packages for the build.
61    """
62    return package_url_pattern() % (devserver_url, build)
63
64
65def get_devserver_build_from_package_url(package_url):
66    """The inverse method of get_package_url.
67
68    @param package_url: a string specifying the package url.
69
70    @return tuple containing the devserver_url, build.
71    """
72    pattern = package_url_pattern()
73    re_pattern = pattern.replace('%s', '(\S+)')
74
75    devserver_build_tuple = re.search(re_pattern, package_url).groups()
76
77    # TODO(beeps): This is a temporary hack around the fact that all
78    # job_repo_urls in the database currently contain 'archive'. Remove
79    # when all hosts have been reimaged at least once. Ref: crbug.com/214373.
80    return (devserver_build_tuple[0],
81            devserver_build_tuple[1].replace('archive/', ''))
82
83
84def get_build_from_image(image):
85    """Get the build name from the image string.
86
87    @param image: A string of image, can be the build name or a url to the
88                  build, e.g.,
89                  http://devserver/update/alex-release/R27-3837.0.0
90
91    @return: Name of the build. Return None if fail to parse build name.
92    """
93    if not image.startswith('http://'):
94        return image
95    else:
96        match = re.match('.*/([^/]+/R\d+-[^/]+)', image)
97        if match:
98            return match.group(1)
99
100
101def get_random_best_host(afe, host_list, require_usable_hosts=True):
102    """
103    Randomly choose the 'best' host from host_list, using fresh status.
104
105    Hit the AFE to get latest status for the listed hosts.  Then apply
106    the following heuristic to pick the 'best' set:
107
108    Remove unusable hosts (not tools.is_usable()), then
109    'Ready' > 'Running, Cleaning, Verifying, etc'
110
111    If any 'Ready' hosts exist, return a random choice.  If not, randomly
112    choose from the next tier.  If there are none of those either, None.
113
114    @param afe: autotest front end that holds the hosts being managed.
115    @param host_list: an iterable of Host objects, per server/frontend.py
116    @param require_usable_hosts: only return hosts currently in a usable
117                                 state.
118    @return a Host object, or None if no appropriate host is found.
119    """
120    if not host_list:
121        return None
122    hostnames = [host.hostname for host in host_list]
123    updated_hosts = afe.get_hosts(hostnames=hostnames)
124    usable_hosts = [host for host in updated_hosts if is_usable(host)]
125    ready_hosts = [host for host in usable_hosts if host.status == 'Ready']
126    unusable_hosts = [h for h in updated_hosts if not is_usable(h)]
127    if ready_hosts:
128        return random.choice(ready_hosts)
129    if usable_hosts:
130        return random.choice(usable_hosts)
131    if not require_usable_hosts and unusable_hosts:
132        return random.choice(unusable_hosts)
133    return None
134
135
136def inject_vars(vars, control_file_in):
137    """
138    Inject the contents of |vars| into |control_file_in|.
139
140    @param vars: a dict to shoehorn into the provided control file string.
141    @param control_file_in: the contents of a control file to munge.
142    @return the modified control file string.
143    """
144    control_file = ''
145    for key, value in vars.iteritems():
146        # None gets injected as 'None' without this check; same for digits.
147        if isinstance(value, str):
148            control_file += "%s=%s\n" % (key, repr(value))
149        else:
150            control_file += "%s=%r\n" % (key, value)
151
152    args_dict_str = "%s=%s\n" % ('args_dict', repr(vars))
153    return control_file + args_dict_str + control_file_in
154
155
156def is_usable(host):
157    """
158    Given a host, determine if the host is usable right now.
159
160    @param host: Host instance (as in server/frontend.py)
161    @return True if host is alive and not incorrectly locked.  Else, False.
162    """
163    return alive(host) and not incorrectly_locked(host)
164
165
166def alive(host):
167    """
168    Given a host, determine if the host is alive.
169
170    @param host: Host instance (as in server/frontend.py)
171    @return True if host is not under, or in need of, repair.  Else, False.
172    """
173    return host.status not in ['Repair Failed', 'Repairing']
174
175
176def incorrectly_locked(host):
177    """
178    Given a host, determine if the host is locked by some user.
179
180    If the host is unlocked, or locked by the test infrastructure,
181    this will return False.  There is only one system user defined as part
182    of the test infrastructure and is listed in global_config.ini under the
183    [CROS] section in the 'infrastructure_user' field.
184
185    @param host: Host instance (as in server/frontend.py)
186    @return False if the host is not locked, or locked by the infra.
187            True if the host is locked by the infra user.
188    """
189    return (host.locked and host.locked_by != infrastructure_user())
190
191
192def _testname_to_keyval_key(testname):
193    """Make a test name acceptable as a keyval key.
194
195    @param  testname Test name that must be converted.
196    @return          A string with selected bad characters replaced
197                     with allowable characters.
198    """
199    # Characters for keys in autotest keyvals are restricted; in
200    # particular, '/' isn't allowed.  Alas, in the case of an
201    # aborted job, the test name will be a path that includes '/'
202    # characters.  We want to file bugs for aborted jobs, so we
203    # apply a transform here to avoid trouble.
204    return testname.replace('/', '_')
205
206
207_BUG_ID_KEYVAL = '-Bug_Id'
208_BUG_COUNT_KEYVAL = '-Bug_Count'
209
210
211def create_bug_keyvals(job_id, testname, bug_info):
212    """Create keyvals to record a bug filed against a test failure.
213
214    @param testname  Name of the test for which to record a bug.
215    @param bug_info  Pair with the id of the bug and the count of
216                     the number of times the bug has been seen.
217    @param job_id    The afe job id of job which the test is associated to.
218                     job_id will be a part of the key.
219    @return          Keyvals to be recorded for the given test.
220    """
221    testname = _testname_to_keyval_key(testname)
222    keyval_base = '%s_%s' % (job_id, testname) if job_id else testname
223    return {
224        keyval_base + _BUG_ID_KEYVAL: bug_info[0],
225        keyval_base + _BUG_COUNT_KEYVAL: bug_info[1]
226    }
227
228
229def get_test_failure_bug_info(keyvals, job_id, testname):
230    """Extract information about a bug filed against a test failure.
231
232    This method tries to extract bug_id and bug_count from the keyvals
233    of a suite. If for some reason it cannot retrieve the bug_id it will
234    return (None, None) and there will be no link to the bug filed. We will
235    instead link directly to the logs of the failed test.
236
237    If it cannot retrieve the bug_count, it will return (int(bug_id), None)
238    and this will result in a link to the bug filed, with an inline message
239    saying we weren't able to determine how many times the bug occured.
240
241    If it retrieved both the bug_id and bug_count, we return a tuple of 2
242    integers and link to the bug filed, as well as mention how many times
243    the bug has occured in the buildbot stages.
244
245    @param keyvals  Keyvals associated with a suite job.
246    @param job_id   The afe job id of the job that runs the test.
247    @param testname Name of a test from the suite.
248    @return         None if there is no bug info, or a pair with the
249                    id of the bug, and the count of the number of
250                    times the bug has been seen.
251    """
252    testname = _testname_to_keyval_key(testname)
253    keyval_base = '%s_%s' % (job_id, testname) if job_id else testname
254    bug_id = keyvals.get(keyval_base + _BUG_ID_KEYVAL)
255    if not bug_id:
256        return None, None
257    bug_id = int(bug_id)
258    bug_count = keyvals.get(keyval_base + _BUG_COUNT_KEYVAL)
259    bug_count = int(bug_count) if bug_count else None
260    return bug_id, bug_count
261
262
263def create_job_name(build, suite, test_name):
264    """Create the name of a test job based on given build, suite, and test_name.
265
266    @param build: name of the build, e.g., lumpy-release/R31-1234.0.0.
267    @param suite: name of the suite, e.g., bvt.
268    @param test_name: name of the test, e.g., dummy_Pass.
269    @return: the test job's name, e.g.,
270             lumpy-release/R31-1234.0.0/bvt/dummy_Pass.
271    """
272    return '/'.join([build, suite, test_name])
273
274
275def get_test_name(build, suite, test_job_name):
276    """Get the test name from test job name.
277
278    Name of test job may contain information like build and suite. This method
279    strips these information and return only the test name.
280
281    @param build: name of the build, e.g., lumpy-release/R31-1234.0.0.
282    @param suite: name of the suite, e.g., bvt.
283    @param test_job_name: name of the test job, e.g.,
284                          lumpy-release/R31-1234.0.0/bvt/dummy_Pass_SERVER_JOB.
285    @return: the test name, e.g., dummy_Pass_SERVER_JOB.
286    """
287    # Do not change this naming convention without updating
288    # site_utils.parse_job_name.
289    return test_job_name.replace('%s/%s/' % (build, suite), '')
290