1# Lint as: python2, python3
2# pylint: disable-msg=C0111
3# Copyright 2008 Google Inc. Released under the GPL v2
4
5from __future__ import absolute_import
6from __future__ import division
7from __future__ import print_function
8
9import ast
10import logging
11import textwrap
12import re
13import six
14
15from autotest_lib.client.common_lib import autotest_enum
16from autotest_lib.client.common_lib import global_config
17from autotest_lib.client.common_lib import priorities
18
19
20REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type'])
21OBSOLETE_VARS = set(['experimental'])
22
23CONTROL_TYPE = autotest_enum.AutotestEnum('Server', 'Client', start_value=1)
24CONTROL_TYPE_NAMES = autotest_enum.AutotestEnum(*CONTROL_TYPE.names,
25                                                string_values=True)
26
27_SUITE_ATTRIBUTE_PREFIX = 'suite:'
28
29CONFIG = global_config.global_config
30
31# Default maximum test result size in kB.
32DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value(
33        'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000)
34
35
36class ControlVariableException(Exception):
37    pass
38
39def _validate_control_file_fields(control_file_path, control_file_vars,
40                                  raise_warnings):
41    """Validate the given set of variables from a control file.
42
43    @param control_file_path: string path of the control file these were
44            loaded from.
45    @param control_file_vars: dict of variables set in a control file.
46    @param raise_warnings: True iff we should raise on invalid variables.
47
48    """
49    diff = REQUIRED_VARS - set(control_file_vars)
50    if diff:
51        warning = ('WARNING: Not all required control '
52                   'variables were specified in %s.  Please define '
53                   '%s.') % (control_file_path, ', '.join(diff))
54        if raise_warnings:
55            raise ControlVariableException(warning)
56        print(textwrap.wrap(warning, 80))
57
58    obsolete = OBSOLETE_VARS & set(control_file_vars)
59    if obsolete:
60        warning = ('WARNING: Obsolete variables were '
61                   'specified in %s.  Please remove '
62                   '%s.') % (control_file_path, ', '.join(obsolete))
63        if raise_warnings:
64            raise ControlVariableException(warning)
65        print(textwrap.wrap(warning, 80))
66
67
68class ControlData(object):
69    # Available TIME settings in control file, the list must be in lower case
70    # and in ascending order, test running faster comes first.
71    TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy']
72    TEST_TIME = autotest_enum.AutotestEnum(*TEST_TIME_LIST,
73                                           string_values=False)
74
75    @staticmethod
76    def get_test_time_index(time):
77        """
78        Get the order of estimated test time, based on the TIME setting in
79        Control file. Faster test gets a lower index number.
80        """
81        try:
82            return ControlData.TEST_TIME.get_value(time.lower())
83        except AttributeError:
84            # Raise exception if time value is not a valid TIME setting.
85            error_msg = '%s is not a valid TIME.' % time
86            logging.error(error_msg)
87            raise ControlVariableException(error_msg)
88
89
90    def __init__(self, vars, path, raise_warnings=False):
91        # Defaults
92        self.path = path
93        self.dependencies = set()
94        # TODO(jrbarnette): This should be removed once outside
95        # code that uses can be changed.
96        self.experimental = False
97        self.run_verify = True
98        self.sync_count = 1
99        self.test_parameters = set()
100        self.test_category = ''
101        self.test_class = ''
102        self.job_retries = 0
103        # Default to require server-side package. Unless require_ssp is
104        # explicitly set to False, server-side package will be used for the
105        # job.
106        self.require_ssp = None
107        self.attributes = set()
108        self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB
109        self.priority = priorities.Priority.DEFAULT
110        self.fast = False
111
112        _validate_control_file_fields(self.path, vars, raise_warnings)
113
114        for key, val in six.iteritems(vars):
115            try:
116                self.set_attr(key, val, raise_warnings)
117            except Exception as e:
118                if raise_warnings:
119                    raise
120                print('WARNING: %s; skipping' % e)
121
122        self._patch_up_suites_from_attributes()
123
124
125    @property
126    def suite_tag_parts(self):
127        """Return the part strings of the test's suite tag."""
128        if hasattr(self, 'suite'):
129            return [part.strip() for part in self.suite.split(',')]
130        else:
131            return []
132
133
134    def set_attr(self, attr, val, raise_warnings=False):
135        attr = attr.lower()
136        try:
137            set_fn = getattr(self, 'set_%s' % attr)
138            set_fn(val)
139        except AttributeError:
140            # This must not be a variable we care about
141            pass
142
143
144    def _patch_up_suites_from_attributes(self):
145        """Patch up the set of suites this test is part of.
146
147        Legacy builds will not have an appropriate ATTRIBUTES field set.
148        Take the union of suites specified via ATTRIBUTES and suites specified
149        via SUITE.
150
151        SUITE used to be its own variable, but now suites are taken only from
152        the attributes.
153
154        """
155
156        suite_names = set()
157        # Extract any suites we know ourselves to be in based on the SUITE
158        # line.  This line is deprecated, but control files in old builds will
159        # still have it.
160        if hasattr(self, 'suite'):
161            existing_suites = self.suite.split(',')
162            existing_suites = [name.strip() for name in existing_suites]
163            existing_suites = [name for name in existing_suites if name]
164            suite_names.update(existing_suites)
165
166        # Figure out if our attributes mention any suites.
167        for attribute in self.attributes:
168            if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX):
169                continue
170            suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):]
171            suite_names.add(suite_name)
172
173        # Rebuild the suite field if necessary.
174        if suite_names:
175            self.set_suite(','.join(sorted(list(suite_names))))
176
177
178    def _set_string(self, attr, val):
179        val = str(val)
180        setattr(self, attr, val)
181
182
183    def _set_option(self, attr, val, options):
184        val = str(val)
185        if val.lower() not in [x.lower() for x in options]:
186            raise ValueError("%s must be one of the following "
187                             "options: %s" % (attr,
188                             ', '.join(options)))
189        setattr(self, attr, val)
190
191
192    def _set_bool(self, attr, val):
193        val = str(val).lower()
194        if val == "false":
195            val = False
196        elif val == "true":
197            val = True
198        else:
199            msg = "%s must be either true or false" % attr
200            raise ValueError(msg)
201        setattr(self, attr, val)
202
203
204    def _set_int(self, attr, val, min=None, max=None):
205        val = int(val)
206        if min is not None and min > val:
207            raise ValueError("%s is %d, which is below the "
208                             "minimum of %d" % (attr, val, min))
209        if max is not None and max < val:
210            raise ValueError("%s is %d, which is above the "
211                             "maximum of %d" % (attr, val, max))
212        setattr(self, attr, val)
213
214
215    def _set_set(self, attr, val):
216        val = str(val)
217        items = [x.strip() for x in val.split(',') if x.strip()]
218        setattr(self, attr, set(items))
219
220
221    def set_author(self, val):
222        self._set_string('author', val)
223
224
225    def set_dependencies(self, val):
226        self._set_set('dependencies', val)
227
228
229    def set_doc(self, val):
230        self._set_string('doc', val)
231
232
233    def set_name(self, val):
234        self._set_string('name', val)
235
236
237    def set_run_verify(self, val):
238        self._set_bool('run_verify', val)
239
240
241    def set_sync_count(self, val):
242        self._set_int('sync_count', val, min=1)
243
244
245    def set_suite(self, val):
246        self._set_string('suite', val)
247
248
249    def set_time(self, val):
250        self._set_option('time', val, ControlData.TEST_TIME_LIST)
251
252
253    def set_test_class(self, val):
254        self._set_string('test_class', val.lower())
255
256
257    def set_test_category(self, val):
258        self._set_string('test_category', val.lower())
259
260
261    def set_test_type(self, val):
262        self._set_option('test_type', val, list(CONTROL_TYPE.names))
263
264
265    def set_test_parameters(self, val):
266        self._set_set('test_parameters', val)
267
268
269    def set_job_retries(self, val):
270        self._set_int('job_retries', val)
271
272
273    def set_bug_template(self, val):
274        if type(val) == dict:
275            setattr(self, 'bug_template', val)
276
277
278    def set_require_ssp(self, val):
279        self._set_bool('require_ssp', val)
280
281
282    def set_build(self, val):
283        self._set_string('build', val)
284
285
286    def set_builds(self, val):
287        if type(val) == dict:
288            setattr(self, 'builds', val)
289
290    def set_max_result_size_kb(self, val):
291        self._set_int('max_result_size_KB', val)
292
293    def set_priority(self, val):
294        self._set_int('priority', val)
295
296    def set_fast(self, val):
297        self._set_bool('fast', val)
298
299    def set_update_type(self, val):
300        self._set_string('update_type', val)
301
302    def set_source_release(self, val):
303        self._set_string('source_release', val)
304
305    def set_target_release(self, val):
306        self._set_string('target_release', val)
307
308    def set_target_payload_uri(self, val):
309        self._set_string('target_payload_uri', val)
310
311    def set_source_payload_uri(self, val):
312        self._set_string('source_payload_uri', val)
313
314    def set_source_archive_uri(self, val):
315        self._set_string('source_archive_uri', val)
316
317    def set_attributes(self, val):
318        self._set_set('attributes', val)
319
320
321def _extract_const(expr):
322    assert (expr.__class__ == ast.Str)
323    if six.PY2:
324        assert (expr.s.__class__ in (str, int, float, unicode))
325    else:
326        assert (expr.s.__class__ in (str, int, float))
327    return str(expr.s).strip()
328
329
330def _extract_dict(expr):
331    assert (expr.__class__ == ast.Dict)
332    assert (expr.keys.__class__ == list)
333    cf_dict = {}
334    for key, value in zip(expr.keys, expr.values):
335        try:
336            key = _extract_const(key)
337            val = _extract_expression(value)
338        except (AssertionError, ValueError):
339            pass
340        else:
341            cf_dict[key] = val
342    return cf_dict
343
344
345def _extract_list(expr):
346    assert (expr.__class__ == ast.List)
347    list_values = []
348    for value in expr.elts:
349        try:
350            list_values.append(_extract_expression(value))
351        except (AssertionError, ValueError):
352            pass
353    return list_values
354
355
356def _extract_name(expr):
357    assert (expr.__class__ == ast.Name)
358    assert (expr.id in ('False', 'True', 'None'))
359    return str(expr.id)
360
361
362def _extract_expression(expr):
363    if expr.__class__ == ast.Str:
364        return _extract_const(expr)
365    if expr.__class__ == ast.Name:
366        return _extract_name(expr)
367    if expr.__class__ == ast.Dict:
368        return _extract_dict(expr)
369    if expr.__class__ == ast.List:
370        return _extract_list(expr)
371    if expr.__class__ == ast.Num:
372        return expr.n
373    if six.PY3 and expr.__class__ == ast.NameConstant:
374        return expr.value
375    if six.PY3 and expr.__class__ == ast.Constant:
376        try:
377            return expr.value.strip()
378        except Exception:
379            return expr.value
380    raise ValueError('Unknown rval %s' % expr)
381
382
383def _extract_assignment(n):
384    assert (n.__class__ == ast.Assign)
385    assert (len(n.targets) == 1)
386    assert (n.targets[0].__class__ == ast.Name)
387    val = _extract_expression(n.value)
388    key = n.targets[0].id.lower()
389    return (key, val)
390
391
392def parse_control_string(control, raise_warnings=False, path=''):
393    """Parse a control file from a string.
394
395    @param control: string containing the text of a control file.
396    @param raise_warnings: True iff ControlData should raise an error on
397            warnings about control file contents.
398    @param path: string path to the control file.
399
400    """
401    try:
402        mod = ast.parse(control)
403    except SyntaxError as e:
404        logging.error('Syntax error (%s) while parsing control string:', e)
405        lines = control.split('\n')
406        for n, l in enumerate(lines):
407            logging.error('Line %d: %s', n + 1, l)
408        raise ControlVariableException("Error parsing data because %s" % e)
409    return finish_parse(mod, path, raise_warnings)
410
411
412def parse_control(path, raise_warnings=False):
413    try:
414        with open(path, 'r') as r:
415            mod = ast.parse(r.read())
416    except SyntaxError as e:
417        raise ControlVariableException("Error parsing %s because %s" %
418                                       (path, e))
419    return finish_parse(mod, path, raise_warnings)
420
421
422def _try_extract_assignment(node, variables):
423    """Try to extract assignment from the given node.
424
425    @param node: An Assign object.
426    @param variables: Dictionary to store the parsed assignments.
427    """
428    try:
429        key, val = _extract_assignment(node)
430        variables[key] = val
431    except (AssertionError, ValueError) as e:
432        pass
433
434
435def finish_parse(mod, path, raise_warnings):
436    assert (mod.__class__ == ast.Module)
437    assert (mod.body.__class__ == list)
438
439    variables = {}
440    injection_variables = {}
441    for n in mod.body:
442        if (n.__class__ == ast.FunctionDef and re.match('step\d+', n.name)):
443            vars_in_step = {}
444            for sub_node in n.body:
445                _try_extract_assignment(sub_node, vars_in_step)
446            if vars_in_step:
447                # Empty the vars collection so assignments from multiple steps
448                # won't be mixed.
449                variables.clear()
450                variables.update(vars_in_step)
451        else:
452            _try_extract_assignment(n, injection_variables)
453
454    variables.update(injection_variables)
455    return ControlData(variables, path, raise_warnings)
456