1#!/usr/bin/env python3
2#
3# Copyright 2017 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from mobly import signals
18
19
20def test_info(predicate=None, **keyed_info):
21    """Adds info about test.
22
23    Extra info to include about the test. This info will be available in the
24    test output. Note that if a key is given multiple times it will be added
25    as a list of all values. If multiples of these are stacked their results
26    will be merged.
27
28    Example:
29        # This test will have a variable my_var
30        @test_info(my_var='THIS IS MY TEST')
31        def my_test(self):
32            return False
33
34    Args:
35        predicate: A func to call that if false will skip adding this test
36                   info. Function signature is bool(test_obj, args, kwargs)
37        **keyed_info: The key, value info to include in the extras for this
38                      test.
39    """
40
41    def test_info_decorator(func):
42        return _TestInfoDecoratorFunc(func, predicate, keyed_info)
43
44    return test_info_decorator
45
46
47def __select_last(test_signals, _):
48    return test_signals[-1]
49
50
51def repeated_test(num_passes, acceptable_failures=0, result_selector=__select_last):
52    """A decorator that runs a test case multiple times.
53
54    This decorator can be used to run a test multiple times and aggregate the
55    data into a single test result. By setting `result_selector`, the user can
56    access the returned result of each run, allowing them to average results,
57    return the median, or gather and return standard deviation values.
58
59    This decorator should be used on test cases, and should not be used on
60    static or class methods. The test case must take in an additional argument,
61    `attempt_number`, which returns the current attempt number, starting from
62    1.
63
64    Note that any TestSignal intended to abort or skip the test will take
65    abort or skip immediately.
66
67    Args:
68        num_passes: The number of times the test needs to pass to report the
69            test case as passing.
70        acceptable_failures: The number of failures accepted. If the failures
71            exceeds this number, the test will stop repeating. The maximum
72            number of runs is `num_passes + acceptable_failures`. If the test
73            does fail, result_selector will still be called.
74        result_selector: A lambda that takes in the list of TestSignals and
75            returns the test signal to report the test case as. Note that the
76            list also contains any uncaught exceptions from the test execution.
77    """
78
79    def decorator(func):
80        if not func.__name__.startswith('test_'):
81            raise ValueError('Tests must start with "test_".')
82
83        def test_wrapper(self):
84            num_failures = 0
85            num_seen_passes = 0
86            test_signals_received = []
87            for i in range(num_passes + acceptable_failures):
88                try:
89                    func(self, i + 1)
90                except (signals.TestFailure, signals.TestError, AssertionError) as signal:
91                    test_signals_received.append(signal)
92                    num_failures += 1
93                except signals.TestPass as signal:
94                    test_signals_received.append(signal)
95                    num_seen_passes += 1
96                except (signals.TestSignal, KeyboardInterrupt):
97                    raise
98                except Exception as signal:
99                    test_signals_received.append(signal)
100                    num_failures += 1
101                else:
102                    num_seen_passes += 1
103                    test_signals_received.append(
104                        signals.TestPass('Test iteration %s of %s passed without details.' % (i, func.__name__)))
105
106                if num_failures > acceptable_failures:
107                    break
108                elif num_seen_passes == num_passes:
109                    break
110                else:
111                    self.teardown_test()
112                    self.setup_test()
113
114            raise result_selector(test_signals_received, self)
115
116        return test_wrapper
117
118    return decorator
119
120
121def test_tracker_info(uuid, extra_environment_info=None, predicate=None):
122    """Decorator for adding test tracker info to tests results.
123
124    Will add test tracker info inside of Extras/test_tracker_info.
125
126    Example:
127        # This test will be linked to test tracker uuid abcd
128        @test_tracker_info(uuid='abcd')
129        def my_test(self):
130            return False
131
132    Args:
133        uuid: The uuid of the test case in test tracker.
134        extra_environment_info: Extra info about the test tracker environment.
135        predicate: A func that if false when called will ignore this info.
136    """
137    return test_info(test_tracker_uuid=uuid, test_tracker_environment_info=extra_environment_info, predicate=predicate)
138
139
140class _TestInfoDecoratorFunc(object):
141    """Object that acts as a function decorator test info."""
142
143    def __init__(self, func, predicate, keyed_info):
144        self.func = func
145        self.predicate = predicate
146        self.keyed_info = keyed_info
147        self.__name__ = func.__name__
148        self.__doc__ = func.__doc__
149        self.__module__ = func.__module__
150
151    def __get__(self, instance, owner):
152        """Called by Python to create a binding for an instance closure.
153
154        When called by Python this object will create a special binding for
155        that instance. That binding will know how to interact with this
156        specific decorator.
157        """
158        return _TestInfoBinding(self, instance)
159
160    def __call__(self, *args, **kwargs):
161        """
162        When called runs the underlying func and then attaches test info
163        to a signal.
164        """
165        cause = None
166        try:
167            result = self.func(*args, **kwargs)
168
169            if result or result is None:
170                new_signal = signals.TestPass('')
171            else:
172                new_signal = signals.TestFailure('')
173        except signals.TestSignal as signal:
174            new_signal = signal
175        except Exception as ex:
176            cause = ex
177            new_signal = signals.TestError(cause)
178
179        if new_signal.extras is None:
180            new_signal.extras = {}
181        if not isinstance(new_signal.extras, dict):
182            raise ValueError('test_info can only append to signal data that has a dict as the extra value.')
183
184        gathered_extras = self._gather_local_info(None, *args, **kwargs)
185        for k, v in gathered_extras.items():
186            if k not in new_signal.extras:
187                new_signal.extras[k] = v
188            else:
189                if not isinstance(new_signal.extras[k], list):
190                    new_signal.extras[k] = [new_signal.extras[k]]
191
192                new_signal.extras[k].insert(0, v)
193
194        raise new_signal from cause
195
196    def gather(self, *args, **kwargs):
197        """
198        Gathers the info from this decorator without invoking the underlying
199        function. This will also gather all child info if the underlying func
200        has that ability.
201
202        Returns: A dictionary of info.
203        """
204        if hasattr(self.func, 'gather'):
205            extras = self.func.gather(*args, **kwargs)
206        else:
207            extras = {}
208
209        self._gather_local_info(extras, *args, **kwargs)
210
211        return extras
212
213    def _gather_local_info(self, gather_into, *args, **kwargs):
214        """Gathers info from this decorator and ignores children.
215
216        Args:
217            gather_into: Gathers into a dictionary that already exists.
218
219        Returns: The dictionary with gathered info in it.
220        """
221        if gather_into is None:
222            extras = {}
223        else:
224            extras = gather_into
225        if not self.predicate or self.predicate(args, kwargs):
226            for k, v in self.keyed_info.items():
227                if v and k not in extras:
228                    extras[k] = v
229                elif v and k in extras:
230                    if not isinstance(extras[k], list):
231                        extras[k] = [extras[k]]
232                    extras[k].insert(0, v)
233
234        return extras
235
236
237class _TestInfoBinding(object):
238    """
239    When Python creates an instance of an object it creates a binding object
240    for each closure that contains what the instance variable should be when
241    called. This object is a similar binding for _TestInfoDecoratorFunc.
242    When Python tries to create a binding of a _TestInfoDecoratorFunc it
243    will return one of these objects to hold the instance for that closure.
244    """
245
246    def __init__(self, target, instance):
247        """
248        Args:
249            target: The target for creating a binding to.
250            instance: The instance to bind the target with.
251        """
252        self.target = target
253        self.instance = instance
254        self.__name__ = target.__name__
255
256    def __call__(self, *args, **kwargs):
257        """
258        When this object is called it will call the target with the bound
259        instance.
260        """
261        return self.target(self.instance, *args, **kwargs)
262
263    def gather(self, *args, **kwargs):
264        """
265        Will gather the target with the bound instance.
266        """
267        return self.target.gather(self.instance, *args, **kwargs)
268