1#!/usr/bin/env python3.4
2#
3# Copyright 2016 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""This module is where all the record definitions and record containers live.
17"""
18
19import json
20import logging
21import pprint
22
23from acts import logger
24from acts import signals
25from acts import utils
26
27
28class TestResultEnums(object):
29    """Enums used for TestResultRecord class.
30
31    Includes the tokens to mark test result with, and the string names for each
32    field in TestResultRecord.
33    """
34
35    RECORD_NAME = "Test Name"
36    RECORD_CLASS = "Test Class"
37    RECORD_BEGIN_TIME = "Begin Time"
38    RECORD_END_TIME = "End Time"
39    RECORD_LOG_BEGIN_TIME = "Log Begin Time"
40    RECORD_LOG_END_TIME = "Log End Time"
41    RECORD_RESULT = "Result"
42    RECORD_UID = "UID"
43    RECORD_EXTRAS = "Extras"
44    RECORD_ADDITIONAL_ERRORS = "Extra Errors"
45    RECORD_DETAILS = "Details"
46    TEST_RESULT_PASS = "PASS"
47    TEST_RESULT_FAIL = "FAIL"
48    TEST_RESULT_SKIP = "SKIP"
49    TEST_RESULT_BLOCKED = "BLOCKED"
50    TEST_RESULT_UNKNOWN = "UNKNOWN"
51
52
53class TestResultRecord(object):
54    """A record that holds the information of a test case execution.
55
56    Attributes:
57        test_name: A string representing the name of the test case.
58        begin_time: Epoch timestamp of when the test case started.
59        end_time: Epoch timestamp of when the test case ended.
60        self.uid: Unique identifier of a test case.
61        self.result: Test result, PASS/FAIL/SKIP.
62        self.extras: User defined extra information of the test result.
63        self.details: A string explaining the details of the test case.
64    """
65
66    def __init__(self, t_name, t_class=None):
67        self.test_name = t_name
68        self.test_class = t_class
69        self.begin_time = None
70        self.end_time = None
71        self.log_begin_time = None
72        self.log_end_time = None
73        self.uid = None
74        self.result = None
75        self.extras = None
76        self.details = None
77        self.additional_errors = {}
78
79    def test_begin(self):
80        """Call this when the test case it records begins execution.
81
82        Sets the begin_time of this record.
83        """
84        self.begin_time = utils.get_current_epoch_time()
85        self.log_begin_time = logger.epoch_to_log_line_timestamp(
86            self.begin_time)
87
88    def _test_end(self, result, e):
89        """Class internal function to signal the end of a test case execution.
90
91        Args:
92            result: One of the TEST_RESULT enums in TestResultEnums.
93            e: A test termination signal (usually an exception object). It can
94                be any exception instance or of any subclass of
95                acts.signals.TestSignal.
96        """
97        self.end_time = utils.get_current_epoch_time()
98        self.log_end_time = logger.epoch_to_log_line_timestamp(self.end_time)
99        self.result = result
100        if self.additional_errors:
101            self.result = TestResultEnums.TEST_RESULT_UNKNOWN
102        if isinstance(e, signals.TestSignal):
103            self.details = e.details
104            self.extras = e.extras
105        elif e:
106            self.details = str(e)
107
108    def test_pass(self, e=None):
109        """To mark the test as passed in this record.
110
111        Args:
112            e: An instance of acts.signals.TestPass.
113        """
114        self._test_end(TestResultEnums.TEST_RESULT_PASS, e)
115
116    def test_fail(self, e=None):
117        """To mark the test as failed in this record.
118
119        Only test_fail does instance check because we want "assert xxx" to also
120        fail the test same way assert_true does.
121
122        Args:
123            e: An exception object. It can be an instance of AssertionError or
124                acts.base_test.TestFailure.
125        """
126        self._test_end(TestResultEnums.TEST_RESULT_FAIL, e)
127
128    def test_skip(self, e=None):
129        """To mark the test as skipped in this record.
130
131        Args:
132            e: An instance of acts.signals.TestSkip.
133        """
134        self._test_end(TestResultEnums.TEST_RESULT_SKIP, e)
135
136    def test_blocked(self, e=None):
137        """To mark the test as blocked in this record.
138
139        Args:
140            e: An instance of acts.signals.Test
141        """
142        self._test_end(TestResultEnums.TEST_RESULT_BLOCKED, e)
143
144    def test_unknown(self, e=None):
145        """To mark the test as unknown in this record.
146
147        Args:
148            e: An exception object.
149        """
150        self._test_end(TestResultEnums.TEST_RESULT_UNKNOWN, e)
151
152    def add_error(self, tag, e):
153        """Add extra error happened during a test mark the test result as
154        UNKNOWN.
155
156        If an error is added the test record, the record's result is equivalent
157        to the case where an uncaught exception happened.
158
159        Args:
160            tag: A string describing where this error came from, e.g. 'on_pass'.
161            e: An exception object.
162        """
163        self.result = TestResultEnums.TEST_RESULT_UNKNOWN
164        self.additional_errors[tag] = str(e)
165
166    def __str__(self):
167        d = self.to_dict()
168        l = ["%s = %s" % (k, v) for k, v in d.items()]
169        s = ', '.join(l)
170        return s
171
172    def __repr__(self):
173        """This returns a short string representation of the test record."""
174        t = utils.epoch_to_human_time(self.begin_time)
175        return "%s %s %s" % (t, self.test_name, self.result)
176
177    def to_dict(self):
178        """Gets a dictionary representing the content of this class.
179
180        Returns:
181            A dictionary representing the content of this class.
182        """
183        d = {}
184        d[TestResultEnums.RECORD_NAME] = self.test_name
185        d[TestResultEnums.RECORD_CLASS] = self.test_class
186        d[TestResultEnums.RECORD_BEGIN_TIME] = self.begin_time
187        d[TestResultEnums.RECORD_END_TIME] = self.end_time
188        d[TestResultEnums.RECORD_LOG_BEGIN_TIME] = self.log_begin_time
189        d[TestResultEnums.RECORD_LOG_END_TIME] = self.log_end_time
190        d[TestResultEnums.RECORD_RESULT] = self.result
191        d[TestResultEnums.RECORD_UID] = self.uid
192        d[TestResultEnums.RECORD_EXTRAS] = self.extras
193        d[TestResultEnums.RECORD_DETAILS] = self.details
194        d[TestResultEnums.RECORD_ADDITIONAL_ERRORS] = self.additional_errors
195        return d
196
197    def json_str(self):
198        """Converts this test record to a string in json format.
199
200        Format of the json string is:
201            {
202                'Test Name': <test name>,
203                'Begin Time': <epoch timestamp>,
204                'Details': <details>,
205                ...
206            }
207
208        Returns:
209            A json-format string representing the test record.
210        """
211        return json.dumps(self.to_dict())
212
213
214class TestResult(object):
215    """A class that contains metrics of a test run.
216
217    This class is essentially a container of TestResultRecord objects.
218
219    Attributes:
220        self.requested: A list of strings, each is the name of a test requested
221            by user.
222        self.failed: A list of records for tests failed.
223        self.executed: A list of records for tests that were actually executed.
224        self.passed: A list of records for tests passed.
225        self.skipped: A list of records for tests skipped.
226        self.unknown: A list of records for tests with unknown result token.
227    """
228
229    def __init__(self):
230        self.requested = []
231        self.failed = []
232        self.executed = []
233        self.passed = []
234        self.skipped = []
235        self.blocked = []
236        self.unknown = []
237        self.controller_info = {}
238        self.post_run_data = {}
239        self.extras = {}
240
241    def __add__(self, r):
242        """Overrides '+' operator for TestResult class.
243
244        The add operator merges two TestResult objects by concatenating all of
245        their lists together.
246
247        Args:
248            r: another instance of TestResult to be added
249
250        Returns:
251            A TestResult instance that's the sum of two TestResult instances.
252        """
253        if not isinstance(r, TestResult):
254            raise TypeError("Operand %s of type %s is not a TestResult." %
255                            (r, type(r)))
256        sum_result = TestResult()
257        for name in sum_result.__dict__:
258            r_value = getattr(r, name)
259            l_value = getattr(self, name)
260            if isinstance(r_value, list):
261                setattr(sum_result, name, l_value + r_value)
262            elif isinstance(r_value, dict):
263                # '+' operator for TestResult is only valid when multiple
264                # TestResult objs were created in the same test run, which means
265                # the controller info would be the same across all of them.
266                # TODO(angli): have a better way to validate this situation.
267                setattr(sum_result, name, l_value)
268        return sum_result
269
270    def add_controller_info(self, name, info):
271        try:
272            json.dumps(info)
273        except TypeError:
274            logging.warning(("Controller info for %s is not JSON serializable!"
275                             " Coercing it to string.") % name)
276            self.controller_info[name] = str(info)
277            return
278        self.controller_info[name] = info
279
280    def set_extra_data(self, name, info):
281        try:
282            json.dumps(info)
283        except TypeError:
284            logging.warning("Controller info for %s is not JSON serializable! "
285                            "Coercing it to string." % name)
286            info = str(info)
287        self.extras[name] = info
288
289    def add_record(self, record):
290        """Adds a test record to test result.
291
292        A record is considered executed once it's added to the test result.
293
294        Args:
295            record: A test record object to add.
296        """
297        if record.result == TestResultEnums.TEST_RESULT_FAIL:
298            self.executed.append(record)
299            self.failed.append(record)
300        elif record.result == TestResultEnums.TEST_RESULT_SKIP:
301            self.skipped.append(record)
302        elif record.result == TestResultEnums.TEST_RESULT_PASS:
303            self.executed.append(record)
304            self.passed.append(record)
305        elif record.result == TestResultEnums.TEST_RESULT_BLOCKED:
306            self.blocked.append(record)
307        else:
308            self.executed.append(record)
309            self.unknown.append(record)
310
311    @property
312    def is_all_pass(self):
313        """True if no tests failed or threw errors, False otherwise."""
314        num_of_failures = (len(self.failed) +
315                           len(self.unknown) +
316                           len(self.blocked))
317        return num_of_failures == 0
318
319    def json_str(self):
320        """Converts this test result to a string in json format.
321
322        Format of the json string is:
323            {
324                "Results": [
325                    {<executed test record 1>},
326                    {<executed test record 2>},
327                    ...
328                ],
329                "Summary": <summary dict>
330            }
331
332        Returns:
333            A json-format string representing the test results.
334        """
335        d = {}
336        d["ControllerInfo"] = self.controller_info
337        d["Results"] = [record.to_dict() for record in self.executed]
338        d["Summary"] = self.summary_dict()
339        d["Extras"] = self.extras
340        json_str = json.dumps(d, indent=4, sort_keys=True)
341        return json_str
342
343    def summary_str(self):
344        """Gets a string that summarizes the stats of this test result.
345
346        The summary rovides the counts of how many test cases fall into each
347        category, like "Passed", "Failed" etc.
348
349        Format of the string is:
350            Requested <int>, Executed <int>, ...
351
352        Returns:
353            A summary string of this test result.
354        """
355        l = ["%s %s" % (k, v) for k, v in self.summary_dict().items()]
356        # Sort the list so the order is the same every time.
357        msg = ", ".join(sorted(l))
358        return msg
359
360    def summary_dict(self):
361        """Gets a dictionary that summarizes the stats of this test result.
362
363        The summary rovides the counts of how many test cases fall into each
364        category, like "Passed", "Failed" etc.
365
366        Returns:
367            A dictionary with the stats of this test result.
368        """
369        d = {}
370        d["ControllerInfo"] = self.controller_info
371        d["Requested"] = len(self.requested)
372        d["Executed"] = len(self.executed)
373        d["Passed"] = len(self.passed)
374        d["Failed"] = len(self.failed)
375        d["Skipped"] = len(self.skipped)
376        d["Blocked"] = len(self.blocked)
377        d["Unknown"] = len(self.unknown)
378        return d
379