1#!/usr/bin/env python
2#
3# Copyright 2019, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Unittests for event_handler."""
18
19import unittest
20import mock
21
22import atest_tf_test_runner as atf_tr
23import event_handler as e_h
24from test_runners import test_runner_base
25
26
27EVENTS_NORMAL = [
28    ('TEST_MODULE_STARTED', {
29        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
30        'moduleName':'someTestModule'}),
31    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
32    ('TEST_STARTED', {'start_time':52, 'className':'someClassName',
33                      'testName':'someTestName'}),
34    ('TEST_ENDED', {'end_time':1048, 'className':'someClassName',
35                    'testName':'someTestName'}),
36    ('TEST_STARTED', {'start_time':48, 'className':'someClassName2',
37                      'testName':'someTestName2'}),
38    ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2',
39                     'trace': 'someTrace'}),
40    ('TEST_ENDED', {'end_time':9876450, 'className':'someClassName2',
41                    'testName':'someTestName2'}),
42    ('TEST_RUN_ENDED', {}),
43    ('TEST_MODULE_ENDED', {'foo': 'bar'}),
44]
45
46EVENTS_RUN_FAILURE = [
47    ('TEST_MODULE_STARTED', {
48        'moduleContextFileName': 'serial-util11462169742772610436.ser',
49        'moduleName': 'someTestModule'}),
50    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
51    ('TEST_STARTED', {'start_time':10, 'className': 'someClassName',
52                      'testName':'someTestName'}),
53    ('TEST_RUN_FAILED', {'reason': 'someRunFailureReason'})
54]
55
56
57EVENTS_INVOCATION_FAILURE = [
58    ('TEST_RUN_STARTED', {'testCount': None, 'runName': 'com.android.UnitTests'}),
59    ('INVOCATION_FAILED', {'cause': 'someInvocationFailureReason'})
60]
61
62EVENTS_MISSING_TEST_RUN_STARTED_EVENT = [
63    ('TEST_STARTED', {'start_time':52, 'className':'someClassName',
64                      'testName':'someTestName'}),
65    ('TEST_ENDED', {'end_time':1048, 'className':'someClassName',
66                    'testName':'someTestName'}),
67]
68
69EVENTS_NOT_BALANCED_BEFORE_RAISE = [
70    ('TEST_MODULE_STARTED', {
71        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
72        'moduleName':'someTestModule'}),
73    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
74    ('TEST_STARTED', {'start_time':10, 'className':'someClassName',
75                      'testName':'someTestName'}),
76    ('TEST_ENDED', {'end_time':18, 'className':'someClassName',
77                    'testName':'someTestName'}),
78    ('TEST_STARTED', {'start_time':19, 'className':'someClassName',
79                      'testName':'someTestName'}),
80    ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2',
81                     'trace': 'someTrace'}),
82]
83
84EVENTS_IGNORE = [
85    ('TEST_MODULE_STARTED', {
86        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
87        'moduleName':'someTestModule'}),
88    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
89    ('TEST_STARTED', {'start_time':8, 'className':'someClassName',
90                      'testName':'someTestName'}),
91    ('TEST_ENDED', {'end_time':18, 'className':'someClassName',
92                    'testName':'someTestName'}),
93    ('TEST_STARTED', {'start_time':28, 'className':'someClassName2',
94                      'testName':'someTestName2'}),
95    ('TEST_IGNORED', {'className':'someClassName2', 'testName':'someTestName2',
96                      'trace': 'someTrace'}),
97    ('TEST_ENDED', {'end_time':90, 'className':'someClassName2',
98                    'testName':'someTestName2'}),
99    ('TEST_RUN_ENDED', {}),
100    ('TEST_MODULE_ENDED', {'foo': 'bar'}),
101]
102
103EVENTS_WITH_PERF_INFO = [
104    ('TEST_MODULE_STARTED', {
105        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
106        'moduleName':'someTestModule'}),
107    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
108    ('TEST_STARTED', {'start_time':52, 'className':'someClassName',
109                      'testName':'someTestName'}),
110    ('TEST_ENDED', {'end_time':1048, 'className':'someClassName',
111                    'testName':'someTestName'}),
112    ('TEST_STARTED', {'start_time':48, 'className':'someClassName2',
113                      'testName':'someTestName2'}),
114    ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2',
115                     'trace': 'someTrace'}),
116    ('TEST_ENDED', {'end_time':9876450, 'className':'someClassName2',
117                    'testName':'someTestName2', 'cpu_time':'1234.1234(ns)',
118                    'real_time':'5678.5678(ns)', 'iterations':'6666'}),
119    ('TEST_STARTED', {'start_time':10, 'className':'someClassName3',
120                      'testName':'someTestName3'}),
121    ('TEST_ENDED', {'end_time':70, 'className':'someClassName3',
122                    'testName':'someTestName3', 'additional_info_min':'102773',
123                    'additional_info_mean':'105973', 'additional_info_median':'103778'}),
124    ('TEST_RUN_ENDED', {}),
125    ('TEST_MODULE_ENDED', {'foo': 'bar'}),
126]
127
128class EventHandlerUnittests(unittest.TestCase):
129    """Unit tests for event_handler.py"""
130
131    def setUp(self):
132        reload(e_h)
133        self.mock_reporter = mock.Mock()
134        self.fake_eh = e_h.EventHandler(self.mock_reporter,
135                                        atf_tr.AtestTradefedTestRunner.NAME)
136
137    def tearDown(self):
138        mock.patch.stopall()
139
140    def test_process_event_normal_results(self):
141        """Test process_event method for normal test results."""
142        for name, data in EVENTS_NORMAL:
143            self.fake_eh.process_event(name, data)
144        call1 = mock.call(test_runner_base.TestResult(
145            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
146            group_name='someTestModule',
147            test_name='someClassName#someTestName',
148            status=test_runner_base.PASSED_STATUS,
149            details=None,
150            test_count=1,
151            test_time='(996ms)',
152            runner_total=None,
153            group_total=2,
154            additional_info={},
155            test_run_name='com.android.UnitTests'
156        ))
157        call2 = mock.call(test_runner_base.TestResult(
158            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
159            group_name='someTestModule',
160            test_name='someClassName2#someTestName2',
161            status=test_runner_base.FAILED_STATUS,
162            details='someTrace',
163            test_count=2,
164            test_time='(2h44m36.402s)',
165            runner_total=None,
166            group_total=2,
167            additional_info={},
168            test_run_name='com.android.UnitTests'
169        ))
170        self.mock_reporter.process_test_result.assert_has_calls([call1, call2])
171
172    def test_process_event_run_failure(self):
173        """Test process_event method run failure."""
174        for name, data in EVENTS_RUN_FAILURE:
175            self.fake_eh.process_event(name, data)
176        call = mock.call(test_runner_base.TestResult(
177            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
178            group_name='someTestModule',
179            test_name='someClassName#someTestName',
180            status=test_runner_base.ERROR_STATUS,
181            details='someRunFailureReason',
182            test_count=1,
183            test_time='',
184            runner_total=None,
185            group_total=2,
186            additional_info={},
187            test_run_name='com.android.UnitTests'
188        ))
189        self.mock_reporter.process_test_result.assert_has_calls([call])
190
191    def test_process_event_invocation_failure(self):
192        """Test process_event method with invocation failure."""
193        for name, data in EVENTS_INVOCATION_FAILURE:
194            self.fake_eh.process_event(name, data)
195        call = mock.call(test_runner_base.TestResult(
196            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
197            group_name=None,
198            test_name=None,
199            status=test_runner_base.ERROR_STATUS,
200            details='someInvocationFailureReason',
201            test_count=0,
202            test_time='',
203            runner_total=None,
204            group_total=None,
205            additional_info={},
206            test_run_name='com.android.UnitTests'
207        ))
208        self.mock_reporter.process_test_result.assert_has_calls([call])
209
210    def test_process_event_missing_test_run_started_event(self):
211        """Test process_event method for normal test results."""
212        for name, data in EVENTS_MISSING_TEST_RUN_STARTED_EVENT:
213            self.fake_eh.process_event(name, data)
214        call = mock.call(test_runner_base.TestResult(
215            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
216            group_name=None,
217            test_name='someClassName#someTestName',
218            status=test_runner_base.PASSED_STATUS,
219            details=None,
220            test_count=1,
221            test_time='(996ms)',
222            runner_total=None,
223            group_total=None,
224            additional_info={},
225            test_run_name=None
226        ))
227        self.mock_reporter.process_test_result.assert_has_calls([call])
228
229    # pylint: disable=protected-access
230    def test_process_event_not_balanced(self):
231        """Test process_event method with start/end event name not balanced."""
232        for name, data in EVENTS_NOT_BALANCED_BEFORE_RAISE:
233            self.fake_eh.process_event(name, data)
234        call = mock.call(test_runner_base.TestResult(
235            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
236            group_name='someTestModule',
237            test_name='someClassName#someTestName',
238            status=test_runner_base.PASSED_STATUS,
239            details=None,
240            test_count=1,
241            test_time='(8ms)',
242            runner_total=None,
243            group_total=2,
244            additional_info={},
245            test_run_name='com.android.UnitTests'
246        ))
247        self.mock_reporter.process_test_result.assert_has_calls([call])
248        # Event pair: TEST_STARTED -> TEST_RUN_ENDED
249        # It should raise TradeFedExitError in _check_events_are_balanced()
250        name = 'TEST_RUN_ENDED'
251        data = {}
252        self.assertRaises(e_h.EventHandleError,
253                          self.fake_eh._check_events_are_balanced,
254                          name, self.mock_reporter)
255        # Event pair: TEST_RUN_STARTED -> TEST_MODULE_ENDED
256        # It should raise TradeFedExitError in _check_events_are_balanced()
257        name = 'TEST_MODULE_ENDED'
258        data = {'foo': 'bar'}
259        self.assertRaises(e_h.EventHandleError,
260                          self.fake_eh._check_events_are_balanced,
261                          name, self.mock_reporter)
262
263    def test_process_event_ignore(self):
264        """Test _process_event method for normal test results."""
265        for name, data in EVENTS_IGNORE:
266            self.fake_eh.process_event(name, data)
267        call1 = mock.call(test_runner_base.TestResult(
268            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
269            group_name='someTestModule',
270            test_name='someClassName#someTestName',
271            status=test_runner_base.PASSED_STATUS,
272            details=None,
273            test_count=1,
274            test_time='(10ms)',
275            runner_total=None,
276            group_total=2,
277            additional_info={},
278            test_run_name='com.android.UnitTests'
279        ))
280        call2 = mock.call(test_runner_base.TestResult(
281            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
282            group_name='someTestModule',
283            test_name='someClassName2#someTestName2',
284            status=test_runner_base.IGNORED_STATUS,
285            details=None,
286            test_count=2,
287            test_time='(62ms)',
288            runner_total=None,
289            group_total=2,
290            additional_info={},
291            test_run_name='com.android.UnitTests'
292        ))
293        self.mock_reporter.process_test_result.assert_has_calls([call1, call2])
294
295    def test_process_event_with_additional_info(self):
296        """Test process_event method with perf information."""
297        for name, data in EVENTS_WITH_PERF_INFO:
298            self.fake_eh.process_event(name, data)
299        call1 = mock.call(test_runner_base.TestResult(
300            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
301            group_name='someTestModule',
302            test_name='someClassName#someTestName',
303            status=test_runner_base.PASSED_STATUS,
304            details=None,
305            test_count=1,
306            test_time='(996ms)',
307            runner_total=None,
308            group_total=2,
309            additional_info={},
310            test_run_name='com.android.UnitTests'
311        ))
312
313        test_additional_info = {'cpu_time':'1234.1234(ns)', 'real_time':'5678.5678(ns)',
314                                'iterations':'6666'}
315        call2 = mock.call(test_runner_base.TestResult(
316            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
317            group_name='someTestModule',
318            test_name='someClassName2#someTestName2',
319            status=test_runner_base.FAILED_STATUS,
320            details='someTrace',
321            test_count=2,
322            test_time='(2h44m36.402s)',
323            runner_total=None,
324            group_total=2,
325            additional_info=test_additional_info,
326            test_run_name='com.android.UnitTests'
327        ))
328
329        test_additional_info2 = {'additional_info_min':'102773',
330                                 'additional_info_mean':'105973',
331                                 'additional_info_median':'103778'}
332        call3 = mock.call(test_runner_base.TestResult(
333            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
334            group_name='someTestModule',
335            test_name='someClassName3#someTestName3',
336            status=test_runner_base.PASSED_STATUS,
337            details=None,
338            test_count=3,
339            test_time='(60ms)',
340            runner_total=None,
341            group_total=2,
342            additional_info=test_additional_info2,
343            test_run_name='com.android.UnitTests'
344        ))
345        self.mock_reporter.process_test_result.assert_has_calls([call1, call2, call3])
346
347if __name__ == '__main__':
348    unittest.main()
349