1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import os
5import unittest
6
7import tradefed_utils
8
9
10def _load_data(filename):
11    """Loads the test data of the given file name."""
12    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
13                           'tradefed_utils_unittest_data', filename), 'r') as f:
14        return f.read()
15
16
17class TradefedTestTest(unittest.TestCase):
18    """Unittest for tradefed_utils."""
19
20    def test_parse_tradefed_result(self):
21        """Test for parse_tradefed_result."""
22
23        waivers = set([
24            'android.app.cts.SystemFeaturesTest#testUsbAccessory',
25            'android.widget.cts.GridViewTest#testSetNumColumns',
26        ])
27
28        # b/35605415 and b/36520623
29        # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/
30        # CTS: Tradefed may split a module to multiple chunks.
31        # Besides, the module name may not end with "TestCases".
32        waived, _ = tradefed_utils.parse_tradefed_result(
33            _load_data('CtsHostsideNetworkTests.txt'),
34            waivers=waivers)
35        self.assertEquals(0, len(waived))
36
37        # b/35530394
38        # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/
39        # Crashed, but the automatic retry by tradefed executed the rest.
40        waived, _ = tradefed_utils.parse_tradefed_result(
41            _load_data('CtsMediaTestCases.txt'),
42            waivers=waivers)
43        self.assertEquals(0, len(waived))
44
45        # b/35530394
46        # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/
47        # Crashed in the middle, and the device didn't came back.
48        waived, _ = tradefed_utils.parse_tradefed_result(
49            _load_data('CtsSecurityTestCases.txt'),
50            waivers=waivers)
51        self.assertEquals(0, len(waived))
52
53        # b/36629187
54        # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/
55        # Crashed in the middle. Tradefed decided not to continue.
56        waived, _ = tradefed_utils.parse_tradefed_result(
57            _load_data('CtsViewTestCases.txt'),
58            waivers=waivers)
59        self.assertEquals(0, len(waived))
60
61        # b/36375690
62        # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/
63        # Mixture of real failures and waivers.
64        waived, _ = tradefed_utils.parse_tradefed_result(
65            _load_data('CtsAppTestCases.txt'),
66            waivers=waivers)
67        self.assertEquals(1, len(waived))
68        # ... and the retry of the above failing iteration.
69        waived, _ = tradefed_utils.parse_tradefed_result(
70            _load_data('CtsAppTestCases-retry.txt'),
71            waivers=waivers)
72        self.assertEquals(1, len(waived))
73
74        # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/
75        # When a test case crashed during teardown, tradefed prints the "fail"
76        # message twice. Tolerate it and still return an (inconsistent) count.
77        waived, _ = tradefed_utils.parse_tradefed_result(
78            _load_data('CtsWidgetTestCases.txt'),
79            waivers=waivers)
80        self.assertEquals(1, len(waived))
81
82        # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/
83        # When a test case unrecoverably crashed during teardown, tradefed
84        # prints the "fail" and failure summary message twice. Tolerate it.
85        waived, _ = tradefed_utils.parse_tradefed_result(
86            _load_data('CtsPrintTestCases.txt'),
87            waivers=waivers)
88        self.assertEquals(0, len(waived))
89
90        gts_waivers = set([
91            ('com.google.android.placement.gts.CoreGmsAppsTest#' +
92                'testCoreGmsAppsPreloaded'),
93            ('com.google.android.placement.gts.CoreGmsAppsTest#' +
94                'testGoogleDuoPreloaded'),
95            'com.google.android.placement.gts.UiPlacementTest#testPlayStore'
96        ])
97
98        # crbug.com/748116
99        # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/
100        # 3 ABIS: x86, x86_64, and armeabi-v7a
101        waived, _ = tradefed_utils.parse_tradefed_result(
102            _load_data('GtsPlacementTestCases.txt'),
103            waivers=gts_waivers)
104        self.assertEquals(9, len(waived))
105
106        # b/64095702
107        # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/
108        # The result of the last chunk not reported by tradefed.
109        # The actual dEQP log is too big, hence the test data here is trimmed.
110        waived, _ = tradefed_utils.parse_tradefed_result(
111            _load_data('CtsDeqpTestCases-trimmed.txt'),
112            waivers=waivers)
113        self.assertEquals(0, len(waived))
114
115        # b/80160772
116        # http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/
117        # The newer tradefed requires different parsing to count waivers.
118        waived, _ = tradefed_utils.parse_tradefed_result(
119            _load_data('CtsAppTestCases_P_simplified.txt'),
120            waivers=waivers)
121        self.assertEquals(1, len(waived))
122
123        # b/66899135, tradefed may reported inaccuratly with `list results`.
124        # Check if summary section shows that the result is inacurrate.
125        _, accurate = tradefed_utils.parse_tradefed_result(
126            _load_data('CtsAppTestCases_P_simplified.txt'),
127            waivers=waivers)
128        self.assertTrue(accurate)
129
130        _, accurate = tradefed_utils.parse_tradefed_result(
131            _load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'),
132            waivers=waivers)
133        self.assertFalse(accurate)
134
135    def test_get_test_result_xml_path(self):
136        path = tradefed_utils.get_test_result_xml_path(os.path.join(
137            os.path.dirname(os.path.realpath(__file__)),
138            'tradefed_utils_unittest_data', 'results'))
139        self.assertEqual(path, os.path.join(
140            os.path.dirname(os.path.realpath(__file__)),
141            'tradefed_utils_unittest_data', 'results', '2019.11.07_10.14.55',
142            'test_result.xml'))
143
144        # assertNoRaises
145        tradefed_utils.get_test_result_xml_path(os.path.join(
146            os.path.dirname(os.path.realpath(__file__)),
147            'tradefed_utils_unittest_data', 'not_exist'))
148
149    def test_parse_tradefed_testresults_xml_no_failure(self):
150        waived, accurate = tradefed_utils.parse_tradefed_testresults_xml(
151            os.path.join(os.path.dirname(os.path.realpath(__file__)),
152                         'tradefed_utils_unittest_data', 'test_result.xml'))
153        self.assertEquals(0, len(waived))
154        self.assertTrue(accurate)
155
156    def test_parse_tradefed_testresult_xml_waivers(self):
157        waived, accurate = tradefed_utils.parse_tradefed_testresults_xml(
158            os.path.join(os.path.dirname(os.path.realpath(__file__)),
159                         'tradefed_utils_unittest_data',
160                         'gtsplacement_test_result.xml'))
161        self.assertEquals(0, len(waived))
162
163        waivers = set([
164            'com.google.android.media.gts.WidevineDashPolicyTests#testL1RenewalDelay5S',
165            'com.google.android.media.gts.MediaDrmTest#testWidevineApi28',
166            'com.google.android.media.gts.WidevineGenericOpsTests#testL3',
167            'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay5S',
168            'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L3WithUHD30',
169            'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL3WithUHD30',
170            'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L1WithUHD30',
171            'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay13S',
172            'com.google.android.gts.backup.BackupHostTest#testGmsBackupTransportIsDefault',
173            'com.google.android.placement.gts.CoreGmsAppsTest#testGoogleDuoPreloaded',
174            'com.google.android.placement.gts.CoreGmsAppsTest#testCoreGmsAppsPreloaded',
175            'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL1WithUHD30'])
176        waived, accurate = tradefed_utils.parse_tradefed_testresults_xml(
177            os.path.join(os.path.dirname(os.path.realpath(__file__)),
178                         'tradefed_utils_unittest_data',
179                         'gtsplacement_test_result.xml'), waivers=waivers)
180        self.assertEquals(4, len(waived))
181
182    def test_get_perf_metrics_from_test_result_xml(self):
183        perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml(
184            os.path.join(os.path.dirname(os.path.realpath(__file__)),
185                         'tradefed_utils_unittest_data', 'test_result.xml'),
186            os.path.join('/', 'resultsdir'))
187        expected_result = [
188            {'units': 'ms',
189             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
190             'description': 'android.media.cts.AudioRecordTest'
191                            '#testAudioRecordLocalMono16Bit',
192             'value': '7.1688596491228065', 'higher_is_better': False},
193            {'units': 'ms',
194             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
195             'description': 'android.media.cts.AudioRecordTest'
196                            '#testAudioRecordLocalMono16BitShort',
197             'value': '2.5416666666666665', 'higher_is_better': False},
198            {'units': 'ms',
199             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
200             'description': 'android.media.cts.AudioRecordTest'
201                            '#testAudioRecordLocalNonblockingStereoFloat',
202             'value': '1.75', 'higher_is_better': False},
203            {'units': 'ms',
204             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
205             'description': 'android.media.cts.AudioRecordTest'
206                            '#testAudioRecordMonoFloat',
207             'value': '12.958881578947368', 'higher_is_better': False},
208            {'units': 'ms',
209             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
210             'description': 'android.media.cts.AudioRecordTest'
211                            '#testAudioRecordResamplerMono8Bit',
212             'value': '0.0', 'higher_is_better': False},
213            {'units': 'ms',
214             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
215             'description': 'android.media.cts.AudioRecordTest'
216                            '#testAudioRecordResamplerStereo8Bit',
217             'value': '3.5', 'higher_is_better': False},
218            {'units': 'ms',
219             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
220             'description': 'android.media.cts.AudioRecordTest'
221                            '#testAudioRecordStereo16Bit',
222             'value': '3.5', 'higher_is_better': False},
223            {'units': 'ms',
224             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
225             'description': 'android.media.cts.AudioTrackTest'
226                            '#testFastTimestamp',
227             'value': '0.1547618955373764', 'higher_is_better': False},
228            {'units': 'ms',
229             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
230             'description': 'android.media.cts.AudioTrackTest'
231                            '#testGetTimestamp',
232             'value': '0.1490119844675064', 'higher_is_better': False},
233            {'units': 'ms',
234             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
235             'description': 'android.media.cts.AudioTrack_ListenerTest'
236                            '#testAudioTrackCallback',
237             'value': '9.347127739984884', 'higher_is_better': False},
238            {'units': 'ms',
239             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
240             'description': 'android.media.cts.AudioTrack_ListenerTest'
241                            '#testAudioTrackCallbackWithHandler',
242             'value': '7.776177955844914', 'higher_is_better': False},
243            {'units': 'ms',
244             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
245             'description': 'android.media.cts.AudioTrack_ListenerTest'
246                            '#testStaticAudioTrackCallback',
247             'value': '7.776177955844914', 'higher_is_better': False},
248            {'units': 'ms',
249             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
250             'description': 'android.media.cts.AudioTrack_ListenerTest'
251                            '#testStaticAudioTrackCallbackWithHandler',
252             'value': '9.514361300075587', 'higher_is_better': False},
253            {'units': 'count',
254             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
255             'description': 'android.media.cts.DecoderTest'
256                            '#testH264ColorAspects',
257             'value': '1.0', 'higher_is_better': True},
258            {'units': 'count',
259             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
260             'description': 'android.media.cts.DecoderTest'
261                            '#testH265ColorAspects',
262             'value': '1.0', 'higher_is_better': True},
263            {'units': 'fps',
264             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
265             'description': 'android.media.cts.VideoDecoderPerfTest'
266                            '#testAvcGoog0Perf0320x0240',
267             'value': '580.1607045151507', 'higher_is_better': True},
268            {'units': 'fps',
269             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
270             'description': 'android.media.cts.VideoDecoderPerfTest'
271                            '#testAvcGoog0Perf0720x0480',
272             'value': '244.18184010611358', 'higher_is_better': True},
273            {'units': 'fps',
274             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
275             'description': 'android.media.cts.VideoDecoderPerfTest'
276                            '#testAvcGoog0Perf1280x0720',
277             'value': '70.96290491279275', 'higher_is_better': True},
278            {'units': 'fps',
279             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
280             'description': 'android.media.cts.VideoDecoderPerfTest'
281                            '#testAvcGoog0Perf1920x1080',
282             'value': '31.299613935451564', 'higher_is_better': True},
283            {'units': 'fps',
284             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
285             'description': 'android.media.cts.VideoDecoderPerfTest'
286                            '#testAvcOther0Perf0320x0240',
287             'value': '1079.6843075197307', 'higher_is_better': True},
288            {'units': 'fps',
289             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
290             'description': 'android.media.cts.VideoDecoderPerfTest'
291                            '#testAvcOther0Perf0720x0480',
292             'value': '873.7785366761784', 'higher_is_better': True},
293            {'units': 'fps',
294             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
295             'description': 'android.media.cts.VideoDecoderPerfTest'
296                            '#testAvcOther0Perf1280x0720',
297             'value': '664.6463289568261', 'higher_is_better': True},
298            {'units': 'fps',
299             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
300             'description': 'android.media.cts.VideoDecoderPerfTest'
301                            '#testAvcOther0Perf1920x1080',
302             'value': '382.10811352923474', 'higher_is_better': True},
303            {'units': 'fps',
304             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
305             'description': 'android.media.cts.VideoDecoderPerfTest'
306                            '#testH263Goog0Perf0176x0144',
307             'value': '1511.3027429644353', 'higher_is_better': True},
308            {'units': 'fps',
309             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
310             'description': 'android.media.cts.VideoDecoderPerfTest'
311                            '#testHevcGoog0Perf0352x0288',
312             'value': '768.8737453173384', 'higher_is_better': True},
313            {'units': 'fps',
314             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
315             'description': 'android.media.cts.VideoDecoderPerfTest'
316                            '#testHevcGoog0Perf0640x0360',
317             'value': '353.7226028743237', 'higher_is_better': True},
318            {'units': 'fps',
319             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
320             'description': 'android.media.cts.VideoDecoderPerfTest'
321                            '#testHevcGoog0Perf0720x0480',
322             'value': '319.3122874170939', 'higher_is_better': True},
323            {'units': 'fps',
324             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
325             'description': 'android.media.cts.VideoDecoderPerfTest'
326                            '#testHevcGoog0Perf1280x0720',
327             'value': '120.89218432028369', 'higher_is_better': True},
328            {'units': 'fps',
329             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
330             'description': 'android.media.cts.VideoDecoderPerfTest'
331                            '#testMpeg4Goog0Perf0176x0144',
332             'value': '1851.890822618321', 'higher_is_better': True},
333            {'units': 'fps',
334             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
335             'description': 'android.media.cts.VideoDecoderPerfTest'
336                            '#testVp8Goog0Perf0320x0180',
337             'value': '1087.946513466716', 'higher_is_better': True},
338            {'units': 'fps',
339             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
340             'description': 'android.media.cts.VideoDecoderPerfTest'
341                            '#testVp8Goog0Perf0640x0360',
342             'value': '410.18461316281423', 'higher_is_better': True},
343            {'units': 'fps',
344             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
345             'description': 'android.media.cts.VideoDecoderPerfTest'
346                            '#testVp8Goog0Perf1920x1080',
347             'value': '36.26433070651982', 'higher_is_better': True},
348            {'units': 'fps',
349             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
350             'description': 'android.media.cts.VideoDecoderPerfTest'
351                            '#testVp8Other0Perf0320x0180',
352             'value': '1066.7819511702078', 'higher_is_better': True},
353            {'units': 'fps',
354             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
355             'description': 'android.media.cts.VideoDecoderPerfTest'
356                            '#testVp8Other0Perf0640x0360',
357             'value': '930.261434505189', 'higher_is_better': True},
358            {'units': 'fps',
359             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
360             'description': 'android.media.cts.VideoDecoderPerfTest'
361                            '#testVp8Other0Perf1280x0720',
362             'value': '720.4170603577236', 'higher_is_better': True},
363            {'units': 'fps',
364             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
365             'description': 'android.media.cts.VideoDecoderPerfTest'
366                            '#testVp8Other0Perf1920x1080',
367             'value': '377.55742437554915', 'higher_is_better': True},
368            {'units': 'fps',
369             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
370             'description': 'android.media.cts.VideoDecoderPerfTest'
371                            '#testVp9Goog0Perf0320x0180',
372             'value': '988.6158776121617', 'higher_is_better': True},
373            {'units': 'fps',
374             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
375             'description': 'android.media.cts.VideoDecoderPerfTest'
376                            '#testVp9Goog0Perf0640x0360',
377             'value': '409.8162085338674', 'higher_is_better': True},
378            {'units': 'fps',
379             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
380             'description': 'android.media.cts.VideoDecoderPerfTest'
381                            '#testVp9Goog0Perf1280x0720',
382             'value': '147.75847359424512', 'higher_is_better': True},
383            {'units': 'fps',
384             'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
385             'description': 'android.media.cts.VideoDecoderPerfTest'
386                            '#testVp9Goog0Perf1920x1080',
387             'value': '83.95677136649255', 'higher_is_better': True}
388        ]
389        self.assertListEqual(list(perf_result), expected_result)
390
391        perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml(
392            os.path.join(os.path.dirname(os.path.realpath(__file__)),
393                         'tradefed_utils_unittest_data',
394                         'malformed_test_result.xml'),
395            os.path.join('/', 'resultsdir'))
396        self.assertListEqual(list(perf_result), [])
397
398        # assertNoRaises
399        tradefed_utils.get_perf_metrics_from_test_result_xml(
400            os.path.join(os.path.dirname(os.path.realpath(__file__)),
401                         'tradefed_utils_unittest_data',
402                         'not_exist'),
403            os.path.join('/', 'resultsdir'))
404
405
406
407if __name__ == '__main__':
408    unittest.main()
409