1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import logging
18import os
19import xml.etree.ElementTree
20
21from vts.runners.host import asserts
22from vts.runners.host import const
23from vts.runners.host import keys
24from vts.runners.host import test_runner
25
26from vts.testcases.template.binary_test import binary_test
27from vts.testcases.template.binary_test import binary_test_case
28from vts.testcases.template.gtest_binary_test import gtest_test_case
29
30_GTEST_RESULT_ATTRIBUTE_WHITE_LIST = ('properties',)
31
32
33class GtestBinaryTest(binary_test.BinaryTest):
34    '''Base class to run gtests binary on target.
35
36    Attributes:
37        DEVICE_TEST_DIR: string, temp location for storing binary
38        TAG_PATH_SEPARATOR: string, separator used to separate tag and path
39        shell: ShellMirrorObject, shell mirror
40        tags: all the tags that appeared in binary list
41        testcases: list of GtestTestCase objects, list of test cases to run
42        _dut: AndroidDevice, the device under test as config
43        _gtest_results: list of GtestResult objects, used during batch mode
44                        for result storage and parsing
45    '''
46
47    # @Override
48    def setUpClass(self):
49        '''Prepare class, push binaries, set permission, create test cases.'''
50        self.collect_tests_only = self.getUserParam(
51            keys.ConfigKeys.IKEY_COLLECT_TESTS_ONLY, default_value=False)
52        self.batch_mode = self.getUserParam(
53            keys.ConfigKeys.IKEY_GTEST_BATCH_MODE, default_value=False)
54
55        if self.batch_mode:
56            if self.collect_tests_only:
57                self.batch_mode = False
58                logging.debug("Disable batch mode when collecting tests.")
59            else:
60                self._gtest_results = []
61
62        super(GtestBinaryTest, self).setUpClass()
63
64    # @Override
65    def CreateTestCase(self, path, tag=''):
66        '''Create a list of GtestTestCase objects from a binary path.
67
68        Args:
69            path: string, absolute path of a gtest binary on device
70            tag: string, a tag that will be appended to the end of test name
71
72        Returns:
73            A list of GtestTestCase objects on success; an empty list otherwise.
74            In non-batch mode, each object respresents a test case in the
75            gtest binary located at the provided path. Usually there are more
76            than one object returned.
77            In batch mode, each object represents a gtest binary located at
78            the provided path; the returned list will always be a one object
79            list in batch mode. Test case names are stored in full_name
80            property in the object, delimited by ':' according to gtest
81            documentation, after being filtered and processed according to
82            host configuration.
83        '''
84        working_directory = self.working_directory[
85            tag] if tag in self.working_directory else None
86        envp = self.envp[tag] if tag in self.envp else ''
87        args = self.args[tag] if tag in self.args else ''
88        ld_library_path = self.ld_library_path[
89            tag] if tag in self.ld_library_path else None
90        profiling_library_path = self.profiling_library_path[
91            tag] if tag in self.profiling_library_path else None
92
93        gtest_list_args = args + " --gtest_list_tests"
94        list_test_case = binary_test_case.BinaryTestCase(
95            'gtest_list_tests',
96            path,
97            path,
98            tag,
99            self.PutTag,
100            working_directory,
101            ld_library_path,
102            profiling_library_path,
103            envp=envp,
104            args=gtest_list_args)
105        cmd = ['chmod 755 %s' % path, list_test_case.GetRunCommand()]
106        cmd_results = self.shell.Execute(cmd)
107        test_cases = []
108        asserts.assertFalse(any(cmd_results[const.EXIT_CODE]),
109                'Failed to list test cases from %s. Command: %s, Result: %s.' %
110                (path, cmd, cmd_results))
111
112        test_suite = ''
113        for line in cmd_results[const.STDOUT][1].split('\n'):
114            line = str(line)
115            if not len(line.strip()):
116                continue
117            elif line.startswith(' '):  # Test case name
118                test_name = line.split('#')[0].strip()
119                test_case = gtest_test_case.GtestTestCase(
120                    test_suite, test_name, path, tag, self.PutTag,
121                    working_directory, ld_library_path, profiling_library_path,
122                    envp=envp, args=args)
123                logging.debug('Gtest test case: %s' % test_case)
124                test_cases.append(test_case)
125            else:  # Test suite name
126                test_suite = line.strip()
127                if test_suite.endswith('.'):
128                    test_suite = test_suite[:-1]
129
130        #if not self.batch_mode:
131        # Avoid batch mode as it creates overly large filters
132        return test_cases
133
134        # Gtest batch mode
135        # test_names = map(lambda test: test.full_name, test_cases)
136        #test_names = {}
137
138        #gtest_batch = gtest_test_case.GtestTestCase(
139        #    path, '', path, tag, self.PutTag, working_directory,
140        #    ld_library_path, profiling_library_path, envp=envp)
141        #gtest_batch.full_name = ':'.join(test_names)
142        #return [gtest_batch]
143
144    # @Override
145    def VerifyTestResult(self, test_case, command_results):
146        '''Parse Gtest xml result output.
147
148        Sample
149        <testsuites tests="1" failures="1" disabled="0" errors="0"
150         timestamp="2017-05-24T18:32:10" time="0.012" name="AllTests">
151          <testsuite name="ConsumerIrHidlTest"
152           tests="1" failures="1" disabled="0" errors="0" time="0.01">
153            <testcase name="TransmitTest" status="run" time="0.01"
154             classname="ConsumerIrHidlTest">
155              <failure message="hardware/interfaces..." type="">
156                <![CDATA[hardware/interfaces...]]>
157              </failure>
158            </testcase>
159          </testsuite>
160        </testsuites>
161
162        Args:
163            test_case: GtestTestCase object, the test being run. This param
164                       is not currently used in this method.
165            command_results: dict of lists, shell command result
166        '''
167        asserts.assertTrue(command_results, 'Empty command response.')
168        asserts.assertEqual(
169            len(command_results), 3, 'Abnormal command response.')
170        for item in command_results.values():
171            asserts.assertEqual(
172                len(item), 2,
173                'Abnormal command result length: %s' % command_results)
174
175        for stderr in command_results[const.STDERR]:
176            if stderr and stderr.strip():
177                for line in stderr.split('\n'):
178                    logging.error(line)
179
180        xml_str = command_results[const.STDOUT][1]
181
182        if self.batch_mode:
183            self._ParseBatchResults(test_case, xml_str)
184            return
185
186        asserts.assertFalse(
187            command_results[const.EXIT_CODE][1],
188            'Failed to show Gtest XML output: %s' % command_results)
189
190        root = self._ParseResultXmlString(xml_str)
191        asserts.assertEqual(root.get('tests'), '1', 'No tests available')
192        success = True
193        if root.get('errors') != '0' or root.get('failures') != '0':
194            messages = [x.get('message') for x in root.findall('.//failure')]
195            success = False
196
197        for stdout in command_results[const.STDOUT]:
198            if stdout and stdout.strip():
199                for line in stdout.split('\n'):
200                    if success:
201                        logging.debug(line)
202                    else:
203                        logging.error(line)
204
205        if not success:
206            asserts.fail('\n'.join([x for x in messages if x]))
207
208        asserts.skipIf(root.get('disabled') == '1', 'Gtest test case disabled')
209
210    def _ParseResultXmlString(self, xml_str):
211        """Parses the xml result string into elements.
212
213        Args:
214            xml_str: string, result xml text content.
215
216        Returns:
217            xml.etree.ElementTree, parsed xml content.
218
219        Raises:
220            assertion failure if xml format is not expected.
221        """
222        asserts.assertTrue(xml_str is not None, 'Test command result not received.')
223        xml_str = xml_str.strip()
224        asserts.assertTrue(xml_str, 'Test command result is empty.')
225
226        try:
227            return xml.etree.ElementTree.fromstring(xml_str)
228        except:
229            asserts.fail('Result xml content is corrupted.')
230
231    def _ParseBatchResults(self, test_case_original, xml_str):
232        '''Parse batch mode gtest results
233
234        Args:
235            test_case_original: GtestTestCase object, original batch test case object
236            xml_str: string, result xml output content
237        '''
238        root = self._ParseResultXmlString(xml_str)
239
240        for test_suite in root:
241            logging.debug('Test tag: %s, attribute: %s',
242                          test_suite.tag,
243                          test_suite.attrib)
244            for test_case in test_suite:
245                result = gtest_test_case.GtestTestCase(
246                    test_suite.get('name'),
247                    test_case.get('name'), '', test_case_original.tag,
248                    self.PutTag, name_appendix=test_case_original.name_appendix)
249
250                failure_message = None
251                for sub in test_case:
252                    if sub.tag == 'failure':
253                        failure_message = sub.get('message')
254
255                test_case_filtered = filter(
256                    lambda sub: sub.tag not in _GTEST_RESULT_ATTRIBUTE_WHITE_LIST, test_case)
257                if len(test_case_filtered) and not failure_message:
258                    failure_message = 'Error: %s\n' % test_case.attrib
259                    for sub in test_case_filtered:
260                        failure_message += '%s: %s\n' % (sub.tag, sub.attrib)
261
262                result.failure_message = failure_message
263
264                self._gtest_results.append(result)
265
266    def _VerifyBatchResult(self, gtest_result):
267        '''Check a gtest test case result in batch mode
268
269        Args:
270            gtest_result: GtestTestCase object, representing gtest result
271        '''
272        asserts.assertFalse(gtest_result.failure_message,
273                            gtest_result.failure_message)
274
275    # @Override
276    def generateAllTests(self):
277        '''Runs all binary tests.'''
278        if self.batch_mode:
279            for test_case in self.testcases:
280                logging.info('Running %s test cases in batch.',
281                             len(test_case.full_name.split(':')))
282                self.RunTestCase(test_case)
283
284                self.runGeneratedTests(
285                    test_func=self._VerifyBatchResult,
286                    settings=self._gtest_results,
287                    name_func=str)
288
289                self._gtest_results = []
290            return
291
292        self.runGeneratedTests(
293            test_func=self.RunTestCase, settings=self.testcases, name_func=str)
294
295
296if __name__ == "__main__":
297    test_runner.main()
298