1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import logging
18import os.path
19import posixpath as targetpath
20import time
21
22from vts.runners.host import asserts
23from vts.runners.host import base_test
24from vts.runners.host import const
25from vts.runners.host import errors
26from vts.runners.host import keys
27from vts.runners.host import test_runner
28from vts.utils.python.common import list_utils
29from vts.utils.python.coverage import coverage_utils
30from vts.utils.python.os import path_utils
31from vts.utils.python.precondition import precondition_utils
32from vts.utils.python.web import feature_utils
33
34from vts.testcases.template.binary_test import binary_test_case
35
36DATA_NATIVETEST = 'data/nativetest'
37DATA_NATIVETEST64 = '%s64' % DATA_NATIVETEST
38
39
40class BinaryTest(base_test.BaseTestClass):
41    '''Base class to run binary tests on target.
42
43    Attributes:
44        _dut: AndroidDevice, the device under test as config
45        shell: ShellMirrorObject, shell mirror
46        testcases: list of BinaryTestCase objects, list of test cases to run
47        tags: all the tags that appeared in binary list
48        DEVICE_TMP_DIR: string, temp location for storing binary
49        TAG_DELIMITER: string, separator used to separate tag and path
50    '''
51    DEVICE_TMP_DIR = '/data/local/tmp'
52    TAG_DELIMITER = '::'
53    PUSH_DELIMITER = '->'
54    DEFAULT_TAG_32 = '_%s' % const.SUFFIX_32BIT
55    DEFAULT_TAG_64 = '_%s' % const.SUFFIX_64BIT
56    DEFAULT_LD_LIBRARY_PATH_32 = '/data/local/tmp/32/'
57    DEFAULT_LD_LIBRARY_PATH_64 = '/data/local/tmp/64/'
58    DEFAULT_PROFILING_LIBRARY_PATH_32 = '/data/local/tmp/32/'
59    DEFAULT_PROFILING_LIBRARY_PATH_64 = '/data/local/tmp/64/'
60
61    def setUpClass(self):
62        '''Prepare class, push binaries, set permission, create test cases.'''
63        required_params = [
64            keys.ConfigKeys.IKEY_DATA_FILE_PATH,
65        ]
66        opt_params = [
67            keys.ConfigKeys.IKEY_BINARY_TEST_SOURCE,
68            keys.ConfigKeys.IKEY_BINARY_TEST_WORKING_DIRECTORY,
69            keys.ConfigKeys.IKEY_BINARY_TEST_ENVP,
70            keys.ConfigKeys.IKEY_BINARY_TEST_ARGS,
71            keys.ConfigKeys.IKEY_BINARY_TEST_LD_LIBRARY_PATH,
72            keys.ConfigKeys.IKEY_BINARY_TEST_PROFILING_LIBRARY_PATH,
73            keys.ConfigKeys.IKEY_NATIVE_SERVER_PROCESS_NAME,
74            keys.ConfigKeys.IKEY_PRECONDITION_FILE_PATH_PREFIX,
75            keys.ConfigKeys.IKEY_PRECONDITION_SYSPROP,
76        ]
77        self.getUserParams(
78            req_param_names=required_params, opt_param_names=opt_params)
79
80        # test-module-name is required in binary tests.
81        self.getUserParam(
82            keys.ConfigKeys.KEY_TESTBED_NAME, error_if_not_found=True)
83
84        logging.debug("%s: %s", keys.ConfigKeys.IKEY_DATA_FILE_PATH,
85                     self.data_file_path)
86
87        self.binary_test_source = self.getUserParam(
88            keys.ConfigKeys.IKEY_BINARY_TEST_SOURCE, default_value=[])
89
90        self.working_directory = {}
91        if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_WORKING_DIRECTORY):
92            self.binary_test_working_directory = map(
93                str, self.binary_test_working_directory)
94            for token in self.binary_test_working_directory:
95                tag = ''
96                path = token
97                if self.TAG_DELIMITER in token:
98                    tag, path = token.split(self.TAG_DELIMITER)
99                self.working_directory[tag] = path
100
101        self.envp = {}
102        if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_ENVP):
103            self.binary_test_envp = map(str, self.binary_test_envp)
104            for token in self.binary_test_envp:
105                tag = ''
106                path = token
107                split = token.find(self.TAG_DELIMITER)
108                if split >= 0:
109                    tag, path = token[:split], token[
110                        split + len(self.TAG_DELIMITER):]
111                if tag in self.envp:
112                    self.envp[tag] += ' %s' % path
113                else:
114                    self.envp[tag] = path
115
116        self.args = {}
117        if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_ARGS):
118            self.binary_test_args = map(str, self.binary_test_args)
119            for token in self.binary_test_args:
120                tag = ''
121                arg = token
122                split = token.find(self.TAG_DELIMITER)
123                if split >= 0:
124                    tag, arg = token[:split], token[
125                        split + len(self.TAG_DELIMITER):]
126                if tag in self.args:
127                    self.args[tag] += ' %s' % arg
128                else:
129                    self.args[tag] = arg
130
131        if hasattr(self, keys.ConfigKeys.IKEY_PRECONDITION_FILE_PATH_PREFIX):
132            self.file_path_prefix = {
133                self.DEFAULT_TAG_32: [],
134                self.DEFAULT_TAG_64: [],
135            }
136            self.precondition_file_path_prefix = map(
137                str, self.precondition_file_path_prefix)
138            for token in self.precondition_file_path_prefix:
139                tag = ''
140                path = token
141                if self.TAG_DELIMITER in token:
142                    tag, path = token.split(self.TAG_DELIMITER)
143                if tag == '':
144                    self.file_path_prefix[self.DEFAULT_TAG_32].append(path)
145                    self.file_path_prefix[self.DEFAULT_TAG_64].append(path)
146                elif tag in self.file_path_prefix:
147                    self.file_path_prefix[tag].append(path)
148                else:
149                    logging.warn(
150                        "Incorrect tag %s in precondition-file-path-prefix",
151                        tag)
152
153        self.ld_library_path = {
154            self.DEFAULT_TAG_32: self.DEFAULT_LD_LIBRARY_PATH_32,
155            self.DEFAULT_TAG_64: self.DEFAULT_LD_LIBRARY_PATH_64,
156        }
157        if hasattr(self, keys.ConfigKeys.IKEY_BINARY_TEST_LD_LIBRARY_PATH):
158            self.binary_test_ld_library_path = map(
159                str, self.binary_test_ld_library_path)
160            for token in self.binary_test_ld_library_path:
161                tag = ''
162                path = token
163                if self.TAG_DELIMITER in token:
164                    tag, path = token.split(self.TAG_DELIMITER)
165                if tag in self.ld_library_path:
166                    self.ld_library_path[tag] = '{}:{}'.format(
167                        path, self.ld_library_path[tag])
168                else:
169                    self.ld_library_path[tag] = path
170
171        self.profiling_library_path = {
172            self.DEFAULT_TAG_32: self.DEFAULT_PROFILING_LIBRARY_PATH_32,
173            self.DEFAULT_TAG_64: self.DEFAULT_PROFILING_LIBRARY_PATH_64,
174        }
175        if hasattr(self,
176                   keys.ConfigKeys.IKEY_BINARY_TEST_PROFILING_LIBRARY_PATH):
177            self.binary_test_profiling_library_path = map(
178                str, self.binary_test_profiling_library_path)
179            for token in self.binary_test_profiling_library_path:
180                tag = ''
181                path = token
182                if self.TAG_DELIMITER in token:
183                    tag, path = token.split(self.TAG_DELIMITER)
184                self.profiling_library_path[tag] = path
185
186        self._dut = self.android_devices[0]
187        self.shell = self._dut.shell
188
189        if self.coverage.enabled and self.coverage.global_coverage:
190            self.coverage.InitializeDeviceCoverage(self._dut)
191            for tag in [self.DEFAULT_TAG_32, self.DEFAULT_TAG_64]:
192                if tag in self.envp:
193                    self.envp[tag] = '%s %s'.format(
194                        self.envp[tag], coverage_utils.COVERAGE_TEST_ENV)
195                else:
196                    self.envp[tag] = coverage_utils.COVERAGE_TEST_ENV
197
198        self.testcases = []
199        if not precondition_utils.CheckSysPropPrecondition(
200                self, self._dut, self.shell):
201            logging.warn('Precondition sysprop not met; '
202                         'all tests skipped.')
203            self.skipAllTests('precondition sysprop not met')
204
205        self.tags = set()
206        self.CreateTestCases()
207        cmd = list(
208            set('chmod 755 %s' % test_case.path
209                for test_case in self.testcases))
210        cmd_results = self.shell.Execute(cmd)
211        if any(cmd_results[const.EXIT_CODE]):
212            logging.error('Failed to set permission to some of the binaries:\n'
213                          '%s\n%s', cmd, cmd_results)
214
215    def CreateTestCases(self):
216        '''Push files to device and create test case objects.'''
217        source_list = list(map(self.ParseTestSource, self.binary_test_source))
218
219        def isValidSource(source):
220            '''Checks that the truth value and bitness of source is valid.
221
222            Args:
223                source: a tuple of (string, string, string or None),
224                representing (host side absolute path, device side absolute
225                path, tag), is the return value of self.ParseTestSource
226
227            Returns:
228                False if source has a false truth value or its bitness does
229                not match the abi_bitness of the test run.
230            '''
231            if not source:
232                return False
233
234            tag = source[2]
235            if tag is None:
236                return True
237
238            tag = str(tag)
239            if (tag.endswith(const.SUFFIX_32BIT) and self.abi_bitness == '64'
240                ) or (tag.endswith(const.SUFFIX_64BIT) and
241                      self.abi_bitness == '32'):
242                logging.debug('Bitness of test source, %s, does not match the '
243                             'abi_bitness, %s, of test run. Skipping',
244                             str(source[0]),
245                             self.abi_bitness)
246                return False
247
248            return True
249
250        source_list = filter(isValidSource, source_list)
251        logging.debug('Parsed test sources: %s', source_list)
252
253        # Push source files first
254        for src, dst, tag in source_list:
255            if src:
256                if os.path.isdir(src):
257                    src = os.path.join(src, '.')
258                logging.debug('Pushing from %s to %s.', src, dst)
259                self._dut.adb.push('{src} {dst}'.format(src=src, dst=dst))
260                self.shell.Execute('ls %s' % dst)
261
262        if not hasattr(self, 'testcases'):
263            self.testcases = []
264
265        # Then create test cases
266        for src, dst, tag in source_list:
267            if tag is not None:
268                # tag not being None means to create a test case
269                self.tags.add(tag)
270                logging.debug('Creating test case from %s with tag %s', dst,
271                             tag)
272                testcase = self.CreateTestCase(dst, tag)
273                if not testcase:
274                    continue
275
276                if type(testcase) is list:
277                    self.testcases.extend(testcase)
278                else:
279                    self.testcases.append(testcase)
280
281        if not self.testcases:
282            logging.warn("No test case is found or generated.")
283
284    def PutTag(self, name, tag):
285        '''Put tag on name and return the resulting string.
286
287        Args:
288            name: string, a test name
289            tag: string
290
291        Returns:
292            String, the result string after putting tag on the name
293        '''
294        return '{}{}'.format(name, tag)
295
296    def ExpandListItemTags(self, input_list):
297        '''Expand list items with tags.
298
299        Since binary test allows a tag to be added in front of the binary
300        path, test names are generated with tags attached. This function is
301        used to expand the filters correspondingly. If a filter contains
302        a tag, only test name with that tag will be included in output.
303        Otherwise, all known tags will be paired to the test name in output
304        list.
305
306        Args:
307            input_list: list of string, the list to expand
308
309        Returns:
310            A list of string
311        '''
312        result = []
313        for item in input_list:
314            if self.TAG_DELIMITER in item:
315                tag, name = item.split(self.TAG_DELIMITER)
316                result.append(self.PutTag(name, tag))
317            for tag in self.tags:
318                result.append(self.PutTag(item, tag))
319        return result
320
321    def tearDownClass(self):
322        '''Perform clean-up tasks'''
323        # Retrieve coverage if applicable
324        if self.coverage.enabled and self.coverage.global_coverage:
325            if not self.isSkipAllTests():
326                self.coverage.SetCoverageData(dut=self._dut, isGlobal=True)
327
328        if self.profiling.enabled:
329            self.profiling.DisableVTSProfiling(self.shell)
330
331        # Clean up the pushed binaries
332        logging.debug('Start class cleaning up jobs.')
333        # Delete pushed files
334
335        sources = [
336            self.ParseTestSource(src) for src in self.binary_test_source
337        ]
338        sources = set(filter(bool, sources))
339        paths = [dst for src, dst, tag in sources if src and dst]
340        cmd = ['rm -rf %s' % dst for dst in paths]
341        cmd_results = self.shell.Execute(cmd, no_except=True)
342        if not cmd_results or any(cmd_results[const.EXIT_CODE]):
343            logging.warning('Failed to clean up test class: %s', cmd_results)
344
345        # Delete empty directories in working directories
346        dir_set = set(path_utils.TargetDirName(dst) for dst in paths)
347        dir_set.add(self.ParseTestSource('')[1])
348        dirs = list(dir_set)
349        dirs.sort(lambda x, y: cmp(len(y), len(x)))
350        cmd = ['rmdir %s' % d for d in dirs]
351        cmd_results = self.shell.Execute(cmd, no_except=True)
352        if not cmd_results or any(cmd_results[const.EXIT_CODE]):
353            logging.warning('Failed to remove: %s', cmd_results)
354
355        if not self.isSkipAllTests() and self.profiling.enabled:
356            self.profiling.ProcessAndUploadTraceData()
357
358        logging.debug('Finished class cleaning up jobs.')
359
360    def ParseTestSource(self, source):
361        '''Convert host side binary path to device side path.
362
363        Args:
364            source: string, binary test source string
365
366        Returns:
367            A tuple of (string, string, string), representing (host side
368            absolute path, device side absolute path, tag). Returned tag
369            will be None if the test source is for pushing file to working
370            directory only. If source file is specified for adb push but does not
371            exist on host, None will be returned.
372        '''
373        tag = ''
374        path = source
375        if self.TAG_DELIMITER in source:
376            tag, path = source.split(self.TAG_DELIMITER)
377
378        src = path
379        dst = None
380        if self.PUSH_DELIMITER in path:
381            src, dst = path.split(self.PUSH_DELIMITER)
382
383        if src:
384            src = os.path.join(self.data_file_path, src)
385            if not os.path.exists(src):
386                logging.warning('binary source file is specified '
387                                'but does not exist on host: %s', src)
388                return None
389
390        push_only = dst is not None and dst == ''
391
392        if not dst:
393            parent = self.working_directory[
394                tag] if tag in self.working_directory else self._GetDefaultBinaryPushDstPath(
395                    src, tag)
396            dst = path_utils.JoinTargetPath(parent, os.path.basename(src))
397
398        if push_only:
399            tag = None
400
401        return str(src), str(dst), tag
402
403    def _GetDefaultBinaryPushDstPath(self, src, tag):
404        '''Get default binary push destination path.
405
406        This method is called to get default push destination path when
407        it is not specified.
408
409        If binary source path contains 'data/nativetest[64]', then the binary
410        will be pushed to /data/nativetest[64] instead of /data/local/tmp
411
412        Args:
413            src: string, source path of binary
414            tag: string, tag of binary source
415
416        Returns:
417            string, default push path
418        '''
419        src_lower = src.lower()
420        if DATA_NATIVETEST64 in src_lower:
421            parent_path = targetpath.sep + DATA_NATIVETEST64
422        elif DATA_NATIVETEST in src_lower:
423            parent_path = targetpath.sep + DATA_NATIVETEST
424        else:
425            parent_path = self.DEVICE_TMP_DIR
426
427        return targetpath.join(
428            parent_path, 'vts_binary_test_%s' % self.__class__.__name__, tag)
429
430    def CreateTestCase(self, path, tag=''):
431        '''Create a list of TestCase objects from a binary path.
432
433        Args:
434            path: string, absolute path of a binary on device
435            tag: string, a tag that will be appended to the end of test name
436
437        Returns:
438            A list of BinaryTestCase objects
439        '''
440        working_directory = self.working_directory[
441            tag] if tag in self.working_directory else None
442        envp = self.envp[tag] if tag in self.envp else ''
443        args = self.args[tag] if tag in self.args else ''
444        ld_library_path = self.ld_library_path[
445            tag] if tag in self.ld_library_path else None
446        profiling_library_path = self.profiling_library_path[
447            tag] if tag in self.profiling_library_path else None
448
449        return binary_test_case.BinaryTestCase(
450            '',
451            path_utils.TargetBaseName(path),
452            path,
453            tag,
454            self.PutTag,
455            working_directory,
456            ld_library_path,
457            profiling_library_path,
458            envp=envp,
459            args=args)
460
461    def VerifyTestResult(self, test_case, command_results):
462        '''Parse test case command result.
463
464        Args:
465            test_case: BinaryTestCase object, the test case whose command
466            command_results: dict of lists, shell command result
467        '''
468        asserts.assertTrue(command_results, 'Empty command response.')
469        asserts.assertFalse(
470            any(command_results[const.EXIT_CODE]),
471            'Test {} failed with the following results: {}'.format(
472                test_case, command_results))
473
474    def RunTestCase(self, test_case):
475        '''Runs a test_case.
476
477        Args:
478            test_case: BinaryTestCase object
479        '''
480        if self.profiling.enabled:
481            self.profiling.EnableVTSProfiling(self.shell,
482                                              test_case.profiling_library_path)
483
484        cmd = test_case.GetRunCommand()
485        logging.debug("Executing binary test command: %s", cmd)
486        command_results = self.shell.Execute(cmd)
487
488        self.VerifyTestResult(test_case, command_results)
489
490        if self.profiling.enabled:
491            self.profiling.ProcessTraceDataForTestCase(self._dut)
492            self.profiling.DisableVTSProfiling(self.shell)
493
494    def generateAllTests(self):
495        '''Runs all binary tests.'''
496        self.runGeneratedTests(
497            test_func=self.RunTestCase, settings=self.testcases, name_func=str)
498
499
500if __name__ == "__main__":
501    test_runner.main()
502