1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18# TODO(b/147454897): Keep the test logic in sync with VtsVndkDependency.py
19#                    until it is removed.
20import collections
21import logging
22import os
23import posixpath as target_path_module
24import re
25import shutil
26import sys
27import tempfile
28import unittest
29
30from vts.testcases.vndk import utils
31from vts.testcases.vndk.golden import vndk_data
32from vts.utils.python.library import elf_parser
33from vts.utils.python.vndk import vndk_utils
34
35
36class VtsVndkDependencyTest(unittest.TestCase):
37    """A test case to verify vendor library dependency.
38
39    Attributes:
40        _dut: The AndroidDevice under test.
41        _temp_dir: The temporary directory to which the odm and vendor
42                   partitions are copied.
43        _ll_ndk: Set of strings. The names of low-level NDK libraries in
44                 /system/lib[64].
45        _sp_hal: List of patterns. The names of the same-process HAL libraries
46                 expected to be in /vendor/lib[64].
47        _vndk: Set of strings. The names of VNDK-core libraries.
48        _vndk_sp: Set of strings. The names of VNDK-SP libraries.
49        _SP_HAL_LINK_PATHS: Format strings of same-process HAL's link paths.
50        _VENDOR_LINK_PATHS: Format strings of vendor processes' link paths.
51        _VENDOR_APP_DIRS: The app directories in vendor partitions.
52    """
53    _TARGET_DIR_SEP = "/"
54    _TARGET_ROOT_DIR = "/"
55    _TARGET_ODM_DIR = "/odm"
56    _TARGET_VENDOR_DIR = "/vendor"
57
58    _SP_HAL_LINK_PATHS = [
59        "/odm/{LIB}/egl", "/odm/{LIB}/hw", "/odm/{LIB}",
60        "/vendor/{LIB}/egl", "/vendor/{LIB}/hw", "/vendor/{LIB}"
61    ]
62    _VENDOR_LINK_PATHS = [
63        "/odm/{LIB}/hw", "/odm/{LIB}/egl", "/odm/{LIB}",
64        "/vendor/{LIB}/hw", "/vendor/{LIB}/egl", "/vendor/{LIB}"
65    ]
66    _VENDOR_APP_DIRS = [
67        "/vendor/app", "/vendor/priv-app", "/odm/app", "/odm/priv-app"
68    ]
69    _DEFAULT_PROGRAM_INTERPRETERS = [
70        "/system/bin/linker", "/system/bin/linker64"
71    ]
72
73    class ElfObject(object):
74        """Contains dependencies of an ELF file on target device.
75
76        Attributes:
77            target_path: String. The path to the ELF file on target.
78            name: String. File name of the ELF.
79            target_dir: String. The directory containing the ELF file on
80                        target.
81            bitness: Integer. Bitness of the ELF.
82            deps: List of strings. The names of the depended libraries.
83            runpaths: List of strings. The library search paths.
84            custom_link_paths: List of strings. The library search paths.
85        """
86
87        def __init__(self, target_path, bitness, deps, runpaths,
88                     custom_link_paths):
89            self.target_path = target_path
90            self.name = target_path_module.basename(target_path)
91            self.target_dir = target_path_module.dirname(target_path)
92            self.bitness = bitness
93            self.deps = deps
94            # Format runpaths
95            self.runpaths = []
96            lib_dir_name = "lib64" if bitness == 64 else "lib"
97            for runpath in runpaths:
98                path = runpath.replace("${LIB}", lib_dir_name)
99                path = path.replace("$LIB", lib_dir_name)
100                path = path.replace("${ORIGIN}", self.target_dir)
101                path = path.replace("$ORIGIN", self.target_dir)
102                self.runpaths.append(path)
103            self.custom_link_paths = custom_link_paths
104
105    def setUp(self):
106        """Initializes device, temporary directory, and VNDK lists."""
107        serial_number = os.environ.get("ANDROID_SERIAL")
108        self.assertTrue(serial_number, "$ANDROID_SERIAL is empty.")
109        self._dut = utils.AndroidDevice(serial_number)
110        self.assertTrue(self._dut.IsRoot(), "This test requires adb root.")
111
112        self._temp_dir = tempfile.mkdtemp()
113        for target_dir in (self._TARGET_ODM_DIR, self._TARGET_VENDOR_DIR):
114            if self._dut.IsDirectory(target_dir):
115                logging.info("adb pull %s %s", target_dir, self._temp_dir)
116                self._dut.AdbPull(target_dir, self._temp_dir)
117            else:
118                logging.info("Skip adb pull %s", target_dir)
119
120        vndk_lists = vndk_data.LoadVndkLibraryListsFromResources(
121            self._dut.GetVndkVersion(),
122            vndk_data.SP_HAL,
123            vndk_data.LL_NDK,
124            vndk_data.VNDK,
125            vndk_data.VNDK_SP)
126        self.assertTrue(vndk_lists, "Cannot load VNDK library lists.")
127
128        sp_hal_strings = vndk_lists[0]
129        self._sp_hal = [re.compile(x) for x in sp_hal_strings]
130        (self._ll_ndk, self._vndk, self._vndk_sp) = vndk_lists[1:]
131
132        logging.debug("LL_NDK: %s", self._ll_ndk)
133        logging.debug("SP_HAL: %s", sp_hal_strings)
134        logging.debug("VNDK: %s", self._vndk)
135        logging.debug("VNDK_SP: %s", self._vndk_sp)
136
137    def tearDown(self):
138        """Deletes the temporary directory."""
139        logging.info("Delete %s", self._temp_dir)
140        shutil.rmtree(self._temp_dir, ignore_errors=True)
141
142    def _IsElfObjectForAp(self, elf, target_path, abi_list):
143        """Checks whether an ELF object is for application processor.
144
145        Args:
146            elf: The object of elf_parser.ElfParser.
147            target_path: The path to the ELF file on target.
148            abi_list: A list of strings, the ABIs of the application processor.
149
150        Returns:
151            A boolean, whether the ELF object is for application processor.
152        """
153        if not any(elf.MatchCpuAbi(x) for x in abi_list):
154            logging.debug("%s does not match the ABI", target_path)
155            return False
156
157        # b/115567177 Skip an ELF file if it meets the following 3 conditions:
158        # The ELF type is executable.
159        if not elf.IsExecutable():
160            return True
161
162        # It requires special program interpreter.
163        interp = elf.GetProgramInterpreter()
164        if not interp or interp in self._DEFAULT_PROGRAM_INTERPRETERS:
165            return True
166
167        # It does not have execute permission in the file system.
168        if self._dut.IsExecutable(target_path):
169            return True
170
171        return False
172
173    def _IsElfObjectBuiltForAndroid(self, elf, target_path):
174        """Checks whether an ELF object is built for Android.
175
176        Some ELF objects in vendor partition require special program
177        interpreters. Such executable files have .interp sections, but shared
178        libraries don't. As there is no reliable way to identify those
179        libraries. This method checks .note.android.ident section which is
180        created by Android build system.
181
182        Args:
183            elf: The object of elf_parser.ElfParser.
184            target_path: The path to the ELF file on target.
185
186        Returns:
187            A boolean, whether the ELF object is built for Android.
188        """
189        # b/133399940 Skip an ELF file if it does not have .note.android.ident
190        # section and meets one of the following conditions:
191        if elf.HasAndroidIdent():
192            return True
193
194        # It's in the specific directory and is a shared library.
195        if (target_path.startswith("/vendor/arib/lib/") and
196                ".so" in target_path and
197                elf.IsSharedObject()):
198            return False
199
200        # It's in the specific directory, requires special program interpreter,
201        # and is executable.
202        if target_path.startswith("/vendor/arib/bin/"):
203            interp = elf.GetProgramInterpreter()
204            if interp and interp not in self._DEFAULT_PROGRAM_INTERPRETERS:
205                if elf.IsExecutable() or self._dut.IsExecutable(target_path):
206                    return False
207
208        return True
209
210    @staticmethod
211    def _IterateFiles(host_dir):
212        """Iterates files in a host directory.
213
214        Args:
215            host_dir: The host directory.
216
217        Yields:
218            The file paths under the directory.
219        """
220        for root_dir, dir_names, file_names in os.walk(host_dir):
221            for file_name in file_names:
222                yield os.path.join(root_dir, file_name)
223
224    def _LoadElfObjects(self, host_dir, target_dir, abi_list,
225                        elf_error_handler):
226        """Scans a host directory recursively and loads all ELF files in it.
227
228        Args:
229            host_dir: The host directory to scan.
230            target_dir: The path from which host_dir is copied.
231            abi_list: A list of strings, the ABIs of the ELF files to load.
232            elf_error_handler: A function that takes 2 arguments
233                               (target_path, exception). It is called when
234                               the parser fails to read an ELF file.
235
236        Returns:
237            List of ElfObject.
238        """
239        objs = []
240        for full_path in self._IterateFiles(host_dir):
241            rel_path = os.path.relpath(full_path, host_dir)
242            target_path = target_path_module.join(
243                target_dir, *rel_path.split(os.path.sep))
244            try:
245                elf = elf_parser.ElfParser(full_path)
246            except elf_parser.ElfError:
247                logging.debug("%s is not an ELF file", target_path)
248                continue
249            try:
250                if not self._IsElfObjectForAp(elf, target_path, abi_list):
251                    logging.info("%s is not for application processor",
252                                 target_path)
253                    continue
254                if not self._IsElfObjectBuiltForAndroid(elf, target_path):
255                    logging.warning("%s is not built for Android, which is no "
256                                    "longer exempted.", target_path)
257
258                deps, runpaths = elf.ListDependencies()
259            except elf_parser.ElfError as e:
260                elf_error_handler(target_path, e)
261                continue
262            finally:
263                elf.Close()
264
265            logging.info("%s depends on: %s", target_path, ", ".join(deps))
266            if runpaths:
267                logging.info("%s has runpaths: %s",
268                             target_path, ":".join(runpaths))
269
270            # b/123216664 App libraries depend on those in the same directory.
271            custom_link_paths = []
272            if any(target_path.startswith(app_dir + self._TARGET_DIR_SEP) for
273                    app_dir in self._VENDOR_APP_DIRS):
274                custom_link_paths.append(
275                    target_path_module.dirname(target_path))
276
277            objs.append(self.ElfObject(target_path, elf.bitness, deps,
278                                       runpaths, custom_link_paths))
279        return objs
280
281    def _FindLibsInLinkPaths(self, bitness, link_paths, objs):
282        """Finds libraries in link paths.
283
284        Args:
285            bitness: 32 or 64, the bitness of the returned libraries.
286            link_paths: List of strings, the default link paths.
287            objs: List of ElfObject, the libraries/executables to be filtered
288                  by bitness and path.
289
290        Returns:
291            A defaultdict, {dir: {name: obj}} where obj is an ElfObject, dir
292            is obj.target_dir, and name is obj.name.
293        """
294        namespace = collections.defaultdict(dict)
295        for obj in objs:
296            if (obj.bitness == bitness and
297                    any(obj.target_path.startswith(link_path +
298                                                   self._TARGET_DIR_SEP)
299                        for link_path in link_paths)):
300                namespace[obj.target_dir][obj.name] = obj
301        return namespace
302
303    def _DfsDependencies(self, lib, searched, namespace, link_paths):
304        """Depth-first-search for library dependencies.
305
306        Args:
307            lib: ElfObject, the library to search for dependencies.
308            searched: The set of searched libraries.
309            namespace: Defaultdict, {dir: {name: obj}} containing all
310                       searchable libraries.
311            link_paths: List of strings, the default link paths.
312        """
313        if lib in searched:
314            return
315        searched.add(lib)
316        for dep_name in lib.deps:
317            for link_path in lib.custom_link_paths + lib.runpaths + link_paths:
318                if dep_name in namespace[link_path]:
319                    self._DfsDependencies(namespace[link_path][dep_name],
320                                          searched, namespace, link_paths)
321                    break
322
323    def _FindDisallowedDependencies(self, objs, namespace, link_paths,
324                                    *vndk_lists):
325        """Tests if libraries/executables have disallowed dependencies.
326
327        Args:
328            objs: Collection of ElfObject, the libraries/executables under
329                  test.
330            namespace: Defaultdict, {dir: {name: obj}} containing all libraries
331                       in the linker namespace.
332            link_paths: List of strings, the default link paths.
333            vndk_lists: Collections of library names in VNDK, VNDK-SP, etc.
334
335        Returns:
336            List of tuples (path, disallowed_dependencies).
337        """
338        dep_errors = []
339        for obj in objs:
340            disallowed_libs = []
341            for dep_name in obj.deps:
342                if any((dep_name in vndk_list) for vndk_list in vndk_lists):
343                    continue
344                if any((dep_name in namespace[link_path]) for link_path in
345                       obj.custom_link_paths + obj.runpaths + link_paths):
346                    continue
347                disallowed_libs.append(dep_name)
348
349            if disallowed_libs:
350                dep_errors.append((obj.target_path, disallowed_libs))
351        return dep_errors
352
353    def _TestElfDependency(self, bitness, objs):
354        """Tests vendor libraries/executables and SP-HAL dependencies.
355
356        Args:
357            bitness: 32 or 64, the bitness of the vendor libraries.
358            objs: List of ElfObject. The libraries/executables in odm and
359                  vendor partitions.
360
361        Returns:
362            List of tuples (path, disallowed_dependencies).
363        """
364        vndk_sp_ext_dirs = vndk_utils.GetVndkSpExtDirectories(bitness)
365
366        vendor_link_paths = [vndk_utils.FormatVndkPath(x, bitness) for
367                             x in self._VENDOR_LINK_PATHS]
368        vendor_namespace = self._FindLibsInLinkPaths(
369            bitness, vendor_link_paths + self._VENDOR_APP_DIRS, objs)
370        # Exclude VNDK and VNDK-SP extensions from vendor libraries.
371        for vndk_ext_dir in (vndk_utils.GetVndkExtDirectories(bitness) +
372                             vndk_utils.GetVndkSpExtDirectories(bitness)):
373            vendor_namespace.pop(vndk_ext_dir, None)
374        logging.info("%d-bit odm, vendor, and SP-HAL libraries:", bitness)
375        for dir_path, libs in vendor_namespace.items():
376            logging.info("%s: %s", dir_path, ",".join(libs.keys()))
377
378        sp_hal_link_paths = [vndk_utils.FormatVndkPath(x, bitness) for
379                             x in self._SP_HAL_LINK_PATHS]
380        sp_hal_namespace = self._FindLibsInLinkPaths(bitness,
381                                                     sp_hal_link_paths, objs)
382
383        # Find same-process HAL and dependencies
384        sp_hal_libs = set()
385        for link_path in sp_hal_link_paths:
386            for obj in sp_hal_namespace[link_path].values():
387                if any(x.match(obj.target_path) for x in self._sp_hal):
388                    self._DfsDependencies(obj, sp_hal_libs, sp_hal_namespace,
389                                          sp_hal_link_paths)
390        logging.info("%d-bit SP-HAL libraries: %s",
391                     bitness, ", ".join(x.name for x in sp_hal_libs))
392
393        # Find VNDK-SP extension libraries and their dependencies.
394        vndk_sp_ext_libs = set(obj for obj in objs if
395                               obj.bitness == bitness and
396                               obj.target_dir in vndk_sp_ext_dirs)
397        vndk_sp_ext_deps = set()
398        for lib in vndk_sp_ext_libs:
399            self._DfsDependencies(lib, vndk_sp_ext_deps, sp_hal_namespace,
400                                  sp_hal_link_paths)
401        logging.info("%d-bit VNDK-SP extension libraries and dependencies: %s",
402                     bitness, ", ".join(x.name for x in vndk_sp_ext_deps))
403
404        # A vendor library/executable is allowed to depend on
405        # LL-NDK
406        # VNDK
407        # VNDK-SP
408        # Other libraries in vendor link paths
409        vendor_objs = {obj for obj in objs if
410                       obj.bitness == bitness and
411                       obj not in sp_hal_libs and
412                       obj not in vndk_sp_ext_deps}
413        dep_errors = self._FindDisallowedDependencies(
414            vendor_objs, vendor_namespace, vendor_link_paths,
415            self._ll_ndk, self._vndk, self._vndk_sp)
416
417        # A VNDK-SP extension library/dependency is allowed to depend on
418        # LL-NDK
419        # VNDK-SP
420        # Libraries in vendor link paths
421        # Other VNDK-SP extension libraries, which is a subset of VNDK-SP
422        #
423        # However, it is not allowed to indirectly depend on VNDK. i.e., the
424        # depended vendor libraries must not depend on VNDK.
425        #
426        # vndk_sp_ext_deps and sp_hal_libs may overlap. Their dependency
427        # restrictions are the same.
428        dep_errors.extend(self._FindDisallowedDependencies(
429            vndk_sp_ext_deps - sp_hal_libs, vendor_namespace,
430            vendor_link_paths, self._ll_ndk, self._vndk_sp))
431
432        if not vndk_utils.IsVndkRuntimeEnforced(self._dut):
433            logging.warning("Ignore dependency errors: %s", dep_errors)
434            dep_errors = []
435
436        # A same-process HAL library is allowed to depend on
437        # LL-NDK
438        # VNDK-SP
439        # Other same-process HAL libraries and dependencies
440        dep_errors.extend(self._FindDisallowedDependencies(
441            sp_hal_libs, sp_hal_namespace, sp_hal_link_paths,
442            self._ll_ndk, self._vndk_sp))
443        return dep_errors
444
445    def testElfDependency(self):
446        """Tests vendor libraries/executables and SP-HAL dependencies."""
447        read_errors = []
448        abi_list = self._dut.GetCpuAbiList()
449        objs = self._LoadElfObjects(
450            self._temp_dir, self._TARGET_ROOT_DIR, abi_list,
451            lambda p, e: read_errors.append((p, str(e))))
452
453        dep_errors = self._TestElfDependency(32, objs)
454        if self._dut.GetCpuAbiList(64):
455            dep_errors.extend(self._TestElfDependency(64, objs))
456
457        assert_lines = []
458        if read_errors:
459            error_lines = ["%s: %s" % (x[0], x[1]) for x in read_errors]
460            logging.error("%d read errors:\n%s",
461                          len(read_errors), "\n".join(error_lines))
462            assert_lines.extend(error_lines[:20])
463
464        if dep_errors:
465            error_lines = ["%s: %s" % (x[0], ", ".join(x[1]))
466                           for x in dep_errors]
467            logging.error("%d disallowed dependencies:\n%s",
468                          len(dep_errors), "\n".join(error_lines))
469            assert_lines.extend(error_lines[:20])
470
471        error_count = len(read_errors) + len(dep_errors)
472        if error_count:
473            if error_count > len(assert_lines):
474                assert_lines.append("...")
475            assert_lines.append("Total number of errors: " + str(error_count))
476            self.fail("\n".join(assert_lines))
477
478
479if __name__ == "__main__":
480    # The logs are written to stdout so that TradeFed test runner can parse the
481    # results from stderr.
482    logging.basicConfig(stream=sys.stdout)
483    # Setting verbosity is required to generate output that the TradeFed test
484    # runner can parse.
485    unittest.main(verbosity=3)
486