1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import json
19import logging
20import os
21import shutil
22import sys
23import tempfile
24import unittest
25
26from vts.testcases.vndk import utils
27from vts.testcases.vndk.golden import vndk_data
28from vts.utils.python.library import elf_parser
29from vts.utils.python.library.vtable import vtable_dumper
30from vts.utils.python.vndk import vndk_utils
31
32
33class VtsVndkAbiTest(unittest.TestCase):
34    """A test module to verify ABI compliance of vendor libraries.
35
36    Attributes:
37        _dut: the AndroidDevice under test.
38        _temp_dir: The temporary directory for libraries copied from device.
39    """
40
41    def setUp(self):
42        """Initializes data file path, device, and temporary directory."""
43        serial_number = os.environ.get("ANDROID_SERIAL", "")
44        self.assertTrue(serial_number, "$ANDROID_SERIAL is empty")
45        self._dut = utils.AndroidDevice(serial_number)
46        self._temp_dir = tempfile.mkdtemp()
47
48    def tearDown(self):
49        """Deletes the temporary directory."""
50        logging.info("Delete %s", self._temp_dir)
51        shutil.rmtree(self._temp_dir)
52
53    def _PullOrCreateDir(self, target_dir, host_dir):
54        """Copies a directory from device. Creates an empty one if not exist.
55
56        Args:
57            target_dir: The directory to copy from device.
58            host_dir: The directory to copy to host.
59        """
60        if not self._dut.IsDirectory(target_dir):
61            logging.info("%s doesn't exist. Create %s.", target_dir, host_dir)
62            os.makedirs(host_dir)
63            return
64        parent_dir = os.path.dirname(host_dir)
65        if parent_dir and not os.path.isdir(parent_dir):
66            os.makedirs(parent_dir)
67        logging.info("adb pull %s %s", target_dir, host_dir)
68        self._dut.AdbPull(target_dir, host_dir)
69
70    def _ToHostPath(self, target_path):
71        """Maps target path to host path in self._temp_dir."""
72        return os.path.join(self._temp_dir, *target_path.strip("/").split("/"))
73
74    @staticmethod
75    def _LoadGlobalSymbolsFromDump(dump_obj):
76        """Loads global symbols from a dump object.
77
78        Args:
79            dump_obj: A dict, the dump in JSON format.
80
81        Returns:
82            A set of strings, the symbol names.
83        """
84        symbols = set()
85        for key in ("elf_functions", "elf_objects"):
86            symbols.update(
87                symbol.get("name", "") for symbol in dump_obj.get(key, []) if
88                symbol.get("binding", "global") == "global")
89        return symbols
90
91    def _DiffElfSymbols(self, dump_obj, parser):
92        """Checks if a library includes all symbols in a dump.
93
94        Args:
95            dump_obj: A dict, the dump in JSON format.
96            parser: An elf_parser.ElfParser that loads the library.
97
98        Returns:
99            A list of strings, the global symbols that are in the dump but not
100            in the library.
101
102        Raises:
103            elf_parser.ElfError if fails to load the library.
104        """
105        dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
106        lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True)
107        return sorted(dump_symbols.difference(lib_symbols))
108
109    @staticmethod
110    def _DiffVtableComponent(offset, expected_symbol, vtable):
111        """Checks if a symbol is in a vtable entry.
112
113        Args:
114            offset: An integer, the offset of the expected symbol.
115            expected_symbol: A string, the name of the expected symbol.
116            vtable: A dict of {offset: [entry]} where offset is an integer and
117                    entry is an instance of vtable_dumper.VtableEntry.
118
119        Returns:
120            A list of strings, the actual possible symbols if expected_symbol
121            does not match the vtable entry.
122            None if expected_symbol matches the entry.
123        """
124        if offset not in vtable:
125            return []
126
127        entry = vtable[offset]
128        if not entry.names:
129            return [hex(entry.value).rstrip('L')]
130
131        if expected_symbol not in entry.names:
132            return entry.names
133
134    def _DiffVtableComponents(self, dump_obj, dumper):
135        """Checks if a library includes all vtable entries in a dump.
136
137        Args:
138            dump_obj: A dict, the dump in JSON format.
139            dumper: An vtable_dumper.VtableDumper that loads the library.
140            bitness: 32 or 64, the size of the vtable entries.
141
142        Returns:
143            A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL).
144            ACTUAL can be "missing", a list of symbol names, or an ELF virtual
145            address.
146
147        Raises:
148            vtable_dumper.VtableError if fails to dump vtable from the library.
149        """
150        function_kinds = [
151            "function_pointer",
152            "complete_dtor_pointer",
153            "deleting_dtor_pointer"
154        ]
155        non_function_kinds = [
156            "vcall_offset",
157            "vbase_offset",
158            "offset_to_top",
159            "rtti",
160            "unused_function_pointer"
161        ]
162        default_vtable_component_kind = "function_pointer"
163
164        global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
165
166        lib_vtables = {vtable.name: vtable
167                       for vtable in dumper.DumpVtables()}
168        logging.debug("\n\n".join(str(vtable)
169                                  for _, vtable in lib_vtables.items()))
170
171        vtables_diff = []
172        for record_type in dump_obj.get("record_types", []):
173            # Since Android R, unique_id has been replaced with linker_set_key.
174            # unique_id starts with "_ZTI"; linker_set_key starts with "_ZTS".
175            type_name_symbol = record_type.get("unique_id", "")
176            if type_name_symbol:
177                vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1)
178            else:
179                type_name_symbol = record_type.get("linker_set_key", "")
180                vtable_symbol = type_name_symbol.replace("_ZTI", "_ZTV", 1)
181
182            # Skip if the vtable symbol isn't global.
183            if vtable_symbol not in global_symbols:
184                continue
185
186            # Collect vtable entries from library dump.
187            if vtable_symbol in lib_vtables:
188                lib_vtable = {entry.offset: entry
189                              for entry in lib_vtables[vtable_symbol].entries}
190            else:
191                lib_vtable = dict()
192
193            for index, entry in enumerate(record_type.get("vtable_components",
194                                                          [])):
195                entry_offset = index * dumper.bitness // 8
196                entry_kind = entry.get("kind", default_vtable_component_kind)
197                entry_symbol = entry.get("mangled_component_name", "")
198                entry_is_pure = entry.get("is_pure", False)
199
200                if entry_kind in non_function_kinds:
201                    continue
202
203                if entry_kind not in function_kinds:
204                    logging.warning("%s: Unexpected vtable entry kind %s",
205                                    vtable_symbol, entry_kind)
206
207                if entry_symbol not in global_symbols:
208                    # Itanium cxx abi doesn't specify pure virtual vtable
209                    # entry's behaviour. However we can still do some checks
210                    # based on compiler behaviour.
211                    # Even though we don't check weak symbols, we can still
212                    # issue a warning when a pure virtual function pointer
213                    # is missing.
214                    if entry_is_pure and entry_offset not in lib_vtable:
215                        logging.warning("%s: Expected pure virtual function"
216                                        "in %s offset %s",
217                                        vtable_symbol, vtable_symbol,
218                                        entry_offset)
219                    continue
220
221                diff_symbols = self._DiffVtableComponent(
222                    entry_offset, entry_symbol, lib_vtable)
223                if diff_symbols is None:
224                    continue
225
226                vtables_diff.append(
227                    (vtable_symbol, str(entry_offset), entry_symbol,
228                     (",".join(diff_symbols) if diff_symbols else "missing")))
229
230        return vtables_diff
231
232    def _ScanLibDirs(self, dump_zip, dump_paths, lib_dirs, dump_version):
233        """Compares dump files with libraries copied from device.
234
235        Args:
236            dump_zip: A zip_file.ZipFile object containing the dumps.
237            dump_paths: A dict of {library name: dump resource path}.
238            lib_dirs: The list of directories containing libraries.
239            dump_version: The VNDK version of the dump files. If the device has
240                          no VNDK version or has extension in vendor partition,
241                          this method compares the unversioned VNDK directories
242                          with the dump directories of the given version.
243
244        Returns:
245            A list of strings, the incompatible libraries.
246        """
247        error_list = []
248        lib_paths = dict()
249        for lib_dir in lib_dirs:
250            for parent_dir, dir_names, lib_names in os.walk(lib_dir):
251                for lib_name in lib_names:
252                    if lib_name not in lib_paths:
253                        lib_paths[lib_name] = os.path.join(parent_dir,
254                                                           lib_name)
255        for lib_name, dump_path in dump_paths.items():
256            if lib_name not in lib_paths:
257                logging.info("%s: Not found on target", lib_name)
258                continue
259            lib_path = lib_paths[lib_name]
260            rel_path = os.path.relpath(lib_path, self._temp_dir)
261
262            has_exception = False
263            missing_symbols = []
264            vtable_diff = []
265
266            try:
267                with dump_zip.open(dump_path, "r") as dump_file:
268                    dump_obj = json.load(dump_file)
269                with vtable_dumper.VtableDumper(lib_path) as dumper:
270                    missing_symbols = self._DiffElfSymbols(
271                        dump_obj, dumper)
272                    vtable_diff = self._DiffVtableComponents(
273                        dump_obj, dumper)
274            except (IOError,
275                    elf_parser.ElfError,
276                    vtable_dumper.VtableError) as e:
277                logging.exception("%s: Cannot diff ABI", rel_path)
278                has_exception = True
279
280            if missing_symbols:
281                logging.error("%s: Missing Symbols:\n%s",
282                              rel_path, "\n".join(missing_symbols))
283            if vtable_diff:
284                logging.error("%s: Vtable Difference:\n"
285                              "vtable offset expected actual\n%s",
286                              rel_path,
287                              "\n".join(" ".join(e) for e in vtable_diff))
288            if (has_exception or missing_symbols or vtable_diff):
289                error_list.append(rel_path)
290            else:
291                logging.info("%s: Pass", rel_path)
292        return error_list
293
294    @staticmethod
295    def _GetLinkerSearchIndex(target_path):
296        """Returns the key for sorting linker search paths."""
297        index = 0
298        for prefix in ("/odm", "/vendor", "/apex"):
299            if target_path.startswith(prefix):
300                return index
301            index += 1
302        return index
303
304    def _TestAbiCompatibility(self, bitness):
305        """Checks ABI compliance of VNDK libraries.
306
307        Args:
308            bitness: 32 or 64, the bitness of the tested libraries.
309        """
310        self.assertTrue(self._dut.IsRoot(), "This test requires adb root.")
311        primary_abi = self._dut.GetCpuAbiList()[0]
312        binder_bitness = self._dut.GetBinderBitness()
313        self.assertTrue(binder_bitness, "Cannot determine binder bitness.")
314        dump_version = self._dut.GetVndkVersion()
315        self.assertTrue(dump_version, "Cannot determine VNDK version.")
316
317        dump_paths = vndk_data.GetAbiDumpPathsFromResources(
318            dump_version,
319            binder_bitness,
320            primary_abi,
321            bitness)
322        self.assertTrue(
323            dump_paths,
324            "No dump files. version: %s ABI: %s bitness: %d" % (
325                dump_version, primary_abi, bitness))
326
327        target_dirs = vndk_utils.GetVndkExtDirectories(bitness)
328        target_dirs += vndk_utils.GetVndkSpExtDirectories(bitness)
329        target_dirs += [vndk_utils.GetVndkDirectory(bitness, dump_version)]
330        target_dirs.sort(key=self._GetLinkerSearchIndex)
331
332        host_dirs = [self._ToHostPath(x) for x in target_dirs]
333        for target_dir, host_dir in zip(target_dirs, host_dirs):
334            self._PullOrCreateDir(target_dir, host_dir)
335
336        with vndk_data.AbiDumpResource() as dump_resource:
337            assert_lines = self._ScanLibDirs(dump_resource.zip_file,
338                                             dump_paths, host_dirs,
339                                             dump_version)
340
341        if assert_lines:
342            error_count = len(assert_lines)
343            if error_count > 20:
344                assert_lines = assert_lines[:20] + ["..."]
345            assert_lines.append("Total number of errors: " + str(error_count))
346            self.fail("\n".join(assert_lines))
347
348    def testAbiCompatibility32(self):
349        """Checks ABI compliance of 32-bit VNDK libraries."""
350        self._TestAbiCompatibility(32)
351
352    def testAbiCompatibility64(self):
353        """Checks ABI compliance of 64-bit VNDK libraries."""
354        if self._dut.GetCpuAbiList(64):
355            self._TestAbiCompatibility(64)
356        else:
357            logging.info("Skip the test as the device doesn't support 64-bit "
358                         "ABI.")
359
360
361if __name__ == "__main__":
362    # The logs are written to stdout so that TradeFed test runner can parse the
363    # results from stderr.
364    logging.basicConfig(stream=sys.stdout)
365    # Setting verbosity is required to generate output that the TradeFed test
366    # runner can parse.
367    unittest.main(verbosity=3)
368