1#!/usr/bin/env python
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import json
19import logging
20import os
21import shutil
22import tempfile
23
24from vts.runners.host import asserts
25from vts.runners.host import base_test
26from vts.runners.host import keys
27from vts.runners.host import test_runner
28from vts.runners.host import utils
29from vts.testcases.vndk.golden import vndk_data
30from vts.utils.python.file import target_file_utils
31from vts.utils.python.library import elf_parser
32from vts.utils.python.library.vtable import vtable_dumper
33from vts.utils.python.vndk import vndk_utils
34
35
36class VtsVndkAbiTest(base_test.BaseTestClass):
37    """A test module to verify ABI compliance of vendor libraries.
38
39    Attributes:
40        _dut: the AndroidDevice under test.
41        _temp_dir: The temporary directory for libraries copied from device.
42        _vndk_version: String, the VNDK version supported by the device.
43        data_file_path: The path to VTS data directory.
44    """
45
46    def setUpClass(self):
47        """Initializes data file path, device, and temporary directory."""
48        required_params = [keys.ConfigKeys.IKEY_DATA_FILE_PATH]
49        self.getUserParams(required_params)
50        self._dut = self.android_devices[0]
51        self._temp_dir = tempfile.mkdtemp()
52        self._vndk_version = self._dut.vndk_version
53
54    def tearDownClass(self):
55        """Deletes the temporary directory."""
56        logging.info("Delete %s", self._temp_dir)
57        shutil.rmtree(self._temp_dir)
58
59    def _PullOrCreateDir(self, target_dir, host_dir):
60        """Copies a directory from device. Creates an empty one if not exist.
61
62        Args:
63            target_dir: The directory to copy from device.
64            host_dir: The directory to copy to host.
65        """
66        if not target_file_utils.IsDirectory(target_dir, self._dut.shell):
67            logging.info("%s doesn't exist. Create %s.", target_dir, host_dir)
68            os.makedirs(host_dir)
69            return
70        parent_dir = os.path.dirname(host_dir)
71        if parent_dir and not os.path.isdir(parent_dir):
72            os.makedirs(parent_dir)
73        logging.info("adb pull %s %s", target_dir, host_dir)
74        self._dut.adb.pull(target_dir, host_dir)
75
76    def _ToHostPath(self, target_path):
77        """Maps target path to host path in self._temp_dir."""
78        return os.path.join(self._temp_dir, *target_path.strip("/").split("/"))
79
80    @staticmethod
81    def _LoadGlobalSymbolsFromDump(dump_obj):
82        """Loads global symbols from a dump object.
83
84        Args:
85            dump_obj: A dict, the dump in JSON format.
86
87        Returns:
88            A set of strings, the symbol names.
89        """
90        symbols = set()
91        for key in ("elf_functions", "elf_objects"):
92            symbols.update(
93                symbol.get("name", "") for symbol in dump_obj.get(key, []) if
94                symbol.get("binding", "global") == "global")
95        return symbols
96
97    def _DiffElfSymbols(self, dump_obj, parser):
98        """Checks if a library includes all symbols in a dump.
99
100        Args:
101            dump_obj: A dict, the dump in JSON format.
102            parser: An elf_parser.ElfParser that loads the library.
103
104        Returns:
105            A list of strings, the global symbols that are in the dump but not
106            in the library.
107
108        Raises:
109            elf_parser.ElfError if fails to load the library.
110        """
111        dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
112        lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True)
113        return sorted(dump_symbols.difference(lib_symbols))
114
115    @staticmethod
116    def _DiffVtableComponent(offset, expected_symbol, vtable):
117        """Checks if a symbol is in a vtable entry.
118
119        Args:
120            offset: An integer, the offset of the expected symbol.
121            exepcted_symbol: A string, the name of the expected symbol.
122            vtable: A dict of {offset: [entry]} where offset is an integer and
123                    entry is an instance of vtable_dumper.VtableEntry.
124
125        Returns:
126            A list of strings, the actual possible symbols if expected_symbol
127            does not match the vtable entry.
128            None if expected_symbol matches the entry.
129        """
130        if offset not in vtable:
131            return []
132
133        entry = vtable[offset]
134        if not entry.names:
135            return [hex(entry.value).rstrip('L')]
136
137        if expected_symbol not in entry.names:
138            return entry.names
139
140    def _DiffVtableComponents(self, dump_obj, dumper):
141        """Checks if a library includes all vtable entries in a dump.
142
143        Args:
144            dump_obj: A dict, the dump in JSON format.
145            dumper: An vtable_dumper.VtableDumper that loads the library.
146
147        Returns:
148            A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL).
149            ACTUAL can be "missing", a list of symbol names, or an ELF virtual
150            address.
151
152        Raises:
153            vtable_dumper.VtableError if fails to dump vtable from the library.
154        """
155        function_kinds = [
156            "function_pointer",
157            "complete_dtor_pointer",
158            "deleting_dtor_pointer"
159        ]
160        non_function_kinds = [
161            "vcall_offset",
162            "vbase_offset",
163            "offset_to_top",
164            "rtti",
165            "unused_function_pointer"
166        ]
167        default_vtable_component_kind = "function_pointer"
168
169        global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
170
171        lib_vtables = {vtable.name: vtable
172                       for vtable in dumper.DumpVtables()}
173        logging.debug("\n\n".join(str(vtable)
174                                  for _, vtable in lib_vtables.iteritems()))
175
176        vtables_diff = []
177        for record_type in dump_obj.get("record_types", []):
178            # Since Android R, unique_id has been replaced with linker_set_key.
179            # unique_id starts with "_ZTI"; linker_set_key starts with "_ZTS".
180            type_name_symbol = record_type.get("unique_id", "")
181            if type_name_symbol:
182                vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1)
183            else:
184                type_name_symbol = record_type.get("linker_set_key", "")
185                vtable_symbol = type_name_symbol.replace("_ZTI", "_ZTV", 1)
186
187            # Skip if the vtable symbol isn't global.
188            if vtable_symbol not in global_symbols:
189                continue
190
191            # Collect vtable entries from library dump.
192            if vtable_symbol in lib_vtables:
193                lib_vtable = {entry.offset: entry
194                              for entry in lib_vtables[vtable_symbol].entries}
195            else:
196                lib_vtable = dict()
197
198            for index, entry in enumerate(record_type.get("vtable_components",
199                                                          [])):
200                entry_offset = index * int(self.abi_bitness) // 8
201                entry_kind = entry.get("kind", default_vtable_component_kind)
202                entry_symbol = entry.get("mangled_component_name", "")
203                entry_is_pure = entry.get("is_pure", False)
204
205                if entry_kind in non_function_kinds:
206                    continue
207
208                if entry_kind not in function_kinds:
209                    logging.warning("%s: Unexpected vtable entry kind %s",
210                                    vtable_symbol, entry_kind)
211
212                if entry_symbol not in global_symbols:
213                    # Itanium cxx abi doesn't specify pure virtual vtable
214                    # entry's behaviour. However we can still do some checks
215                    # based on compiler behaviour.
216                    # Even though we don't check weak symbols, we can still
217                    # issue a warning when a pure virtual function pointer
218                    # is missing.
219                    if entry_is_pure and entry_offset not in lib_vtable:
220                        logging.warning("%s: Expected pure virtual function"
221                                        "in %s offset %s",
222                                        vtable_symbol, vtable_symbol,
223                                        entry_offset)
224                    continue
225
226                diff_symbols = self._DiffVtableComponent(
227                    entry_offset, entry_symbol, lib_vtable)
228                if diff_symbols is None:
229                    continue
230
231                vtables_diff.append(
232                    (vtable_symbol, str(entry_offset), entry_symbol,
233                     (",".join(diff_symbols) if diff_symbols else "missing")))
234
235        return vtables_diff
236
237    def _ScanLibDirs(self, dump_dir, lib_dirs, dump_version):
238        """Compares dump files with libraries copied from device.
239
240        Args:
241            dump_dir: The directory containing dump files.
242            lib_dirs: The list of directories containing libraries.
243            dump_version: The VNDK version of the dump files. If the device has
244                          no VNDK version or has extension in vendor partition,
245                          this method compares the unversioned VNDK directories
246                          with the dump directories of the given version.
247
248        Returns:
249            A list of strings, the incompatible libraries.
250        """
251        error_list = []
252        dump_paths = dict()
253        lib_paths = dict()
254        for parent_dir, dump_name in utils.iterate_files(dump_dir):
255            dump_path = os.path.join(parent_dir, dump_name)
256            if dump_path.endswith(".dump"):
257                lib_name = dump_name.rpartition(".dump")[0]
258                dump_paths[lib_name] = dump_path
259            else:
260                logging.warning("Unknown dump: %s", dump_path)
261
262        for lib_dir in lib_dirs:
263            for parent_dir, lib_name in utils.iterate_files(lib_dir):
264                if lib_name not in lib_paths:
265                    lib_paths[lib_name] = os.path.join(parent_dir, lib_name)
266
267        for lib_name, dump_path in dump_paths.iteritems():
268            if lib_name not in lib_paths:
269                logging.info("%s: Not found on target", lib_name)
270                continue
271            lib_path = lib_paths[lib_name]
272            rel_path = os.path.relpath(lib_path, self._temp_dir)
273
274            has_exception = False
275            missing_symbols = []
276            vtable_diff = []
277
278            try:
279                with open(dump_path, "r") as dump_file:
280                    dump_obj = json.load(dump_file)
281                with vtable_dumper.VtableDumper(lib_path) as dumper:
282                    missing_symbols = self._DiffElfSymbols(
283                        dump_obj, dumper)
284                    vtable_diff = self._DiffVtableComponents(
285                        dump_obj, dumper)
286            except (IOError,
287                    elf_parser.ElfError,
288                    vtable_dumper.VtableError) as e:
289                logging.exception("%s: Cannot diff ABI", rel_path)
290                has_exception = True
291
292            if missing_symbols:
293                logging.error("%s: Missing Symbols:\n%s",
294                              rel_path, "\n".join(missing_symbols))
295            if vtable_diff:
296                logging.error("%s: Vtable Difference:\n"
297                              "vtable offset expected actual\n%s",
298                              rel_path,
299                              "\n".join(" ".join(e) for e in vtable_diff))
300            if (has_exception or missing_symbols or vtable_diff):
301                error_list.append(rel_path)
302            else:
303                logging.info("%s: Pass", rel_path)
304        return error_list
305
306    @staticmethod
307    def _GetLinkerSearchIndex(target_path):
308        """Returns the key for sorting linker search paths."""
309        index = 0
310        for prefix in ("/odm", "/vendor", "/apex"):
311            if target_path.startswith(prefix):
312                return index
313            index += 1
314        return index
315
316    def testAbiCompatibility(self):
317        """Checks ABI compliance of VNDK libraries."""
318        primary_abi = self._dut.getCpuAbiList()[0]
319        binder_bitness = self._dut.getBinderBitness()
320        asserts.assertTrue(binder_bitness,
321                           "Cannot determine binder bitness.")
322        dump_version = (self._vndk_version if self._vndk_version else
323                        vndk_data.LoadDefaultVndkVersion(self.data_file_path))
324        asserts.assertTrue(dump_version,
325                           "Cannot load default VNDK version.")
326
327        dump_dir = vndk_data.GetAbiDumpDirectory(
328            self.data_file_path,
329            dump_version,
330            binder_bitness,
331            primary_abi,
332            self.abi_bitness)
333        asserts.assertTrue(
334            dump_dir,
335            "No dump files. version: %s ABI: %s bitness: %s" % (
336                self._vndk_version, primary_abi, self.abi_bitness))
337        logging.info("dump dir: %s", dump_dir)
338
339        target_dirs = vndk_utils.GetVndkExtDirectories(self.abi_bitness)
340        target_dirs += vndk_utils.GetVndkSpExtDirectories(self.abi_bitness)
341        target_dirs += [vndk_utils.GetVndkDirectory(self.abi_bitness,
342                                                    self._vndk_version)]
343        target_dirs.sort(key=self._GetLinkerSearchIndex)
344
345        host_dirs = [self._ToHostPath(x) for x in target_dirs]
346        for target_dir, host_dir in zip(target_dirs, host_dirs):
347            self._PullOrCreateDir(target_dir, host_dir)
348
349        assert_lines = self._ScanLibDirs(dump_dir, host_dirs, dump_version)
350        if assert_lines:
351            error_count = len(assert_lines)
352            if error_count > 20:
353                assert_lines = assert_lines[:20] + ["..."]
354            assert_lines.append("Total number of errors: " + str(error_count))
355            asserts.fail("\n".join(assert_lines))
356
357
358if __name__ == "__main__":
359    test_runner.main()
360