1#!/usr/bin/env python
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import json
19import logging
20import os
21import shutil
22import tempfile
23
24from vts.runners.host import asserts
25from vts.runners.host import base_test
26from vts.runners.host import keys
27from vts.runners.host import test_runner
28from vts.runners.host import utils
29from vts.testcases.vndk.golden import vndk_data
30from vts.utils.python.file import target_file_utils
31from vts.utils.python.library import elf_parser
32from vts.utils.python.library.vtable import vtable_dumper
33from vts.utils.python.vndk import vndk_utils
34
35
36class VtsVndkAbiTest(base_test.BaseTestClass):
37    """A test module to verify ABI compliance of vendor libraries.
38
39    Attributes:
40        _dut: the AndroidDevice under test.
41        _temp_dir: The temporary directory for libraries copied from device.
42        _vndk_version: String, the VNDK version supported by the device.
43        data_file_path: The path to VTS data directory.
44    """
45
46    def setUpClass(self):
47        """Initializes data file path, device, and temporary directory."""
48        required_params = [keys.ConfigKeys.IKEY_DATA_FILE_PATH]
49        self.getUserParams(required_params)
50        self._dut = self.android_devices[0]
51        self._temp_dir = tempfile.mkdtemp()
52        self._vndk_version = self._dut.vndk_version
53
54    def tearDownClass(self):
55        """Deletes the temporary directory."""
56        logging.info("Delete %s", self._temp_dir)
57        shutil.rmtree(self._temp_dir)
58
59    def _PullOrCreateDir(self, target_dir, host_dir):
60        """Copies a directory from device. Creates an empty one if not exist.
61
62        Args:
63            target_dir: The directory to copy from device.
64            host_dir: The directory to copy to host.
65        """
66        if not target_file_utils.IsDirectory(target_dir, self._dut.shell):
67            logging.info("%s doesn't exist. Create %s.", target_dir, host_dir)
68            os.makedirs(host_dir)
69            return
70        parent_dir = os.path.dirname(host_dir)
71        if parent_dir and not os.path.isdir(parent_dir):
72            os.makedirs(parent_dir)
73        logging.info("adb pull %s %s", target_dir, host_dir)
74        self._dut.adb.pull(target_dir, host_dir)
75
76    def _ToHostPath(self, target_path):
77        """Maps target path to host path in self._temp_dir."""
78        return os.path.join(self._temp_dir, *target_path.strip("/").split("/"))
79
80    @staticmethod
81    def _LoadGlobalSymbolsFromDump(dump_obj):
82        """Loads global symbols from a dump object.
83
84        Args:
85            dump_obj: A dict, the dump in JSON format.
86
87        Returns:
88            A set of strings, the symbol names.
89        """
90        symbols = set()
91        for key in ("elf_functions", "elf_objects"):
92            symbols.update(
93                symbol.get("name", "") for symbol in dump_obj.get(key, []) if
94                symbol.get("binding", "global") == "global")
95        return symbols
96
97    def _DiffElfSymbols(self, dump_obj, parser):
98        """Checks if a library includes all symbols in a dump.
99
100        Args:
101            dump_obj: A dict, the dump in JSON format.
102            parser: An elf_parser.ElfParser that loads the library.
103
104        Returns:
105            A list of strings, the global symbols that are in the dump but not
106            in the library.
107
108        Raises:
109            elf_parser.ElfError if fails to load the library.
110        """
111        dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
112        lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True)
113        return sorted(dump_symbols.difference(lib_symbols))
114
115    @staticmethod
116    def _DiffVtableComponent(offset, expected_symbol, vtable):
117        """Checks if a symbol is in a vtable entry.
118
119        Args:
120            offset: An integer, the offset of the expected symbol.
121            exepcted_symbol: A string, the name of the expected symbol.
122            vtable: A dict of {offset: [entry]} where offset is an integer and
123                    entry is an instance of vtable_dumper.VtableEntry.
124
125        Returns:
126            A list of strings, the actual possible symbols if expected_symbol
127            does not match the vtable entry.
128            None if expected_symbol matches the entry.
129        """
130        if offset not in vtable:
131            return []
132
133        entry = vtable[offset]
134        if not entry.names:
135            return [hex(entry.value).rstrip('L')]
136
137        if expected_symbol not in entry.names:
138            return entry.names
139
140    def _DiffVtableComponents(self, dump_obj, dumper):
141        """Checks if a library includes all vtable entries in a dump.
142
143        Args:
144            dump_obj: A dict, the dump in JSON format.
145            dumper: An vtable_dumper.VtableDumper that loads the library.
146
147        Returns:
148            A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL).
149            ACTUAL can be "missing", a list of symbol names, or an ELF virtual
150            address.
151
152        Raises:
153            vtable_dumper.VtableError if fails to dump vtable from the library.
154        """
155        function_kinds = [
156            "function_pointer",
157            "complete_dtor_pointer",
158            "deleting_dtor_pointer"
159        ]
160        non_function_kinds = [
161            "vcall_offset",
162            "vbase_offset",
163            "offset_to_top",
164            "rtti",
165            "unused_function_pointer"
166        ]
167        default_vtable_component_kind = "function_pointer"
168
169        global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj)
170
171        lib_vtables = {vtable.name: vtable
172                       for vtable in dumper.DumpVtables()}
173        logging.debug("\n\n".join(str(vtable)
174                                  for _, vtable in lib_vtables.iteritems()))
175
176        vtables_diff = []
177        for record_type in dump_obj.get("record_types", []):
178            type_name_symbol = record_type.get("unique_id", "")
179            vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1)
180
181            # Skip if the vtable symbol isn't global.
182            if vtable_symbol not in global_symbols:
183                continue
184
185            # Collect vtable entries from library dump.
186            if vtable_symbol in lib_vtables:
187                lib_vtable = {entry.offset: entry
188                              for entry in lib_vtables[vtable_symbol].entries}
189            else:
190                lib_vtable = dict()
191
192            for index, entry in enumerate(record_type.get("vtable_components",
193                                                          [])):
194                entry_offset = index * int(self.abi_bitness) // 8
195                entry_kind = entry.get("kind", default_vtable_component_kind)
196                entry_symbol = entry.get("mangled_component_name", "")
197                entry_is_pure = entry.get("is_pure", False)
198
199                if entry_kind in non_function_kinds:
200                    continue
201
202                if entry_kind not in function_kinds:
203                    logging.warning("%s: Unexpected vtable entry kind %s",
204                                    vtable_symbol, entry_kind)
205
206                if entry_symbol not in global_symbols:
207                    # Itanium cxx abi doesn't specify pure virtual vtable
208                    # entry's behaviour. However we can still do some checks
209                    # based on compiler behaviour.
210                    # Even though we don't check weak symbols, we can still
211                    # issue a warning when a pure virtual function pointer
212                    # is missing.
213                    if entry_is_pure and entry_offset not in lib_vtable:
214                        logging.warning("%s: Expected pure virtual function"
215                                        "in %s offset %s",
216                                        vtable_symbol, vtable_symbol,
217                                        entry_offset)
218                    continue
219
220                diff_symbols = self._DiffVtableComponent(
221                    entry_offset, entry_symbol, lib_vtable)
222                if diff_symbols is None:
223                    continue
224
225                vtables_diff.append(
226                    (vtable_symbol, str(entry_offset), entry_symbol,
227                     (",".join(diff_symbols) if diff_symbols else "missing")))
228
229        return vtables_diff
230
231    def _ScanLibDirs(self, dump_dir, lib_dirs, dump_version):
232        """Compares dump files with libraries copied from device.
233
234        Args:
235            dump_dir: The directory containing dump files.
236            lib_dirs: The list of directories containing libraries.
237            dump_version: The VNDK version of the dump files. If the device has
238                          no VNDK version or has extension in vendor partition,
239                          this method compares the unversioned VNDK directories
240                          with the dump directories of the given version.
241
242        Returns:
243            An integer, number of incompatible libraries.
244        """
245        error_count = 0
246        dump_paths = dict()
247        lib_paths = dict()
248        for parent_dir, dump_name in utils.iterate_files(dump_dir):
249            dump_path = os.path.join(parent_dir, dump_name)
250            if dump_path.endswith(".dump"):
251                lib_name = dump_name.rpartition(".dump")[0]
252                dump_paths[lib_name] = dump_path
253            else:
254                logging.warning("Unknown dump: %s", dump_path)
255
256        for lib_dir in lib_dirs:
257            for parent_dir, lib_name in utils.iterate_files(lib_dir):
258                if lib_name not in lib_paths:
259                    lib_paths[lib_name] = os.path.join(parent_dir, lib_name)
260
261        for lib_name, dump_path in dump_paths.iteritems():
262            if lib_name not in lib_paths:
263                logging.info("%s: Not found on target", lib_name)
264                continue
265            lib_path = lib_paths[lib_name]
266            rel_path = os.path.relpath(lib_path, self._temp_dir)
267
268            has_exception = False
269            missing_symbols = []
270            vtable_diff = []
271
272            try:
273                with open(dump_path, "r") as dump_file:
274                    dump_obj = json.load(dump_file)
275                with vtable_dumper.VtableDumper(lib_path) as dumper:
276                    missing_symbols = self._DiffElfSymbols(
277                        dump_obj, dumper)
278                    vtable_diff = self._DiffVtableComponents(
279                        dump_obj, dumper)
280            except (IOError,
281                    elf_parser.ElfError,
282                    vtable_dumper.VtableError) as e:
283                logging.exception("%s: Cannot diff ABI", rel_path)
284                has_exception = True
285
286            if missing_symbols:
287                logging.error("%s: Missing Symbols:\n%s",
288                              rel_path, "\n".join(missing_symbols))
289            if vtable_diff:
290                logging.error("%s: Vtable Difference:\n"
291                              "vtable offset expected actual\n%s",
292                              rel_path,
293                              "\n".join(" ".join(e) for e in vtable_diff))
294            if (has_exception or missing_symbols or vtable_diff):
295                error_count += 1
296            else:
297                logging.info("%s: Pass", rel_path)
298        return error_count
299
300    @staticmethod
301    def _GetLinkerSearchIndex(target_path):
302        """Returns the key for sorting linker search paths."""
303        index = 0
304        for prefix in ("/odm", "/vendor", "/system"):
305            if target_path.startswith(prefix):
306                return index
307            index += 1
308        return index
309
310    def testAbiCompatibility(self):
311        """Checks ABI compliance of VNDK libraries."""
312        primary_abi = self._dut.getCpuAbiList()[0]
313        binder_bitness = self._dut.getBinderBitness()
314        asserts.assertTrue(binder_bitness,
315                           "Cannot determine binder bitness.")
316        dump_version = (self._vndk_version if self._vndk_version else
317                        vndk_data.LoadDefaultVndkVersion(self.data_file_path))
318        asserts.assertTrue(dump_version,
319                           "Cannot load default VNDK version.")
320
321        dump_dir = vndk_data.GetAbiDumpDirectory(
322            self.data_file_path,
323            dump_version,
324            binder_bitness,
325            primary_abi,
326            self.abi_bitness)
327        asserts.assertTrue(
328            dump_dir,
329            "No dump files. version: %s ABI: %s bitness: %s" % (
330                self._vndk_version, primary_abi, self.abi_bitness))
331        logging.info("dump dir: %s", dump_dir)
332
333        target_vndk_dir = vndk_utils.GetVndkCoreDirectory(self.abi_bitness,
334                                                          self._vndk_version)
335        target_vndk_sp_dir = vndk_utils.GetVndkSpDirectory(self.abi_bitness,
336                                                           self._vndk_version)
337        target_dirs = vndk_utils.GetVndkExtDirectories(self.abi_bitness)
338        target_dirs += vndk_utils.GetVndkSpExtDirectories(self.abi_bitness)
339        target_dirs += [target_vndk_dir, target_vndk_sp_dir]
340        target_dirs.sort(key=self._GetLinkerSearchIndex)
341
342        host_dirs = [self._ToHostPath(x) for x in target_dirs]
343        for target_dir, host_dir in zip(target_dirs, host_dirs):
344            self._PullOrCreateDir(target_dir, host_dir)
345
346        error_count = self._ScanLibDirs(dump_dir, host_dirs, dump_version)
347        asserts.assertEqual(error_count, 0,
348                            "Total number of errors: " + str(error_count))
349
350
351if __name__ == "__main__":
352    test_runner.main()
353