1#!/usr/bin/env python3
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""binary_cache_builder.py: read perf.data, collect binaries needed by
19    it, and put them in binary_cache.
20"""
21
22from __future__ import print_function
23import argparse
24from dataclasses import dataclass
25import os
26import os.path
27from pathlib import Path
28import shutil
29from typing import List, Optional, Union
30
31from simpleperf_report_lib import ReportLib
32from simpleperf_utils import (AdbHelper, extant_dir, extant_file, flatten_arg_list, log_info,
33                              log_warning, ReadElf, set_log_level, str_to_bytes)
34
35
36def is_jit_symfile(dso_name):
37    return dso_name.split('/')[-1].startswith('TemporaryFile')
38
39
40class BinaryCacheBuilder(object):
41    """Collect all binaries needed by perf.data in binary_cache."""
42
43    def __init__(self, ndk_path: Optional[str], disable_adb_root: bool):
44        self.adb = AdbHelper(enable_switch_to_root=not disable_adb_root)
45        self.readelf = ReadElf(ndk_path)
46        self.binary_cache_dir = 'binary_cache'
47        if not os.path.isdir(self.binary_cache_dir):
48            os.makedirs(self.binary_cache_dir)
49        self.binaries = {}
50
51    def build_binary_cache(self, perf_data_path: str, symfs_dirs: List[Union[Path, str]]):
52        self.collect_used_binaries(perf_data_path)
53        self.copy_binaries_from_symfs_dirs(symfs_dirs)
54        if self.adb.is_device_available():
55            self.pull_binaries_from_device()
56            self._pull_kernel_symbols()
57        self.create_build_id_list()
58
59    def collect_used_binaries(self, perf_data_path):
60        """read perf.data, collect all used binaries and their build id (if available)."""
61        # A dict mapping from binary name to build_id
62        binaries = {}
63        lib = ReportLib()
64        lib.SetRecordFile(perf_data_path)
65        lib.SetLogSeverity('error')
66        while True:
67            sample = lib.GetNextSample()
68            if sample is None:
69                lib.Close()
70                break
71            symbols = [lib.GetSymbolOfCurrentSample()]
72            callchain = lib.GetCallChainOfCurrentSample()
73            for i in range(callchain.nr):
74                symbols.append(callchain.entries[i].symbol)
75
76            for symbol in symbols:
77                dso_name = symbol.dso_name
78                if dso_name not in binaries:
79                    if is_jit_symfile(dso_name):
80                        continue
81                    name = 'vmlinux' if dso_name == '[kernel.kallsyms]' else dso_name
82                    binaries[name] = lib.GetBuildIdForPath(dso_name)
83        self.binaries = binaries
84
85    def copy_binaries_from_symfs_dirs(self, symfs_dirs: List[Union[Path, str]]):
86        """collect all files in symfs_dirs."""
87        if not symfs_dirs:
88            return
89
90        # It is possible that the path of the binary in symfs_dirs doesn't match
91        # the one recorded in perf.data. For example, a file in symfs_dirs might
92        # be "debug/arm/obj/armeabi-v7a/libsudo-game-jni.so", but the path in
93        # perf.data is "/data/app/xxxx/lib/arm/libsudo-game-jni.so". So we match
94        # binaries if they have the same filename (like libsudo-game-jni.so)
95        # and same build_id.
96
97        # Map from filename to binary paths.
98        filename_dict = {}
99        for binary in self.binaries:
100            index = binary.rfind('/')
101            filename = binary[index+1:]
102            paths = filename_dict.get(filename)
103            if paths is None:
104                filename_dict[filename] = paths = []
105            paths.append(binary)
106
107        # Walk through all files in symfs_dirs, and copy matching files to build_cache.
108        for symfs_dir in symfs_dirs:
109            for root, _, files in os.walk(symfs_dir):
110                for filename in files:
111                    paths = filename_dict.get(filename)
112                    if not paths:
113                        continue
114                    build_id = self._read_build_id(os.path.join(root, filename))
115                    for binary in paths:
116                        expected_build_id = self.binaries.get(binary)
117                        if expected_build_id == build_id:
118                            self._copy_to_binary_cache(os.path.join(root, filename),
119                                                       expected_build_id, binary)
120                            break
121
122    def _copy_to_binary_cache(self, from_path, expected_build_id, target_file):
123        if target_file[0] == '/':
124            target_file = target_file[1:]
125        target_file = target_file.replace('/', os.sep)
126        target_file = os.path.join(self.binary_cache_dir, target_file)
127        if not self._need_to_copy(from_path, target_file, expected_build_id):
128            # The existing file in binary_cache can provide more information, so no need to copy.
129            return
130        target_dir = os.path.dirname(target_file)
131        if not os.path.isdir(target_dir):
132            os.makedirs(target_dir)
133        log_info('copy to binary_cache: %s to %s' % (from_path, target_file))
134        shutil.copy(from_path, target_file)
135
136    def _need_to_copy(self, source_file, target_file, expected_build_id):
137        if not os.path.isfile(target_file):
138            return True
139        if self._read_build_id(target_file) != expected_build_id:
140            return True
141        return self._get_file_stripped_level(source_file) < self._get_file_stripped_level(
142            target_file)
143
144    def _get_file_stripped_level(self, file_path):
145        """Return stripped level of an ELF file. Larger value means more stripped."""
146        sections = self.readelf.get_sections(file_path)
147        if '.debug_line' in sections:
148            return 0
149        if '.symtab' in sections:
150            return 1
151        return 2
152
153    def pull_binaries_from_device(self):
154        """pull binaries needed in perf.data to binary_cache."""
155        for binary in self.binaries:
156            build_id = self.binaries[binary]
157            if not binary.startswith('/') or binary == "//anon" or binary.startswith("/dev/"):
158                # [kernel.kallsyms] or unknown, or something we can't find binary.
159                continue
160            binary_cache_file = binary[1:].replace('/', os.sep)
161            binary_cache_file = os.path.join(self.binary_cache_dir, binary_cache_file)
162            self._check_and_pull_binary(binary, build_id, binary_cache_file)
163
164    def _check_and_pull_binary(self, binary, expected_build_id, binary_cache_file):
165        """If the binary_cache_file exists and has the expected_build_id, there
166           is no need to pull the binary from device. Otherwise, pull it.
167        """
168        need_pull = True
169        if os.path.isfile(binary_cache_file):
170            need_pull = False
171            if expected_build_id:
172                build_id = self._read_build_id(binary_cache_file)
173                if expected_build_id != build_id:
174                    need_pull = True
175        if need_pull:
176            target_dir = os.path.dirname(binary_cache_file)
177            if not os.path.isdir(target_dir):
178                os.makedirs(target_dir)
179            if os.path.isfile(binary_cache_file):
180                os.remove(binary_cache_file)
181            log_info('pull file to binary_cache: %s to %s' % (binary, binary_cache_file))
182            self._pull_file_from_device(binary, binary_cache_file)
183        else:
184            log_info('use current file in binary_cache: %s' % binary_cache_file)
185
186    def _read_build_id(self, file_path):
187        """read build id of a binary on host."""
188        return self.readelf.get_build_id(file_path)
189
190    def _pull_file_from_device(self, device_path, host_path):
191        if self.adb.run(['pull', device_path, host_path]):
192            return True
193        # In non-root device, we can't pull /data/app/XXX/base.odex directly.
194        # Instead, we can first copy the file to /data/local/tmp, then pull it.
195        filename = device_path[device_path.rfind('/')+1:]
196        if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and
197                self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])):
198            self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename])
199            return True
200        log_warning('failed to pull %s from device' % device_path)
201        return False
202
203    def _pull_kernel_symbols(self):
204        file_path = os.path.join(self.binary_cache_dir, 'kallsyms')
205        if os.path.isfile(file_path):
206            os.remove(file_path)
207        if self.adb.switch_to_root():
208            self.adb.run(['shell', 'echo', '0', '>/proc/sys/kernel/kptr_restrict'])
209            self.adb.run(['pull', '/proc/kallsyms', file_path])
210
211    def create_build_id_list(self):
212        """ Create build_id_list. So report scripts can find a binary by its build_id instead of
213            path.
214        """
215        build_id_list_path = os.path.join(self.binary_cache_dir, 'build_id_list')
216        with open(build_id_list_path, 'wb') as fh:
217            for root, _, files in os.walk(self.binary_cache_dir):
218                for filename in files:
219                    path = os.path.join(root, filename)
220                    relative_path = path[len(self.binary_cache_dir) + 1:]
221                    build_id = self._read_build_id(path)
222                    if build_id:
223                        line = f'{build_id}={relative_path}\n'
224                        fh.write(str_to_bytes(line))
225
226
227def main():
228    parser = argparse.ArgumentParser(description="""
229        Pull binaries needed by perf.data from device to binary_cache directory.""")
230    parser.add_argument('-i', '--perf_data_path', default='perf.data', type=extant_file, help="""
231        The path of profiling data.""")
232    parser.add_argument('-lib', '--native_lib_dir', type=extant_dir, nargs='+', help="""
233        Path to find debug version of native shared libraries used in the app.""", action='append')
234    parser.add_argument('--disable_adb_root', action='store_true', help="""
235        Force adb to run in non root mode.""")
236    parser.add_argument('--ndk_path', nargs=1, help='Find tools in the ndk path.')
237    parser.add_argument(
238        '--log', choices=['debug', 'info', 'warning'], default='info', help='set log level')
239    args = parser.parse_args()
240    set_log_level(args.log)
241    ndk_path = None if not args.ndk_path else args.ndk_path[0]
242    builder = BinaryCacheBuilder(ndk_path, args.disable_adb_root)
243    symfs_dirs = flatten_arg_list(args.native_lib_dir)
244    builder.build_binary_cache(args.perf_data_path, symfs_dirs)
245
246
247if __name__ == '__main__':
248    main()
249