1#!/usr/bin/env python3
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""app_profiler.py: Record cpu profiling data of an android app or native program.
19
20    It downloads simpleperf on device, uses it to collect profiling data on the selected app,
21    and pulls profiling data and related binaries on host.
22"""
23
24from __future__ import print_function
25import argparse
26import os
27import os.path
28import subprocess
29import sys
30import time
31
32from simpleperf_utils import (
33    AdbHelper, bytes_to_str, extant_dir, get_script_dir, get_target_binary_path, log_debug,
34    log_info, log_exit, ReadElf, remove, set_log_level, str_to_bytes)
35
36NATIVE_LIBS_DIR_ON_DEVICE = '/data/local/tmp/native_libs/'
37
38
39class HostElfEntry(object):
40    """Represent a native lib on host in NativeLibDownloader."""
41
42    def __init__(self, path, name, score):
43        self.path = path
44        self.name = name
45        self.score = score
46
47    def __repr__(self):
48        return self.__str__()
49
50    def __str__(self):
51        return '[path: %s, name %s, score %s]' % (self.path, self.name, self.score)
52
53
54class NativeLibDownloader(object):
55    """Download native libs on device.
56
57    1. Collect info of all native libs in the native_lib_dir on host.
58    2. Check the available native libs in /data/local/tmp/native_libs on device.
59    3. Sync native libs on device.
60    """
61
62    def __init__(self, ndk_path, device_arch, adb):
63        self.adb = adb
64        self.readelf = ReadElf(ndk_path)
65        self.device_arch = device_arch
66        self.need_archs = self._get_need_archs()
67        self.host_build_id_map = {}  # Map from build_id to HostElfEntry.
68        self.device_build_id_map = {}  # Map from build_id to relative_path on device.
69        # Map from filename to HostElfEntry for elf files without build id.
70        self.no_build_id_file_map = {}
71        self.name_count_map = {}  # Used to give a unique name for each library.
72        self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
73        self.build_id_list_file = 'build_id_list'
74
75    def _get_need_archs(self):
76        """Return the archs of binaries needed on device."""
77        if self.device_arch == 'arm64':
78            return ['arm', 'arm64']
79        if self.device_arch == 'arm':
80            return ['arm']
81        if self.device_arch == 'x86_64':
82            return ['x86', 'x86_64']
83        if self.device_arch == 'x86':
84            return ['x86']
85        return []
86
87    def collect_native_libs_on_host(self, native_lib_dir):
88        self.host_build_id_map.clear()
89        for root, _, files in os.walk(native_lib_dir):
90            for name in files:
91                if not name.endswith('.so'):
92                    continue
93                self.add_native_lib_on_host(os.path.join(root, name), name)
94
95    def add_native_lib_on_host(self, path, name):
96        arch = self.readelf.get_arch(path)
97        if arch not in self.need_archs:
98            return
99        sections = self.readelf.get_sections(path)
100        score = 0
101        if '.debug_info' in sections:
102            score = 3
103        elif '.gnu_debugdata' in sections:
104            score = 2
105        elif '.symtab' in sections:
106            score = 1
107        build_id = self.readelf.get_build_id(path)
108        if build_id:
109            entry = self.host_build_id_map.get(build_id)
110            if entry:
111                if entry.score < score:
112                    entry.path = path
113                    entry.score = score
114            else:
115                repeat_count = self.name_count_map.get(name, 0)
116                self.name_count_map[name] = repeat_count + 1
117                unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
118                self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)
119        else:
120            entry = self.no_build_id_file_map.get(name)
121            if entry:
122                if entry.score < score:
123                    entry.path = path
124                    entry.score = score
125            else:
126                self.no_build_id_file_map[name] = HostElfEntry(path, name, score)
127
128    def collect_native_libs_on_device(self):
129        self.device_build_id_map.clear()
130        self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
131        if os.path.exists(self.build_id_list_file):
132            os.remove(self.build_id_list_file)
133        result, output = self.adb.run_and_return_output(['shell', 'ls', self.dir_on_device])
134        if not result:
135            return
136        file_set = set(output.strip().split())
137        if self.build_id_list_file not in file_set:
138            return
139        self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
140        if os.path.exists(self.build_id_list_file):
141            with open(self.build_id_list_file, 'rb') as fh:
142                for line in fh.readlines():
143                    line = bytes_to_str(line).strip()
144                    items = line.split('=')
145                    if len(items) == 2:
146                        build_id, filename = items
147                        if filename in file_set:
148                            self.device_build_id_map[build_id] = filename
149            remove(self.build_id_list_file)
150
151    def sync_native_libs_on_device(self):
152        # Push missing native libs on device.
153        for build_id in self.host_build_id_map:
154            if build_id not in self.device_build_id_map:
155                entry = self.host_build_id_map[build_id]
156                self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
157        # Remove native libs not exist on host.
158        for build_id in self.device_build_id_map:
159            if build_id not in self.host_build_id_map:
160                name = self.device_build_id_map[build_id]
161                self.adb.run(['shell', 'rm', self.dir_on_device + name])
162        # Push new build_id_list on device.
163        with open(self.build_id_list_file, 'wb') as fh:
164            for build_id in self.host_build_id_map:
165                s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
166                fh.write(s)
167        self.adb.check_run(['push', self.build_id_list_file,
168                            self.dir_on_device + self.build_id_list_file])
169        os.remove(self.build_id_list_file)
170
171        # Push elf files without build id on device.
172        for entry in self.no_build_id_file_map.values():
173            target = self.dir_on_device + entry.name
174
175            # Skip download if we have a file with the same name and size on device.
176            result, output = self.adb.run_and_return_output(['shell', 'ls', '-l', target])
177            if result:
178                items = output.split()
179                if len(items) > 5:
180                    try:
181                        file_size = int(items[4])
182                    except ValueError:
183                        file_size = 0
184                    if file_size == os.path.getsize(entry.path):
185                        continue
186            self.adb.check_run(['push', entry.path, target])
187
188
189class ProfilerBase(object):
190    """Base class of all Profilers."""
191
192    def __init__(self, args):
193        self.args = args
194        self.adb = AdbHelper(enable_switch_to_root=not args.disable_adb_root)
195        self.is_root_device = self.adb.switch_to_root()
196        self.android_version = self.adb.get_android_version()
197        if self.android_version < 7:
198            log_exit("""app_profiler.py isn't supported on Android < N, please switch to use
199                        simpleperf binary directly.""")
200        self.device_arch = self.adb.get_device_arch()
201        self.record_subproc = None
202
203    def profile(self):
204        log_info('prepare profiling')
205        self.prepare()
206        log_info('start profiling')
207        self.start()
208        self.wait_profiling()
209        log_info('collect profiling data')
210        self.collect_profiling_data()
211        log_info('profiling is finished.')
212
213    def prepare(self):
214        """Prepare recording. """
215        self.download_simpleperf()
216        if self.args.native_lib_dir:
217            self.download_libs()
218
219    def download_simpleperf(self):
220        simpleperf_binary = get_target_binary_path(self.device_arch, 'simpleperf')
221        self.adb.check_run(['push', simpleperf_binary, '/data/local/tmp'])
222        self.adb.check_run(['shell', 'chmod', 'a+x', '/data/local/tmp/simpleperf'])
223
224    def download_libs(self):
225        downloader = NativeLibDownloader(self.args.ndk_path, self.device_arch, self.adb)
226        downloader.collect_native_libs_on_host(self.args.native_lib_dir)
227        downloader.collect_native_libs_on_device()
228        downloader.sync_native_libs_on_device()
229
230    def start(self):
231        raise NotImplementedError
232
233    def start_profiling(self, target_args):
234        """Start simpleperf reocrd process on device."""
235        args = ['/data/local/tmp/simpleperf', 'record', '-o', '/data/local/tmp/perf.data',
236                self.args.record_options]
237        if self.adb.run(['shell', 'ls', NATIVE_LIBS_DIR_ON_DEVICE]):
238            args += ['--symfs', NATIVE_LIBS_DIR_ON_DEVICE]
239        args += ['--log', self.args.log]
240        args += target_args
241        adb_args = [self.adb.adb_path, 'shell'] + args
242        log_info('run adb cmd: %s' % adb_args)
243        self.record_subproc = subprocess.Popen(adb_args)
244
245    def wait_profiling(self):
246        """Wait until profiling finishes, or stop profiling when user presses Ctrl-C."""
247        returncode = None
248        try:
249            returncode = self.record_subproc.wait()
250        except KeyboardInterrupt:
251            self.stop_profiling()
252            self.record_subproc = None
253            # Don't check return value of record_subproc. Because record_subproc also
254            # receives Ctrl-C, and always returns non-zero.
255            returncode = 0
256        log_debug('profiling result [%s]' % (returncode == 0))
257        if returncode != 0:
258            log_exit('Failed to record profiling data.')
259
260    def stop_profiling(self):
261        """Stop profiling by sending SIGINT to simpleperf, and wait until it exits
262           to make sure perf.data is completely generated."""
263        has_killed = False
264        while True:
265            (result, _) = self.adb.run_and_return_output(['shell', 'pidof', 'simpleperf'])
266            if not result:
267                break
268            if not has_killed:
269                has_killed = True
270                self.adb.run_and_return_output(['shell', 'pkill', '-l', '2', 'simpleperf'])
271            time.sleep(1)
272
273    def collect_profiling_data(self):
274        self.adb.check_run_and_return_output(['pull', '/data/local/tmp/perf.data',
275                                              self.args.perf_data_path])
276        if not self.args.skip_collect_binaries:
277            binary_cache_args = [sys.executable,
278                                 os.path.join(get_script_dir(), 'binary_cache_builder.py')]
279            binary_cache_args += ['-i', self.args.perf_data_path, '--log', self.args.log]
280            if self.args.native_lib_dir:
281                binary_cache_args += ['-lib', self.args.native_lib_dir]
282            if self.args.disable_adb_root:
283                binary_cache_args += ['--disable_adb_root']
284            if self.args.ndk_path:
285                binary_cache_args += ['--ndk_path', self.args.ndk_path]
286            subprocess.check_call(binary_cache_args)
287
288
289class AppProfiler(ProfilerBase):
290    """Profile an Android app."""
291
292    def prepare(self):
293        super(AppProfiler, self).prepare()
294        if self.args.compile_java_code:
295            self.compile_java_code()
296
297    def compile_java_code(self):
298        self.kill_app_process()
299        # Fully compile Java code on Android >= N.
300        self.adb.set_property('debug.generate-debug-info', 'true')
301        self.adb.check_run(['shell', 'cmd', 'package', 'compile', '-f', '-m', 'speed',
302                            self.args.app])
303
304    def kill_app_process(self):
305        if self.find_app_process():
306            self.adb.check_run(['shell', 'am', 'force-stop', self.args.app])
307            count = 0
308            while True:
309                time.sleep(1)
310                pid = self.find_app_process()
311                if not pid:
312                    break
313                # When testing on Android N, `am force-stop` sometimes can't kill
314                # com.example.simpleperf.simpleperfexampleofkotlin. So use kill when this happens.
315                count += 1
316                if count >= 3:
317                    self.run_in_app_dir(['kill', '-9', str(pid)])
318
319    def find_app_process(self):
320        result, output = self.adb.run_and_return_output(['shell', 'pidof', self.args.app])
321        return int(output) if result else None
322
323    def run_in_app_dir(self, args):
324        if self.is_root_device:
325            adb_args = ['shell', 'cd /data/data/' + self.args.app + ' && ' + (' '.join(args))]
326        else:
327            adb_args = ['shell', 'run-as', self.args.app] + args
328        return self.adb.run_and_return_output(adb_args)
329
330    def start(self):
331        if self.args.activity or self.args.test:
332            self.kill_app_process()
333        self.start_profiling(['--app', self.args.app])
334        if self.args.activity:
335            self.start_activity()
336        elif self.args.test:
337            self.start_test()
338        # else: no need to start an activity or test.
339
340    def start_activity(self):
341        activity = self.args.app + '/' + self.args.activity
342        result = self.adb.run(['shell', 'am', 'start', '-n', activity])
343        if not result:
344            self.record_subproc.terminate()
345            log_exit("Can't start activity %s" % activity)
346
347    def start_test(self):
348        runner = self.args.app + '/androidx.test.runner.AndroidJUnitRunner'
349        result = self.adb.run(['shell', 'am', 'instrument', '-e', 'class',
350                               self.args.test, runner])
351        if not result:
352            self.record_subproc.terminate()
353            log_exit("Can't start instrumentation test  %s" % self.args.test)
354
355
356class NativeProgramProfiler(ProfilerBase):
357    """Profile a native program."""
358
359    def start(self):
360        log_info('Waiting for native process %s' % self.args.native_program)
361        while True:
362            (result, pid) = self.adb.run_and_return_output(['shell', 'pidof',
363                                                            self.args.native_program])
364            if not result:
365                # Wait for 1 millisecond.
366                time.sleep(0.001)
367            else:
368                self.start_profiling(['-p', str(int(pid))])
369                break
370
371
372class NativeCommandProfiler(ProfilerBase):
373    """Profile running a native command."""
374
375    def start(self):
376        self.start_profiling([self.args.cmd])
377
378
379class NativeProcessProfiler(ProfilerBase):
380    """Profile processes given their pids."""
381
382    def start(self):
383        self.start_profiling(['-p', ','.join(self.args.pid)])
384
385
386class NativeThreadProfiler(ProfilerBase):
387    """Profile threads given their tids."""
388
389    def start(self):
390        self.start_profiling(['-t', ','.join(self.args.tid)])
391
392
393class SystemWideProfiler(ProfilerBase):
394    """Profile system wide."""
395
396    def start(self):
397        self.start_profiling(['-a'])
398
399
400def main():
401    parser = argparse.ArgumentParser(description=__doc__,
402                                     formatter_class=argparse.RawDescriptionHelpFormatter)
403
404    target_group = parser.add_argument_group(title='Select profiling target'
405                                             ).add_mutually_exclusive_group(required=True)
406    target_group.add_argument('-p', '--app', help="""Profile an Android app, given the package name.
407                              Like `-p com.example.android.myapp`.""")
408
409    target_group.add_argument('-np', '--native_program', help="""Profile a native program running on
410                              the Android device. Like `-np surfaceflinger`.""")
411
412    target_group.add_argument('-cmd', help="""Profile running a command on the Android device.
413                              Like `-cmd "pm -l"`.""")
414
415    target_group.add_argument('--pid', nargs='+', help="""Profile native processes running on device
416                              given their process ids.""")
417
418    target_group.add_argument('--tid', nargs='+', help="""Profile native threads running on device
419                              given their thread ids.""")
420
421    target_group.add_argument('--system_wide', action='store_true', help="""Profile system wide.""")
422
423    app_target_group = parser.add_argument_group(title='Extra options for profiling an app')
424    app_target_group.add_argument('--compile_java_code', action='store_true', help="""Used with -p.
425                                  On Android N and Android O, we need to compile Java code into
426                                  native instructions to profile Java code. Android O also needs
427                                  wrap.sh in the apk to use the native instructions.""")
428
429    app_start_group = app_target_group.add_mutually_exclusive_group()
430    app_start_group.add_argument('-a', '--activity', help="""Used with -p. Profile the launch time
431                                 of an activity in an Android app. The app will be started or
432                                 restarted to run the activity. Like `-a .MainActivity`.""")
433
434    app_start_group.add_argument('-t', '--test', help="""Used with -p. Profile the launch time of an
435                                 instrumentation test in an Android app. The app will be started or
436                                 restarted to run the instrumentation test. Like
437                                 `-t test_class_name`.""")
438
439    record_group = parser.add_argument_group('Select recording options')
440    record_group.add_argument('-r', '--record_options',
441                              default='-e task-clock:u -f 1000 -g --duration 10', help="""Set
442                              recording options for `simpleperf record` command. Use
443                              `run_simpleperf_on_device.py record -h` to see all accepted options.
444                              Default is "-e task-clock:u -f 1000 -g --duration 10".""")
445
446    record_group.add_argument('-lib', '--native_lib_dir', type=extant_dir,
447                              help="""When profiling an Android app containing native libraries,
448                                      the native libraries are usually stripped and lake of symbols
449                                      and debug information to provide good profiling result. By
450                                      using -lib, you tell app_profiler.py the path storing
451                                      unstripped native libraries, and app_profiler.py will search
452                                      all shared libraries with suffix .so in the directory. Then
453                                      the native libraries will be downloaded on device and
454                                      collected in build_cache.""")
455
456    record_group.add_argument('-o', '--perf_data_path', default='perf.data',
457                              help='The path to store profiling data. Default is perf.data.')
458
459    record_group.add_argument('-nb', '--skip_collect_binaries', action='store_true',
460                              help="""By default we collect binaries used in profiling data from
461                                      device to binary_cache directory. It can be used to annotate
462                                      source code and disassembly. This option skips it.""")
463
464    other_group = parser.add_argument_group('Other options')
465    other_group.add_argument('--ndk_path', type=extant_dir,
466                             help="""Set the path of a ndk release. app_profiler.py needs some
467                                     tools in ndk, like readelf.""")
468
469    other_group.add_argument('--disable_adb_root', action='store_true',
470                             help="""Force adb to run in non root mode. By default, app_profiler.py
471                                     will try to switch to root mode to be able to profile released
472                                     Android apps.""")
473    other_group.add_argument(
474        '--log', choices=['debug', 'info', 'warning'], default='info', help='set log level')
475
476    def check_args(args):
477        if (not args.app) and (args.compile_java_code or args.activity or args.test):
478            log_exit('--compile_java_code, -a, -t can only be used when profiling an Android app.')
479
480    args = parser.parse_args()
481    set_log_level(args.log)
482    check_args(args)
483    if args.app:
484        profiler = AppProfiler(args)
485    elif args.native_program:
486        profiler = NativeProgramProfiler(args)
487    elif args.cmd:
488        profiler = NativeCommandProfiler(args)
489    elif args.pid:
490        profiler = NativeProcessProfiler(args)
491    elif args.tid:
492        profiler = NativeThreadProfiler(args)
493    elif args.system_wide:
494        profiler = SystemWideProfiler(args)
495    profiler.profile()
496
497
498if __name__ == '__main__':
499    main()
500