1#!/usr/bin/env python
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""utils.py: export utility functions.
19"""
20
21from __future__ import print_function
22import logging
23import os
24import os.path
25import shutil
26import subprocess
27import sys
28import time
29
30def get_script_dir():
31    return os.path.dirname(os.path.realpath(__file__))
32
33def is_windows():
34    return sys.platform == 'win32' or sys.platform == 'cygwin'
35
36def is_darwin():
37    return sys.platform == 'darwin'
38
39def get_platform():
40    if is_windows():
41        return 'windows'
42    if is_darwin():
43        return 'darwin'
44    return 'linux'
45
46def is_python3():
47    return sys.version_info >= (3, 0)
48
49
50def log_debug(msg):
51    logging.debug(msg)
52
53
54def log_info(msg):
55    logging.info(msg)
56
57
58def log_warning(msg):
59    logging.warning(msg)
60
61
62def log_fatal(msg):
63    raise Exception(msg)
64
65def log_exit(msg):
66    sys.exit(msg)
67
68def disable_debug_log():
69    logging.getLogger().setLevel(logging.WARN)
70
71def str_to_bytes(str):
72    if not is_python3():
73        return str
74    # In python 3, str are wide strings whereas the C api expects 8 bit strings,
75    # hence we have to convert. For now using utf-8 as the encoding.
76    return str.encode('utf-8')
77
78def bytes_to_str(bytes):
79    if not is_python3():
80        return bytes
81    return bytes.decode('utf-8')
82
83def get_target_binary_path(arch, binary_name):
84    if arch == 'aarch64':
85        arch = 'arm64'
86    arch_dir = os.path.join(get_script_dir(), "bin", "android", arch)
87    if not os.path.isdir(arch_dir):
88        log_fatal("can't find arch directory: %s" % arch_dir)
89    binary_path = os.path.join(arch_dir, binary_name)
90    if not os.path.isfile(binary_path):
91        log_fatal("can't find binary: %s" % binary_path)
92    return binary_path
93
94
95def get_host_binary_path(binary_name):
96    dir = os.path.join(get_script_dir(), 'bin')
97    if is_windows():
98        if binary_name.endswith('.so'):
99            binary_name = binary_name[0:-3] + '.dll'
100        elif '.' not in binary_name:
101            binary_name += '.exe'
102        dir = os.path.join(dir, 'windows')
103    elif sys.platform == 'darwin': # OSX
104        if binary_name.endswith('.so'):
105            binary_name = binary_name[0:-3] + '.dylib'
106        dir = os.path.join(dir, 'darwin')
107    else:
108        dir = os.path.join(dir, 'linux')
109    dir = os.path.join(dir, 'x86_64' if sys.maxsize > 2 ** 32 else 'x86')
110    binary_path = os.path.join(dir, binary_name)
111    if not os.path.isfile(binary_path):
112        log_fatal("can't find binary: %s" % binary_path)
113    return binary_path
114
115
116def is_executable_available(executable, option='--help'):
117    """ Run an executable to see if it exists. """
118    try:
119        subproc = subprocess.Popen([executable, option], stdout=subprocess.PIPE,
120                                   stderr=subprocess.PIPE)
121        subproc.communicate()
122        return subproc.returncode == 0
123    except:
124        return False
125
126DEFAULT_NDK_PATH = {
127    'darwin': 'Library/Android/sdk/ndk-bundle',
128    'linux': 'Android/Sdk/ndk-bundle',
129    'windows': 'AppData/Local/Android/sdk/ndk-bundle',
130}
131
132EXPECTED_TOOLS = {
133    'adb': {
134        'is_binutils': False,
135        'test_option': 'version',
136        'path_in_ndk': '../platform-tools/adb',
137    },
138    'readelf': {
139        'is_binutils': True,
140        'accept_tool_without_arch': True,
141    },
142    'addr2line': {
143        'is_binutils': True,
144        'accept_tool_without_arch': True
145    },
146    'objdump': {
147        'is_binutils': True,
148    },
149}
150
151def _get_binutils_path_in_ndk(toolname, arch, platform):
152    if not arch:
153        arch = 'arm64'
154    if arch == 'arm64':
155        name = 'aarch64-linux-android-' + toolname
156        path = 'toolchains/aarch64-linux-android-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name)
157    elif arch == 'arm':
158        name = 'arm-linux-androideabi-' + toolname
159        path = 'toolchains/arm-linux-androideabi-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name)
160    elif arch == 'x86_64':
161        name = 'x86_64-linux-android-' + toolname
162        path = 'toolchains/x86_64-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name)
163    elif arch == 'x86':
164        name = 'i686-linux-android-' + toolname
165        path = 'toolchains/x86-4.9/prebuilt/%s-x86_64/bin/%s' % (platform, name)
166    else:
167        log_fatal('unexpected arch %s' % arch)
168    return (name, path)
169
170def find_tool_path(toolname, ndk_path=None, arch=None):
171    if toolname not in EXPECTED_TOOLS:
172        return None
173    tool_info = EXPECTED_TOOLS[toolname]
174    is_binutils = tool_info['is_binutils']
175    test_option = tool_info.get('test_option', '--help')
176    platform = get_platform()
177    if is_binutils:
178        toolname_with_arch, path_in_ndk = _get_binutils_path_in_ndk(toolname, arch, platform)
179    else:
180        toolname_with_arch = toolname
181        path_in_ndk = tool_info['path_in_ndk']
182    path_in_ndk = path_in_ndk.replace('/', os.sep)
183
184    # 1. Find tool in the given ndk path.
185    if ndk_path:
186        path = os.path.join(ndk_path, path_in_ndk)
187        if is_executable_available(path, test_option):
188            return path
189
190    # 2. Find tool in the ndk directory containing simpleperf scripts.
191    path = os.path.join('..', path_in_ndk)
192    if is_executable_available(path, test_option):
193        return path
194
195    # 3. Find tool in the default ndk installation path.
196    home = os.environ.get('HOMEPATH') if is_windows() else os.environ.get('HOME')
197    if home:
198        default_ndk_path = os.path.join(home, DEFAULT_NDK_PATH[platform].replace('/', os.sep))
199        path = os.path.join(default_ndk_path, path_in_ndk)
200        if is_executable_available(path, test_option):
201            return path
202
203    # 4. Find tool in $PATH.
204    if is_executable_available(toolname_with_arch, test_option):
205        return toolname_with_arch
206
207    # 5. Find tool without arch in $PATH.
208    if is_binutils and tool_info.get('accept_tool_without_arch'):
209        if is_executable_available(toolname, test_option):
210            return toolname
211    return None
212
213
214class AdbHelper(object):
215    def __init__(self, enable_switch_to_root=True):
216        adb_path = find_tool_path('adb')
217        if not adb_path:
218            log_exit("Can't find adb in PATH environment.")
219        self.adb_path = adb_path
220        self.enable_switch_to_root = enable_switch_to_root
221
222
223    def run(self, adb_args):
224        return self.run_and_return_output(adb_args)[0]
225
226
227    def run_and_return_output(self, adb_args, stdout_file=None, log_output=True):
228        adb_args = [self.adb_path] + adb_args
229        log_debug('run adb cmd: %s' % adb_args)
230        if stdout_file:
231            with open(stdout_file, 'wb') as stdout_fh:
232                returncode = subprocess.call(adb_args, stdout=stdout_fh)
233            stdoutdata = ''
234        else:
235            subproc = subprocess.Popen(adb_args, stdout=subprocess.PIPE)
236            (stdoutdata, _) = subproc.communicate()
237            returncode = subproc.returncode
238        result = (returncode == 0)
239        if stdoutdata and adb_args[1] != 'push' and adb_args[1] != 'pull':
240            stdoutdata = bytes_to_str(stdoutdata)
241            if log_output:
242                log_debug(stdoutdata)
243        log_debug('run adb cmd: %s  [result %s]' % (adb_args, result))
244        return (result, stdoutdata)
245
246    def check_run(self, adb_args):
247        self.check_run_and_return_output(adb_args)
248
249
250    def check_run_and_return_output(self, adb_args, stdout_file=None, log_output=True):
251        result, stdoutdata = self.run_and_return_output(adb_args, stdout_file, log_output)
252        if not result:
253            log_exit('run "adb %s" failed' % adb_args)
254        return stdoutdata
255
256
257    def _unroot(self):
258        result, stdoutdata = self.run_and_return_output(['shell', 'whoami'])
259        if not result:
260            return
261        if 'root' not in stdoutdata:
262            return
263        log_info('unroot adb')
264        self.run(['unroot'])
265        self.run(['wait-for-device'])
266        time.sleep(1)
267
268
269    def switch_to_root(self):
270        if not self.enable_switch_to_root:
271            self._unroot()
272            return False
273        result, stdoutdata = self.run_and_return_output(['shell', 'whoami'])
274        if not result:
275            return False
276        if 'root' in stdoutdata:
277            return True
278        build_type = self.get_property('ro.build.type')
279        if build_type == 'user':
280            return False
281        self.run(['root'])
282        time.sleep(1)
283        self.run(['wait-for-device'])
284        result, stdoutdata = self.run_and_return_output(['shell', 'whoami'])
285        return result and 'root' in stdoutdata
286
287    def get_property(self, name):
288        result, stdoutdata = self.run_and_return_output(['shell', 'getprop', name])
289        return stdoutdata if result else None
290
291    def set_property(self, name, value):
292        return self.run(['shell', 'setprop', name, value])
293
294
295    def get_device_arch(self):
296        output = self.check_run_and_return_output(['shell', 'uname', '-m'])
297        if 'aarch64' in output:
298            return 'arm64'
299        if 'arm' in output:
300            return 'arm'
301        if 'x86_64' in output:
302            return 'x86_64'
303        if '86' in output:
304            return 'x86'
305        log_fatal('unsupported architecture: %s' % output.strip())
306
307
308    def get_android_version(self):
309        build_version = self.get_property('ro.build.version.release')
310        android_version = 0
311        if build_version:
312            if not build_version[0].isdigit():
313                c = build_version[0].upper()
314                if c.isupper() and c >= 'L':
315                    android_version = ord(c) - ord('L') + 5
316            else:
317                strs = build_version.split('.')
318                if strs:
319                    android_version = int(strs[0])
320        return android_version
321
322
323def flatten_arg_list(arg_list):
324    res = []
325    if arg_list:
326        for items in arg_list:
327            res += items
328    return res
329
330
331def remove(dir_or_file):
332    if os.path.isfile(dir_or_file):
333        os.remove(dir_or_file)
334    elif os.path.isdir(dir_or_file):
335        shutil.rmtree(dir_or_file, ignore_errors=True)
336
337
338def open_report_in_browser(report_path):
339    if is_darwin():
340        # On darwin 10.12.6, webbrowser can't open browser, so try `open` cmd first.
341        try:
342            subprocess.check_call(['open', report_path])
343            return
344        except:
345            pass
346    import webbrowser
347    try:
348        # Try to open the report with Chrome
349        browser_key = ''
350        for key, _ in webbrowser._browsers.items():
351            if 'chrome' in key:
352                browser_key = key
353        browser = webbrowser.get(browser_key)
354        browser.open(report_path, new=0, autoraise=True)
355    except:
356        # webbrowser.get() doesn't work well on darwin/windows.
357        webbrowser.open_new_tab(report_path)
358
359
360def find_real_dso_path(dso_path_in_record_file, binary_cache_path):
361    """ Given the path of a shared library in perf.data, find its real path in the file system. """
362    if dso_path_in_record_file[0] != '/' or dso_path_in_record_file == '//anon':
363        return None
364    if binary_cache_path:
365        tmp_path = os.path.join(binary_cache_path, dso_path_in_record_file[1:])
366        if os.path.isfile(tmp_path):
367            return tmp_path
368    if os.path.isfile(dso_path_in_record_file):
369        return dso_path_in_record_file
370    return None
371
372def get_arch_of_dso_path(readelf_path, dso_path):
373    try:
374        output = subprocess.check_output([readelf_path, '-h', dso_path])
375        if output.find('AArch64') != -1:
376            return 'arm64'
377        if output.find('ARM') != -1:
378            return 'arm'
379        if output.find('X86-64') != -1:
380            return 'x86_64'
381        if output.find('80386') != -1:
382            return 'x86'
383    except subprocess.CalledProcessError:
384        pass
385    return 'unknown'
386
387
388class Addr2Nearestline(object):
389    """ Use addr2line to convert (dso_path, func_addr, addr) to (source_file, line) pairs.
390        For instructions generated by C++ compilers without a matching statement in source code
391        (like stack corruption check, switch optimization, etc.), addr2line can't generate
392        line information. However, we want to assign the instruction to the nearest line before
393        the instruction (just like objdump -dl). So we use below strategy:
394        Instead of finding the exact line of the instruction in an address, we find the nearest
395        line to the instruction in an address. If an address doesn't have a line info, we find
396        the line info of address - 1. If still no line info, then use address - 2, address - 3,
397        etc.
398
399        The implementation steps are as below:
400        1. Collect all (dso_path, func_addr, addr) requests before converting. This saves the
401        times to call addr2line.
402        2. Convert addrs to (source_file, line) pairs for each dso_path as below:
403          2.1 Check if the dso_path has .debug_line. If not, omit its conversion.
404          2.2 Get arch of the dso_path, and decide the addr_step for it. addr_step is the step we
405          change addr each time. For example, since instructions of arm64 are all 4 bytes long,
406          addr_step for arm64 can be 4.
407          2.3 Use addr2line to find line info for each addr in the dso_path.
408          2.4 For each addr without line info, use addr2line to find line info for
409              range(addr - addr_step, addr - addr_step * 4 - 1, -addr_step).
410          2.5 For each addr without line info, use addr2line to find line info for
411              range(addr - addr_step * 5, addr - addr_step * 128 - 1, -addr_step).
412              (128 is a guess number. A nested switch statement in
413               system/core/demangle/Demangler.cpp has >300 bytes without line info in arm64.)
414    """
415    class Dso(object):
416        """ Info of a dynamic shared library.
417            addrs: a map from address to Addr object in this dso.
418        """
419        def __init__(self):
420            self.addrs = {}
421
422    class Addr(object):
423        """ Info of an addr request.
424            func_addr: start_addr of the function containing addr.
425            source_lines: a list of [file_id, line_number] for addr.
426                          source_lines[:-1] are all for inlined functions.
427        """
428        def __init__(self, func_addr):
429            self.func_addr = func_addr
430            self.source_lines = None
431
432    def __init__(self, ndk_path, binary_cache_path):
433        self.addr2line_path = find_tool_path('addr2line', ndk_path)
434        if not self.addr2line_path:
435            log_exit("Can't find addr2line. Please set ndk path by --ndk-path option.")
436        self.readelf_path = find_tool_path('readelf', ndk_path)
437        if not self.readelf_path:
438            log_exit("Can't find readelf. Please set ndk path by --ndk-path option.")
439        self.dso_map = {}  # map from dso_path to Dso.
440        self.binary_cache_path = binary_cache_path
441        # Saving file names for each addr takes a lot of memory. So we store file ids in Addr,
442        # and provide data structures connecting file id and file name here.
443        self.file_name_to_id = {}
444        self.file_id_to_name = []
445
446    def add_addr(self, dso_path, func_addr, addr):
447        dso = self.dso_map.get(dso_path)
448        if dso is None:
449            dso = self.dso_map[dso_path] = self.Dso()
450        if addr not in dso.addrs:
451            dso.addrs[addr] = self.Addr(func_addr)
452
453    def convert_addrs_to_lines(self):
454        for dso_path in self.dso_map:
455            self._convert_addrs_in_one_dso(dso_path, self.dso_map[dso_path])
456
457    def _convert_addrs_in_one_dso(self, dso_path, dso):
458        real_path = find_real_dso_path(dso_path, self.binary_cache_path)
459        if not real_path:
460            if dso_path not in ['//anon', 'unknown', '[kernel.kallsyms]']:
461                log_debug("Can't find dso %s" % dso_path)
462            return
463
464        if not self._check_debug_line_section(real_path):
465            log_debug("file %s doesn't contain .debug_line section." % real_path)
466            return
467
468        addr_step = self._get_addr_step(real_path)
469        self._collect_line_info(dso, real_path, [0])
470        self._collect_line_info(dso, real_path, range(-addr_step, -addr_step * 4 - 1, -addr_step))
471        self._collect_line_info(dso, real_path,
472                                range(-addr_step * 5, -addr_step * 128 - 1, -addr_step))
473
474    def _check_debug_line_section(self, real_path):
475        try:
476            output = subprocess.check_output([self.readelf_path, '-S', real_path])
477            return output.find('.debug_line') != -1
478        except subprocess.CalledProcessError:
479            return False
480
481    def _get_addr_step(self, real_path):
482        arch = get_arch_of_dso_path(self.readelf_path, real_path)
483        if arch == 'arm64':
484            return 4
485        if arch == 'arm':
486            return 2
487        return 1
488
489    def _collect_line_info(self, dso, real_path, addr_shifts):
490        """ Use addr2line to get line info in a dso, with given addr shifts. """
491        # 1. Collect addrs to send to addr2line.
492        addr_set = set()
493        for addr in dso.addrs:
494            addr_obj = dso.addrs[addr]
495            if addr_obj.source_lines:  # already has source line, no need to search.
496                continue
497            for shift in addr_shifts:
498                # The addr after shift shouldn't change to another function.
499                shifted_addr = max(addr + shift, addr_obj.func_addr)
500                addr_set.add(shifted_addr)
501                if shifted_addr == addr_obj.func_addr:
502                    break
503        if not addr_set:
504            return
505        addr_request = '\n'.join(['%x' % addr for addr in sorted(addr_set)])
506
507        # 2. Use addr2line to collect line info.
508        try:
509            subproc = subprocess.Popen([self.addr2line_path, '-ai', '-e', real_path],
510                                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
511            (stdoutdata, _) = subproc.communicate(str_to_bytes(addr_request))
512            stdoutdata = bytes_to_str(stdoutdata)
513        except:
514            return
515        addr_map = {}
516        cur_line_list = None
517        for line in stdoutdata.strip().split('\n'):
518            if line[:2] == '0x':
519                # a new address
520                cur_line_list = addr_map[int(line, 16)] = []
521            else:
522                # a file:line.
523                if cur_line_list is None:
524                    continue
525                # Handle lines like "C:\Users\...\file:32".
526                items = line.rsplit(':', 1)
527                if len(items) != 2:
528                    continue
529                if '?' in line:
530                    # if ? in line, it doesn't have a valid line info.
531                    # An addr can have a list of (file, line), when the addr belongs to an inlined
532                    # function. Sometimes only part of the list has ? mark. In this case, we think
533                    # the line info is valid if the first line doesn't have ? mark.
534                    if not cur_line_list:
535                        cur_line_list = None
536                    continue
537                (file_path, line_number) = items
538                line_number = line_number.split()[0]  # Remove comments after line number
539                try:
540                    line_number = int(line_number)
541                except ValueError:
542                    continue
543                file_id = self._get_file_id(file_path)
544                cur_line_list.append((file_id, line_number))
545
546        # 3. Fill line info in dso.addrs.
547        for addr in dso.addrs:
548            addr_obj = dso.addrs[addr]
549            if addr_obj.source_lines:
550                continue
551            for shift in addr_shifts:
552                shifted_addr = max(addr + shift, addr_obj.func_addr)
553                lines = addr_map.get(shifted_addr)
554                if lines:
555                    addr_obj.source_lines = lines
556                    break
557                if shifted_addr == addr_obj.func_addr:
558                    break
559
560    def _get_file_id(self, file_path):
561        file_id = self.file_name_to_id.get(file_path)
562        if file_id is None:
563            file_id = self.file_name_to_id[file_path] = len(self.file_id_to_name)
564            self.file_id_to_name.append(file_path)
565        return file_id
566
567    def get_dso(self, dso_path):
568        return self.dso_map.get(dso_path)
569
570    def get_addr_source(self, dso, addr):
571        source = dso.addrs[addr].source_lines
572        if source is None:
573            return None
574        return [(self.file_id_to_name[file_id], line) for (file_id, line) in source]
575
576
577class Objdump(object):
578    """ A wrapper of objdump to disassemble code. """
579    def __init__(self, ndk_path, binary_cache_path):
580        self.ndk_path = ndk_path
581        self.binary_cache_path = binary_cache_path
582        self.readelf_path = find_tool_path('readelf', ndk_path)
583        if not self.readelf_path:
584            log_exit("Can't find readelf. Please set ndk path by --ndk_path option.")
585        self.objdump_paths = {}
586
587    def disassemble_code(self, dso_path, start_addr, addr_len):
588        """ Disassemble [start_addr, start_addr + addr_len] of dso_path.
589            Return a list of pair (disassemble_code_line, addr).
590        """
591        # 1. Find real path.
592        real_path = find_real_dso_path(dso_path, self.binary_cache_path)
593        if real_path is None:
594            return None
595
596        # 2. Get path of objdump.
597        arch = get_arch_of_dso_path(self.readelf_path, real_path)
598        objdump_path = self.objdump_paths.get(arch)
599        if not objdump_path:
600            objdump_path = find_tool_path('objdump', self.ndk_path, arch)
601            if not objdump_path:
602                log_exit("Can't find objdump. Please set ndk path by --ndk_path option.")
603            self.objdump_paths[arch] = objdump_path
604
605        # 3. Run objdump.
606        args = [objdump_path, '-dlC', '--no-show-raw-insn',
607                '--start-address=0x%x' % start_addr,
608                '--stop-address=0x%x' % (start_addr + addr_len),
609                real_path]
610        try:
611            subproc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
612            (stdoutdata, _) = subproc.communicate()
613            stdoutdata = bytes_to_str(stdoutdata)
614        except:
615            return None
616
617        if not stdoutdata:
618            return None
619        result = []
620        for line in stdoutdata.split('\n'):
621            line = line.rstrip()  # Remove '\r' on Windows.
622            items = line.split(':', 1)
623            try:
624                addr = int(items[0], 16)
625            except ValueError:
626                addr = 0
627            result.append((line, addr))
628        return result
629
630
631logging.getLogger().setLevel(logging.DEBUG)
632