1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import glob
6import hashlib
7import logging
8import os
9import platform
10import re
11import shutil
12import subprocess
13
14from telemetry.internal.util import binary_manager
15from telemetry.core import platform as telemetry_platform
16from telemetry.core import util
17from telemetry import decorators
18from telemetry.internal.platform.profiler import android_prebuilt_profiler_helper
19
20from devil.android import md5sum  # pylint: disable=import-error
21
22
23try:
24  import sqlite3
25except ImportError:
26  sqlite3 = None
27
28
29
30_TEXT_SECTION = '.text'
31
32
33def _ElfMachineId(elf_file):
34  headers = subprocess.check_output(['readelf', '-h', elf_file])
35  return re.match(r'.*Machine:\s+(\w+)', headers, re.DOTALL).group(1)
36
37
38def _ElfSectionAsString(elf_file, section):
39  return subprocess.check_output(['readelf', '-p', section, elf_file])
40
41
42def _ElfSectionMd5Sum(elf_file, section):
43  result = subprocess.check_output(
44      'readelf -p%s "%s" | md5sum' % (section, elf_file), shell=True)
45  return result.split(' ', 1)[0]
46
47
48def _FindMatchingUnstrippedLibraryOnHost(device, lib):
49  lib_base = os.path.basename(lib)
50
51  device_md5 = device.RunShellCommand('md5 "%s"' % lib, as_root=True)[0]
52  device_md5 = device_md5.split(' ', 1)[0]
53
54  def FindMatchingStrippedLibrary(out_path):
55    # First find a matching stripped library on the host. This avoids the need
56    # to pull the stripped library from the device, which can take tens of
57    # seconds.
58    host_lib_pattern = os.path.join(out_path, '*_apk', 'libs', '*', lib_base)
59    for stripped_host_lib in glob.glob(host_lib_pattern):
60      with open(stripped_host_lib) as f:
61        host_md5 = hashlib.md5(f.read()).hexdigest()
62        if host_md5 == device_md5:
63          return stripped_host_lib
64    return None
65
66  out_path = None
67  stripped_host_lib = None
68  for out_path in util.GetBuildDirectories():
69    stripped_host_lib = FindMatchingStrippedLibrary(out_path)
70    if stripped_host_lib:
71      break
72
73  if not stripped_host_lib:
74    return None
75
76  # The corresponding unstripped library will be under out/Release/lib.
77  unstripped_host_lib = os.path.join(out_path, 'lib', lib_base)
78
79  # Make sure the unstripped library matches the stripped one. We do this
80  # by comparing the hashes of text sections in both libraries. This isn't an
81  # exact guarantee, but should still give reasonable confidence that the
82  # libraries are compatible.
83  # TODO(skyostil): Check .note.gnu.build-id instead once we're using
84  # --build-id=sha1.
85  # pylint: disable=undefined-loop-variable
86  if (_ElfSectionMd5Sum(unstripped_host_lib, _TEXT_SECTION) !=
87      _ElfSectionMd5Sum(stripped_host_lib, _TEXT_SECTION)):
88    return None
89  return unstripped_host_lib
90
91
92@decorators.Cache
93def GetPerfhostName():
94  return 'perfhost_' + telemetry_platform.GetHostPlatform().GetOSVersionName()
95
96
97# Ignored directories for libraries that aren't useful for symbolization.
98_IGNORED_LIB_PATHS = [
99  '/data/dalvik-cache',
100  '/tmp'
101]
102
103
104def GetRequiredLibrariesForPerfProfile(profile_file):
105  """Returns the set of libraries necessary to symbolize a given perf profile.
106
107  Args:
108    profile_file: Path to perf profile to analyse.
109
110  Returns:
111    A set of required library file names.
112  """
113  with open(os.devnull, 'w') as dev_null:
114    perfhost_path = binary_manager.FetchPath(
115        GetPerfhostName(), 'x86_64', 'linux')
116    perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file],
117                             stdout=dev_null, stderr=subprocess.PIPE)
118    _, output = perf.communicate()
119  missing_lib_re = re.compile(
120      ('^Failed to open (.*), continuing without symbols|'
121       '^(.*[.]so).*not found, continuing without symbols'))
122  libs = set()
123  for line in output.split('\n'):
124    lib = missing_lib_re.match(line)
125    if lib:
126      lib = lib.group(1) or lib.group(2)
127      path = os.path.dirname(lib)
128      if (any(path.startswith(ignored_path)
129              for ignored_path in _IGNORED_LIB_PATHS)
130          or path == '/' or not path):
131        continue
132      libs.add(lib)
133  return libs
134
135
136def GetRequiredLibrariesForVTuneProfile(profile_file):
137  """Returns the set of libraries necessary to symbolize a given VTune profile.
138
139  Args:
140    profile_file: Path to VTune profile to analyse.
141
142  Returns:
143    A set of required library file names.
144  """
145  db_file = os.path.join(profile_file, 'sqlite-db', 'dicer.db')
146  conn = sqlite3.connect(db_file)
147
148  try:
149    # The 'dd_module_file' table lists all libraries on the device. Only the
150    # ones with 'bin_located_path' are needed for the profile.
151    query = 'SELECT bin_path, bin_located_path FROM dd_module_file'
152    return set(row[0] for row in conn.cursor().execute(query) if row[1])
153  finally:
154    conn.close()
155
156
157def _FileMetadataMatches(filea, fileb):
158  """Check if the metadata of two files matches."""
159  assert os.path.exists(filea)
160  if not os.path.exists(fileb):
161    return False
162
163  fields_to_compare = [
164      'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid']
165
166  filea_stat = os.stat(filea)
167  fileb_stat = os.stat(fileb)
168  for field in fields_to_compare:
169    # shutil.copy2 doesn't get ctime/mtime identical when the file system
170    # provides sub-second accuracy.
171    if int(getattr(filea_stat, field)) != int(getattr(fileb_stat, field)):
172      return False
173  return True
174
175
176def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
177  """Creates a symfs directory to be used for symbolizing profiles.
178
179  Prepares a set of files ("symfs") to be used with profilers such as perf for
180  converting binary addresses into human readable function names.
181
182  Args:
183    device: DeviceUtils instance identifying the target device.
184    symfs_dir: Path where the symfs should be created.
185    libraries: Set of library file names that should be included in the symfs.
186    use_symlinks: If True, link instead of copy unstripped libraries into the
187      symfs. This will speed up the operation, but the resulting symfs will no
188      longer be valid if the linked files are modified, e.g., by rebuilding.
189
190  Returns:
191    The absolute path to the kernel symbols within the created symfs.
192  """
193  logging.info('Building symfs into %s.' % symfs_dir)
194
195  for lib in libraries:
196    device_dir = os.path.dirname(lib)
197    output_dir = os.path.join(symfs_dir, device_dir[1:])
198    if not os.path.exists(output_dir):
199      os.makedirs(output_dir)
200    output_lib = os.path.join(output_dir, os.path.basename(lib))
201
202    if lib.startswith('/data/app'):
203      # If this is our own library instead of a system one, look for a matching
204      # unstripped library under the out directory.
205      unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(device, lib)
206      if not unstripped_host_lib:
207        logging.warning('Could not find symbols for %s.' % lib)
208        logging.warning('Is the correct output directory selected '
209                        '(CHROMIUM_OUT_DIR)? Did you install the APK after '
210                        'building?')
211        continue
212      if use_symlinks:
213        if os.path.lexists(output_lib):
214          os.remove(output_lib)
215        os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
216      # Copy the unstripped library only if it has been changed to avoid the
217      # delay.
218      elif not _FileMetadataMatches(unstripped_host_lib, output_lib):
219        logging.info('Copying %s to %s' % (unstripped_host_lib, output_lib))
220        shutil.copy2(unstripped_host_lib, output_lib)
221    else:
222      # Otherwise save a copy of the stripped system library under the symfs so
223      # the profiler can at least use the public symbols of that library. To
224      # speed things up, only pull files that don't match copies we already
225      # have in the symfs.
226      if not os.path.exists(output_lib):
227        pull = True
228      else:
229        host_md5sums = md5sum.CalculateHostMd5Sums([output_lib])
230        try:
231          device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], device)
232        except:
233          logging.exception('New exception caused by DeviceUtils conversion')
234          raise
235
236        pull = True
237        if host_md5sums and device_md5sums and output_lib in host_md5sums \
238          and lib in device_md5sums:
239          pull = host_md5sums[output_lib] != device_md5sums[lib]
240
241      if pull:
242        logging.info('Pulling %s to %s', lib, output_lib)
243        device.PullFile(lib, output_lib)
244
245  # Also pull a copy of the kernel symbols.
246  output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
247  if not os.path.exists(output_kallsyms):
248    device.PullFile('/proc/kallsyms', output_kallsyms)
249  return output_kallsyms
250
251
252def PrepareDeviceForPerf(device):
253  """Set up a device for running perf.
254
255  Args:
256    device: DeviceUtils instance identifying the target device.
257
258  Returns:
259    The path to the installed perf binary on the device.
260  """
261  android_prebuilt_profiler_helper.InstallOnDevice(device, 'perf')
262  # Make sure kernel pointers are not hidden.
263  device.WriteFile('/proc/sys/kernel/kptr_restrict', '0', as_root=True)
264  return android_prebuilt_profiler_helper.GetDevicePath('perf')
265
266
267def GetToolchainBinaryPath(library_file, binary_name):
268  """Return the path to an Android toolchain binary on the host.
269
270  Args:
271    library_file: ELF library which is used to identify the used ABI,
272        architecture and toolchain.
273    binary_name: Binary to search for, e.g., 'objdump'
274  Returns:
275    Full path to binary or None if the binary was not found.
276  """
277  # Mapping from ELF machine identifiers to GNU toolchain names.
278  toolchain_configs = {
279    'x86': 'i686-linux-android',
280    'MIPS': 'mipsel-linux-android',
281    'ARM': 'arm-linux-androideabi',
282    'x86-64': 'x86_64-linux-android',
283    'AArch64': 'aarch64-linux-android',
284  }
285  toolchain_config = toolchain_configs[_ElfMachineId(library_file)]
286  host_os = platform.uname()[0].lower()
287  host_machine = platform.uname()[4]
288
289  elf_comment = _ElfSectionAsString(library_file, '.comment')
290  toolchain_version = re.match(r'.*GCC: \(GNU\) ([\w.]+)',
291                               elf_comment, re.DOTALL)
292  if not toolchain_version:
293    return None
294  toolchain_version = toolchain_version.group(1)
295  toolchain_version = toolchain_version.replace('.x', '')
296
297  toolchain_path = os.path.abspath(os.path.join(
298      util.GetChromiumSrcDir(), 'third_party', 'android_tools', 'ndk',
299      'toolchains', '%s-%s' % (toolchain_config, toolchain_version)))
300  if not os.path.exists(toolchain_path):
301    logging.warning(
302        'Unable to find toolchain binary %s: toolchain not found at %s',
303        binary_name, toolchain_path)
304    return None
305
306  path = os.path.join(
307      toolchain_path, 'prebuilt', '%s-%s' % (host_os, host_machine), 'bin',
308      '%s-%s' % (toolchain_config, binary_name))
309  if not os.path.exists(path):
310    logging.warning(
311        'Unable to find toolchain binary %s: binary not found at %s',
312        binary_name, path)
313    return None
314
315  return path
316