1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import glob
6import hashlib
7import logging
8import os
9import platform
10import re
11import shutil
12import subprocess
13
14from telemetry import decorators
15from telemetry.core import platform as telemetry_platform
16from telemetry.core import util
17from telemetry.core.platform.profiler import android_prebuilt_profiler_helper
18from telemetry.util import support_binaries
19
20try:
21  import sqlite3
22except ImportError:
23  sqlite3 = None
24
25
26
27_TEXT_SECTION = '.text'
28
29
30def _ElfMachineId(elf_file):
31  headers = subprocess.check_output(['readelf', '-h', elf_file])
32  return re.match(r'.*Machine:\s+(\w+)', headers, re.DOTALL).group(1)
33
34
35def _ElfSectionAsString(elf_file, section):
36  return subprocess.check_output(['readelf', '-p', section, elf_file])
37
38
39def _ElfSectionMd5Sum(elf_file, section):
40  result = subprocess.check_output(
41      'readelf -p%s "%s" | md5sum' % (section, elf_file), shell=True)
42  return result.split(' ', 1)[0]
43
44
45def _FindMatchingUnstrippedLibraryOnHost(device, lib):
46  lib_base = os.path.basename(lib)
47
48  device_md5 = device.RunShellCommand('md5 "%s"' % lib, as_root=True)[0]
49  device_md5 = device_md5.split(' ', 1)[0]
50
51  def FindMatchingStrippedLibrary(out_path):
52    # First find a matching stripped library on the host. This avoids the need
53    # to pull the stripped library from the device, which can take tens of
54    # seconds.
55    host_lib_pattern = os.path.join(out_path, '*_apk', 'libs', '*', lib_base)
56    for stripped_host_lib in glob.glob(host_lib_pattern):
57      with open(stripped_host_lib) as f:
58        host_md5 = hashlib.md5(f.read()).hexdigest()
59        if host_md5 == device_md5:
60          return stripped_host_lib
61
62  for build_dir, build_type in util.GetBuildDirectories():
63    out_path = os.path.join(build_dir, build_type)
64    stripped_host_lib = FindMatchingStrippedLibrary(out_path)
65    if stripped_host_lib:
66      break
67  else:
68    return None
69
70  # The corresponding unstripped library will be under out/Release/lib.
71  unstripped_host_lib = os.path.join(out_path, 'lib', lib_base)
72
73  # Make sure the unstripped library matches the stripped one. We do this
74  # by comparing the hashes of text sections in both libraries. This isn't an
75  # exact guarantee, but should still give reasonable confidence that the
76  # libraries are compatible.
77  # TODO(skyostil): Check .note.gnu.build-id instead once we're using
78  # --build-id=sha1.
79  # pylint: disable=W0631
80  if (_ElfSectionMd5Sum(unstripped_host_lib, _TEXT_SECTION) !=
81      _ElfSectionMd5Sum(stripped_host_lib, _TEXT_SECTION)):
82    return None
83  return unstripped_host_lib
84
85
86@decorators.Cache
87def GetPerfhostName():
88  return 'perfhost_' + telemetry_platform.GetHostPlatform().GetOSVersionName()
89
90
91# Ignored directories for libraries that aren't useful for symbolization.
92_IGNORED_LIB_PATHS = [
93  '/data/dalvik-cache',
94  '/tmp'
95]
96
97
98def GetRequiredLibrariesForPerfProfile(profile_file):
99  """Returns the set of libraries necessary to symbolize a given perf profile.
100
101  Args:
102    profile_file: Path to perf profile to analyse.
103
104  Returns:
105    A set of required library file names.
106  """
107  with open(os.devnull, 'w') as dev_null:
108    perfhost_path = support_binaries.FindPath(GetPerfhostName(), 'linux')
109    perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file],
110                             stdout=dev_null, stderr=subprocess.PIPE)
111    _, output = perf.communicate()
112  missing_lib_re = re.compile(
113      r'^Failed to open (.*), continuing without symbols')
114  libs = set()
115  for line in output.split('\n'):
116    lib = missing_lib_re.match(line)
117    if lib:
118      lib = lib.group(1)
119      path = os.path.dirname(lib)
120      if any(path.startswith(ignored_path)
121             for ignored_path in _IGNORED_LIB_PATHS) or path == '/':
122        continue
123      libs.add(lib)
124  return libs
125
126
127def GetRequiredLibrariesForVTuneProfile(profile_file):
128  """Returns the set of libraries necessary to symbolize a given VTune profile.
129
130  Args:
131    profile_file: Path to VTune profile to analyse.
132
133  Returns:
134    A set of required library file names.
135  """
136  db_file = os.path.join(profile_file, 'sqlite-db', 'dicer.db')
137  conn = sqlite3.connect(db_file)
138
139  try:
140    # The 'dd_module_file' table lists all libraries on the device. Only the
141    # ones with 'bin_located_path' are needed for the profile.
142    query = 'SELECT bin_path, bin_located_path FROM dd_module_file'
143    return set(row[0] for row in conn.cursor().execute(query) if row[1])
144  finally:
145    conn.close()
146
147
148def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
149  """Creates a symfs directory to be used for symbolizing profiles.
150
151  Prepares a set of files ("symfs") to be used with profilers such as perf for
152  converting binary addresses into human readable function names.
153
154  Args:
155    device: DeviceUtils instance identifying the target device.
156    symfs_dir: Path where the symfs should be created.
157    libraries: Set of library file names that should be included in the symfs.
158    use_symlinks: If True, link instead of copy unstripped libraries into the
159      symfs. This will speed up the operation, but the resulting symfs will no
160      longer be valid if the linked files are modified, e.g., by rebuilding.
161
162  Returns:
163    The absolute path to the kernel symbols within the created symfs.
164  """
165  logging.info('Building symfs into %s.' % symfs_dir)
166
167  mismatching_files = {}
168  for lib in libraries:
169    device_dir = os.path.dirname(lib)
170    output_dir = os.path.join(symfs_dir, device_dir[1:])
171    if not os.path.exists(output_dir):
172      os.makedirs(output_dir)
173    output_lib = os.path.join(output_dir, os.path.basename(lib))
174
175    if lib.startswith('/data/app/'):
176      # If this is our own library instead of a system one, look for a matching
177      # unstripped library under the out directory.
178      unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(device, lib)
179      if not unstripped_host_lib:
180        logging.warning('Could not find symbols for %s.' % lib)
181        logging.warning('Is the correct output directory selected '
182                        '(CHROMIUM_OUT_DIR)? Did you install the APK after '
183                        'building?')
184        continue
185      if use_symlinks:
186        if os.path.lexists(output_lib):
187          os.remove(output_lib)
188        os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
189      # Copy the unstripped library only if it has been changed to avoid the
190      # delay. Add one second to the modification time to guard against file
191      # systems with poor timestamp resolution.
192      elif not os.path.exists(output_lib) or \
193          (os.stat(unstripped_host_lib).st_mtime >
194           os.stat(output_lib).st_mtime + 1):
195        logging.info('Copying %s to %s' % (unstripped_host_lib, output_lib))
196        shutil.copy2(unstripped_host_lib, output_lib)
197    else:
198      # Otherwise save a copy of the stripped system library under the symfs so
199      # the profiler can at least use the public symbols of that library. To
200      # speed things up, only pull files that don't match copies we already
201      # have in the symfs.
202      if not device_dir in mismatching_files:
203        changed_files = device.old_interface.GetFilesChanged(output_dir,
204                                                             device_dir)
205        mismatching_files[device_dir] = [
206            device_path for _, device_path in changed_files]
207
208      if not os.path.exists(output_lib) or lib in mismatching_files[device_dir]:
209        logging.info('Pulling %s to %s' % (lib, output_lib))
210        device.PullFile(lib, output_lib)
211
212  # Also pull a copy of the kernel symbols.
213  output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
214  if not os.path.exists(output_kallsyms):
215    device.PullFile('/proc/kallsyms', output_kallsyms)
216  return output_kallsyms
217
218
219def PrepareDeviceForPerf(device):
220  """Set up a device for running perf.
221
222  Args:
223    device: DeviceUtils instance identifying the target device.
224
225  Returns:
226    The path to the installed perf binary on the device.
227  """
228  android_prebuilt_profiler_helper.InstallOnDevice(device, 'perf')
229  # Make sure kernel pointers are not hidden.
230  device.WriteFile('/proc/sys/kernel/kptr_restrict', '0', as_root=True)
231  return android_prebuilt_profiler_helper.GetDevicePath('perf')
232
233
234def GetToolchainBinaryPath(library_file, binary_name):
235  """Return the path to an Android toolchain binary on the host.
236
237  Args:
238    library_file: ELF library which is used to identify the used ABI,
239        architecture and toolchain.
240    binary_name: Binary to search for, e.g., 'objdump'
241  Returns:
242    Full path to binary or None if the binary was not found.
243  """
244  # Mapping from ELF machine identifiers to GNU toolchain names.
245  toolchain_configs = {
246    'x86': 'i686-linux-android',
247    'MIPS': 'mipsel-linux-android',
248    'ARM': 'arm-linux-androideabi',
249    'x86-64': 'x86_64-linux-android',
250    'AArch64': 'aarch64-linux-android',
251  }
252  toolchain_config = toolchain_configs[_ElfMachineId(library_file)]
253  host_os = platform.uname()[0].lower()
254  host_machine = platform.uname()[4]
255
256  elf_comment = _ElfSectionAsString(library_file, '.comment')
257  toolchain_version = re.match(r'.*GCC: \(GNU\) ([\w.]+)',
258                               elf_comment, re.DOTALL)
259  if not toolchain_version:
260    return None
261  toolchain_version = toolchain_version.group(1)
262
263  path = os.path.join(util.GetChromiumSrcDir(), 'third_party', 'android_tools',
264                      'ndk', 'toolchains',
265                      '%s-%s' % (toolchain_config, toolchain_version),
266                      'prebuilt', '%s-%s' % (host_os, host_machine), 'bin',
267                      '%s-%s' % (toolchain_config, binary_name))
268  path = os.path.abspath(path)
269  return path if os.path.exists(path) else None
270