android_backend.py revision 1320f92c476a1ad9d19dba2a48c72b75566198e9
1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Android-specific implementation of the core backend interfaces. 6 7See core/backends.py for more docs. 8""" 9 10import datetime 11import glob 12import hashlib 13import json 14import os 15import posixpath 16 17from memory_inspector import constants 18from memory_inspector.backends import memdump_parser 19from memory_inspector.backends import native_heap_dump_parser 20from memory_inspector.backends import prebuilts_fetcher 21from memory_inspector.core import backends 22from memory_inspector.core import exceptions 23from memory_inspector.core import native_heap 24from memory_inspector.core import symbol 25 26# The memory_inspector/__init__ module will add the <CHROME_SRC>/build/android 27# deps to the PYTHONPATH for pylib. 28from pylib import android_commands 29from pylib.device import device_errors 30from pylib.device import device_utils 31from pylib.symbols import elf_symbolizer 32 33 34_SUPPORTED_32BIT_ABIS = {'armeabi': 'arm', 'armeabi-v7a': 'arm'} 35_SUPPORTED_64BIT_ABIS = {'arm64-v8a': 'arm64'} 36_MEMDUMP_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH, 37 'memdump-android-%(arch)s') 38_MEMDUMP_PATH_ON_DEVICE = '/data/local/tmp/memdump' 39_PSEXT_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH, 40 'ps_ext-android-%(arch)s') 41_PSEXT_PATH_ON_DEVICE = '/data/local/tmp/ps_ext' 42_HEAP_DUMP_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH, 43 'heap_dump-android-%(arch)s') 44_HEAP_DUMP_PATH_ON_DEVICE = '/data/local/tmp/heap_dump' 45_LIBHEAPPROF_PREBUILT_PATH = os.path.join(constants.PREBUILTS_PATH, 46 'libheap_profiler-android-%(arch)s') 47_LIBHEAPPROF_FILE_NAME = 'libheap_profiler.so' 48 49 50class AndroidBackend(backends.Backend): 51 """Android-specific implementation of the core |Backend| interface.""" 52 53 _SETTINGS_KEYS = { 54 'adb_path': 'Path of directory containing the adb binary', 55 'toolchain_path': 'Path of toolchain (for addr2line)'} 56 57 def __init__(self): 58 super(AndroidBackend, self).__init__( 59 settings=backends.Settings(AndroidBackend._SETTINGS_KEYS)) 60 self._devices = {} # 'device id' -> |Device|. 61 62 def EnumerateDevices(self): 63 # If a custom adb_path has been setup through settings, prepend that to the 64 # PATH. The android_commands module will use that to locate adb. 65 if (self.settings['adb_path'] and 66 not os.environ['PATH'].startswith(self.settings['adb_path'])): 67 os.environ['PATH'] = os.pathsep.join([self.settings['adb_path'], 68 os.environ['PATH']]) 69 for device_id in android_commands.GetAttachedDevices(): 70 device = self._devices.get(device_id) 71 if not device: 72 device = AndroidDevice( 73 self, device_utils.DeviceUtils(device_id)) 74 self._devices[device_id] = device 75 yield device 76 77 def ExtractSymbols(self, native_heaps, sym_paths): 78 """Performs symbolization. Returns a |symbol.Symbols| from |NativeHeap|s. 79 80 This method performs the symbolization but does NOT decorate (i.e. add 81 symbol/source info) to the stack frames of |native_heaps|. The heaps 82 can be decorated as needed using the native_heap.SymbolizeUsingSymbolDB() 83 method. Rationale: the most common use case in this application is: 84 symbolize-and-store-symbols and load-symbols-and-decorate-heaps (in two 85 different stages at two different times). 86 87 Args: 88 native_heaps: a collection of native_heap.NativeHeap instances. 89 sym_paths: either a list of or a string of semicolon-sep. symbol paths. 90 """ 91 assert(all(isinstance(x, native_heap.NativeHeap) for x in native_heaps)) 92 symbols = symbol.Symbols() 93 94 # Find addr2line in toolchain_path. 95 if isinstance(sym_paths, basestring): 96 sym_paths = sym_paths.split(';') 97 matches = glob.glob(os.path.join(self.settings['toolchain_path'], 98 '*addr2line')) 99 if not matches: 100 raise exceptions.MemoryInspectorException('Cannot find addr2line') 101 addr2line_path = matches[0] 102 103 # First group all the stack frames together by lib path. 104 frames_by_lib = {} 105 for nheap in native_heaps: 106 for stack_frame in nheap.stack_frames.itervalues(): 107 frames = frames_by_lib.setdefault(stack_frame.exec_file_rel_path, set()) 108 frames.add(stack_frame) 109 110 # The symbolization process is asynchronous (but yet single-threaded). This 111 # callback is invoked every time the symbol info for a stack frame is ready. 112 def SymbolizeAsyncCallback(sym_info, stack_frame): 113 if not sym_info.name: 114 return 115 sym = symbol.Symbol(name=sym_info.name, 116 source_file_path=sym_info.source_path, 117 line_number=sym_info.source_line) 118 symbols.Add(stack_frame.exec_file_rel_path, stack_frame.offset, sym) 119 # TODO(primiano): support inline sym info (i.e. |sym_info.inlined_by|). 120 121 # Perform the actual symbolization (ordered by lib). 122 for exec_file_rel_path, frames in frames_by_lib.iteritems(): 123 # Look up the full path of the symbol in the sym paths. 124 exec_file_name = posixpath.basename(exec_file_rel_path) 125 if exec_file_rel_path.startswith('/'): 126 exec_file_rel_path = exec_file_rel_path[1:] 127 if not exec_file_rel_path: 128 continue 129 exec_file_abs_path = '' 130 for sym_path in sym_paths: 131 # First try to locate the symbol file following the full relative path 132 # e.g. /host/syms/ + /system/lib/foo.so => /host/syms/system/lib/foo.so. 133 exec_file_abs_path = os.path.join(sym_path, exec_file_rel_path) 134 if os.path.exists(exec_file_abs_path): 135 break 136 137 # If no luck, try looking just for the file name in the sym path, 138 # e.g. /host/syms/ + (/system/lib/)foo.so => /host/syms/foo.so. 139 exec_file_abs_path = os.path.join(sym_path, exec_file_name) 140 if os.path.exists(exec_file_abs_path): 141 break 142 143 # In the case of a Chrome component=shared_library build, the libs are 144 # renamed to .cr.so. Look for foo.so => foo.cr.so. 145 exec_file_abs_path = os.path.join( 146 sym_path, exec_file_name.replace('.so', '.cr.so')) 147 if os.path.exists(exec_file_abs_path): 148 break 149 150 if not os.path.isfile(exec_file_abs_path): 151 continue 152 153 symbolizer = elf_symbolizer.ELFSymbolizer( 154 elf_file_path=exec_file_abs_path, 155 addr2line_path=addr2line_path, 156 callback=SymbolizeAsyncCallback, 157 inlines=False) 158 159 # Kick off the symbolizer and then wait that all callbacks are issued. 160 for stack_frame in sorted(frames, key=lambda x: x.offset): 161 symbolizer.SymbolizeAsync(stack_frame.offset, stack_frame) 162 symbolizer.Join() 163 164 return symbols 165 166 @property 167 def name(self): 168 return 'Android' 169 170 171class AndroidDevice(backends.Device): 172 """Android-specific implementation of the core |Device| interface.""" 173 174 _SETTINGS_KEYS = { 175 'native_symbol_paths': 'Semicolon-sep. list of native libs search path'} 176 177 def __init__(self, backend, adb): 178 super(AndroidDevice, self).__init__( 179 backend=backend, 180 settings=backends.Settings(AndroidDevice._SETTINGS_KEYS)) 181 self.adb = adb 182 self._name = '%s %s' % (adb.GetProp('ro.product.model'), 183 adb.GetProp('ro.build.id')) 184 self._id = str(adb) 185 self._sys_stats = None 186 self._last_device_stats = None 187 self._sys_stats_last_update = None 188 self._processes = {} # pid (int) -> |Process| 189 self._initialized = False 190 191 # Determine the available ABIs, |_arch| will contain the primary ABI. 192 # TODO(primiano): For the moment we support only one ABI per device (i.e. we 193 # assume that all processes are 64 bit on 64 bit device, failing to profile 194 # 32 bit ones). Dealing properly with multi-ABIs requires work on ps_ext and 195 # at the moment is not an interesting use case. 196 self._arch = None 197 self._arch32 = None 198 self._arch64 = None 199 abi = adb.GetProp('ro.product.cpu.abi') 200 if abi in _SUPPORTED_64BIT_ABIS: 201 self._arch = self._arch64 = _SUPPORTED_64BIT_ABIS[abi] 202 elif abi in _SUPPORTED_32BIT_ABIS: 203 self._arch = self._arch32 = _SUPPORTED_32BIT_ABIS[abi] 204 else: 205 raise exceptions.MemoryInspectorException('ABI %s not supported' % abi) 206 207 def Initialize(self): 208 """Starts adb root and deploys the prebuilt binaries on initialization.""" 209 try: 210 self.adb.EnableRoot() 211 except device_errors.CommandFailedError: 212 # TODO(jbudorick): Handle this exception appropriately after interface 213 # conversions are finished. 214 raise exceptions.MemoryInspectorException( 215 'The device must be adb root-able in order to use memory_inspector') 216 217 # Download (from GCS) and deploy prebuilt helper binaries on the device. 218 self._DeployPrebuiltOnDeviceIfNeeded( 219 _MEMDUMP_PREBUILT_PATH % {'arch': self._arch}, _MEMDUMP_PATH_ON_DEVICE) 220 self._DeployPrebuiltOnDeviceIfNeeded( 221 _PSEXT_PREBUILT_PATH % {'arch': self._arch}, _PSEXT_PATH_ON_DEVICE) 222 self._DeployPrebuiltOnDeviceIfNeeded( 223 _HEAP_DUMP_PREBUILT_PATH % {'arch': self._arch}, 224 _HEAP_DUMP_PATH_ON_DEVICE) 225 226 self._initialized = True 227 228 def IsNativeTracingEnabled(self): 229 """Checks whether the libheap_profiler is preloaded in the zygote.""" 230 zygote_name = 'zygote64' if self._arch64 else 'zygote' 231 zygote_process = [p for p in self.ListProcesses() if p.name == zygote_name] 232 if not zygote_process: 233 raise exceptions.MemoryInspectorException('Zygote process not found') 234 zygote_pid = zygote_process[0].pid 235 zygote_maps = self.adb.RunShellCommand('cat /proc/%d/maps' % zygote_pid) 236 return any(('libheap_profiler' in line for line in zygote_maps)) 237 238 def EnableNativeTracing(self, enabled): 239 """Installs libheap_profiler in and injects it in the Zygote.""" 240 241 def WrapZygote(app_process): 242 WRAPPER_SCRIPT = ('#!/system/bin/sh\n' 243 'LD_PRELOAD="libheap_profiler.so:$LD_PRELOAD" ' 244 'exec %s.real "$@"\n' % app_process) 245 self.adb.RunShellCommand('mv %(0)s %(0)s.real' % {'0': app_process}) 246 self.adb.WriteFile(app_process, WRAPPER_SCRIPT) 247 self.adb.RunShellCommand('chown root.shell ' + app_process) 248 self.adb.RunShellCommand('chmod 755 ' + app_process) 249 250 def UnwrapZygote(): 251 for suffix in ('', '32', '64'): 252 # We don't really care if app_processX.real doesn't exists and mv fails. 253 # If app_processX.real doesn't exists, either app_processX is already 254 # unwrapped or it doesn't exists for the current arch. 255 app_process = '/system/bin/app_process' + suffix 256 self.adb.RunShellCommand('mv %(0)s.real %(0)s' % {'0': app_process}) 257 258 assert(self._initialized) 259 self.adb.old_interface.MakeSystemFolderWritable() 260 261 # Start restoring the original state in any case. 262 UnwrapZygote() 263 264 if enabled: 265 # Temporarily disable SELinux (until next reboot). 266 self.adb.RunShellCommand('setenforce 0') 267 268 # Wrap the Zygote startup binary (app_process) with a script which 269 # LD_PRELOADs libheap_profiler and invokes the original Zygote process. 270 if self._arch64: 271 app_process = '/system/bin/app_process64' 272 assert(self.adb.FileExists(app_process)) 273 self._DeployPrebuiltOnDeviceIfNeeded( 274 _LIBHEAPPROF_PREBUILT_PATH % {'arch': self._arch64}, 275 '/system/lib64/' + _LIBHEAPPROF_FILE_NAME) 276 WrapZygote(app_process) 277 278 if self._arch32: 279 # Path is app_process32 for Android >= L, app_process when < L. 280 app_process = '/system/bin/app_process32' 281 if not self.adb.FileExists(app_process): 282 app_process = '/system/bin/app_process' 283 assert(self.adb.FileExists(app_process)) 284 self._DeployPrebuiltOnDeviceIfNeeded( 285 _LIBHEAPPROF_PREBUILT_PATH % {'arch': self._arch32}, 286 '/system/lib/' + _LIBHEAPPROF_FILE_NAME) 287 WrapZygote(app_process) 288 289 # Respawn the zygote (the device will kind of reboot at this point). 290 self.adb.old_interface.RestartShell() 291 self.adb.old_interface.Adb().WaitForDevicePm(wait_time=30) 292 293 # Remove the wrapper. This won't have effect until the next reboot, when 294 # the profiler will be automatically disarmed. 295 UnwrapZygote() 296 297 # We can also unlink the lib files at this point. Once the Zygote has 298 # started it will keep the inodes refcounted anyways through its lifetime. 299 self.adb.RunShellCommand('rm /system/lib*/' + _LIBHEAPPROF_FILE_NAME) 300 301 def ListProcesses(self): 302 """Returns a sequence of |AndroidProcess|.""" 303 self._RefreshProcessesList() 304 return self._processes.itervalues() 305 306 def GetProcess(self, pid): 307 """Returns an instance of |AndroidProcess| (None if not found).""" 308 assert(isinstance(pid, int)) 309 self._RefreshProcessesList() 310 return self._processes.get(pid) 311 312 def GetStats(self): 313 """Returns an instance of |DeviceStats| with the OS CPU/Memory stats.""" 314 cur = self.UpdateAndGetSystemStats() 315 old = self._last_device_stats or cur # Handle 1st call case. 316 uptime = cur['time']['ticks'] / cur['time']['rate'] 317 ticks = max(1, cur['time']['ticks'] - old['time']['ticks']) 318 319 cpu_times = [] 320 for i in xrange(len(cur['cpu'])): 321 cpu_time = { 322 'usr': 100 * (cur['cpu'][i]['usr'] - old['cpu'][i]['usr']) / ticks, 323 'sys': 100 * (cur['cpu'][i]['sys'] - old['cpu'][i]['sys']) / ticks, 324 'idle': 100 * (cur['cpu'][i]['idle'] - old['cpu'][i]['idle']) / ticks} 325 # The idle tick count on many Linux kernels is frozen when the CPU is 326 # offline, and bumps up (compensating all the offline period) when it 327 # reactivates. For this reason it needs to be saturated at [0, 100]. 328 cpu_time['idle'] = max(0, min(cpu_time['idle'], 329 100 - cpu_time['usr'] - cpu_time['sys'])) 330 331 cpu_times.append(cpu_time) 332 333 memory_stats = {'Free': cur['mem']['MemFree:'], 334 'Cache': cur['mem']['Buffers:'] + cur['mem']['Cached:'], 335 'Swap': cur['mem']['SwapCached:'], 336 'Anonymous': cur['mem']['AnonPages:'], 337 'Kernel': cur['mem']['VmallocUsed:']} 338 self._last_device_stats = cur 339 340 return backends.DeviceStats(uptime=uptime, 341 cpu_times=cpu_times, 342 memory_stats=memory_stats) 343 344 def UpdateAndGetSystemStats(self): 345 """Grabs and caches system stats through ps_ext (max cache TTL = 0.5s). 346 347 Rationale of caching: avoid invoking adb too often, it is slow. 348 """ 349 assert(self._initialized) 350 max_ttl = datetime.timedelta(seconds=0.5) 351 if (self._sys_stats_last_update and 352 datetime.datetime.now() - self._sys_stats_last_update <= max_ttl): 353 return self._sys_stats 354 355 dump_out = '\n'.join( 356 self.adb.RunShellCommand(_PSEXT_PATH_ON_DEVICE)) 357 stats = json.loads(dump_out) 358 assert(all([x in stats for x in ['cpu', 'processes', 'time', 'mem']])), ( 359 'ps_ext returned a malformed JSON dictionary.') 360 self._sys_stats = stats 361 self._sys_stats_last_update = datetime.datetime.now() 362 return self._sys_stats 363 364 def _RefreshProcessesList(self): 365 sys_stats = self.UpdateAndGetSystemStats() 366 processes_to_delete = set(self._processes.keys()) 367 for pid, proc in sys_stats['processes'].iteritems(): 368 pid = int(pid) 369 process = self._processes.get(pid) 370 if not process or process.name != proc['name']: 371 process = AndroidProcess(self, int(pid), proc['name']) 372 self._processes[pid] = process 373 processes_to_delete.discard(pid) 374 for pid in processes_to_delete: 375 del self._processes[pid] 376 377 def _DeployPrebuiltOnDeviceIfNeeded(self, local_path, path_on_device): 378 # TODO(primiano): check that the md5 binary is built-in also on pre-KK. 379 # Alternatively add tools/android/md5sum to prebuilts and use that one. 380 prebuilts_fetcher.GetIfChanged(local_path) 381 with open(local_path, 'rb') as f: 382 local_hash = hashlib.md5(f.read()).hexdigest() 383 device_md5_out = self.adb.RunShellCommand( 384 'md5 "%s"' % path_on_device) 385 if local_hash in device_md5_out: 386 return 387 self.adb.old_interface.Adb().Push(local_path, path_on_device) 388 self.adb.RunShellCommand('chmod 755 "%s"' % path_on_device) 389 390 @property 391 def name(self): 392 """Device name, as defined in the |backends.Device| interface.""" 393 return self._name 394 395 @property 396 def id(self): 397 """Device id, as defined in the |backends.Device| interface.""" 398 return self._id 399 400 401class AndroidProcess(backends.Process): 402 """Android-specific implementation of the core |Process| interface.""" 403 404 def __init__(self, device, pid, name): 405 super(AndroidProcess, self).__init__(device, pid, name) 406 self._last_sys_stats = None 407 408 def DumpMemoryMaps(self): 409 """Grabs and parses memory maps through memdump.""" 410 cmd = '%s %d' % (_MEMDUMP_PATH_ON_DEVICE, self.pid) 411 dump_out = self.device.adb.RunShellCommand(cmd) 412 return memdump_parser.Parse(dump_out) 413 414 def DumpNativeHeap(self): 415 """Grabs and parses native heap traces using heap_dump.""" 416 cmd = '%s -n -x %d' % (_HEAP_DUMP_PATH_ON_DEVICE, self.pid) 417 out_lines = self.device.adb.RunShellCommand(cmd) 418 return native_heap_dump_parser.Parse('\n'.join(out_lines)) 419 420 def Freeze(self): 421 self.device.adb.RunShellCommand('kill -STOP %d' % self.pid) 422 423 def Unfreeze(self): 424 self.device.adb.RunShellCommand('kill -CONT %d' % self.pid) 425 426 def GetStats(self): 427 """Calculate process CPU/VM stats (CPU stats are relative to last call).""" 428 # Process must retain its own copy of _last_sys_stats because CPU times 429 # are calculated relatively to the last GetStats() call (for the process). 430 cur_sys_stats = self.device.UpdateAndGetSystemStats() 431 old_sys_stats = self._last_sys_stats or cur_sys_stats 432 cur_proc_stats = cur_sys_stats['processes'].get(str(self.pid)) 433 old_proc_stats = old_sys_stats['processes'].get(str(self.pid)) 434 435 # The process might have gone in the meanwhile. 436 if (not cur_proc_stats or not old_proc_stats): 437 return None 438 439 run_time = (((cur_sys_stats['time']['ticks'] - 440 cur_proc_stats['start_time']) / cur_sys_stats['time']['rate'])) 441 ticks = max(1, cur_sys_stats['time']['ticks'] - 442 old_sys_stats['time']['ticks']) 443 cpu_usage = (100 * 444 ((cur_proc_stats['user_time'] + cur_proc_stats['sys_time']) - 445 (old_proc_stats['user_time'] + old_proc_stats['sys_time'])) / 446 ticks) / len(cur_sys_stats['cpu']) 447 proc_stats = backends.ProcessStats( 448 threads=cur_proc_stats['n_threads'], 449 run_time=run_time, 450 cpu_usage=cpu_usage, 451 vm_rss=cur_proc_stats['vm_rss'], 452 page_faults=( 453 (cur_proc_stats['maj_faults'] + cur_proc_stats['min_faults']) - 454 (old_proc_stats['maj_faults'] + old_proc_stats['min_faults']))) 455 self._last_sys_stats = cur_sys_stats 456 return proc_stats 457