1#!/usr/bin/env python 2# Copyright 2016 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""A script to recover devices in a known bad state.""" 7 8import argparse 9import logging 10import os 11import psutil 12import signal 13import sys 14 15if __name__ == '__main__': 16 sys.path.append( 17 os.path.abspath(os.path.join(os.path.dirname(__file__), 18 '..', '..', '..'))) 19from devil import devil_env 20from devil.android import device_blacklist 21from devil.android import device_errors 22from devil.android import device_utils 23from devil.android.tools import device_status 24from devil.utils import lsusb 25# TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset. 26from devil.utils import reset_usb # pylint: disable=unused-import 27from devil.utils import run_tests_helper 28 29logger = logging.getLogger(__name__) 30 31 32def KillAllAdb(): 33 def get_all_adb(): 34 for p in psutil.process_iter(): 35 try: 36 if 'adb' in p.name: 37 yield p 38 except (psutil.NoSuchProcess, psutil.AccessDenied): 39 pass 40 41 for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: 42 for p in get_all_adb(): 43 try: 44 logger.info('kill %d %d (%s [%s])', sig, p.pid, p.name, 45 ' '.join(p.cmdline)) 46 p.send_signal(sig) 47 except (psutil.NoSuchProcess, psutil.AccessDenied): 48 pass 49 for p in get_all_adb(): 50 try: 51 logger.error('Unable to kill %d (%s [%s])', p.pid, p.name, 52 ' '.join(p.cmdline)) 53 except (psutil.NoSuchProcess, psutil.AccessDenied): 54 pass 55 56 57def RecoverDevice(device, blacklist, should_reboot=lambda device: True): 58 if device_status.IsBlacklisted(device.adb.GetDeviceSerial(), 59 blacklist): 60 logger.debug('%s is blacklisted, skipping recovery.', str(device)) 61 return 62 63 if should_reboot(device): 64 try: 65 device.WaitUntilFullyBooted(retries=0) 66 except (device_errors.CommandTimeoutError, 67 device_errors.CommandFailedError): 68 logger.exception('Failure while waiting for %s. ' 69 'Attempting to recover.', str(device)) 70 try: 71 try: 72 device.Reboot(block=False, timeout=5, retries=0) 73 except device_errors.CommandTimeoutError: 74 logger.warning('Timed out while attempting to reboot %s normally.' 75 'Attempting alternative reboot.', str(device)) 76 # The device drops offline before we can grab the exit code, so 77 # we don't check for status. 78 try: 79 device.adb.Root() 80 finally: 81 # We are already in a failure mode, attempt to reboot regardless of 82 # what device.adb.Root() returns. If the sysrq reboot fails an 83 # exception willbe thrown at that level. 84 device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None, 85 timeout=5, retries=0) 86 except device_errors.CommandFailedError: 87 logger.exception('Failed to reboot %s.', str(device)) 88 if blacklist: 89 blacklist.Extend([device.adb.GetDeviceSerial()], 90 reason='reboot_failure') 91 except device_errors.CommandTimeoutError: 92 logger.exception('Timed out while rebooting %s.', str(device)) 93 if blacklist: 94 blacklist.Extend([device.adb.GetDeviceSerial()], 95 reason='reboot_timeout') 96 97 try: 98 device.WaitUntilFullyBooted( 99 retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT) 100 except device_errors.CommandFailedError: 101 logger.exception('Failure while waiting for %s.', str(device)) 102 if blacklist: 103 blacklist.Extend([device.adb.GetDeviceSerial()], 104 reason='reboot_failure') 105 except device_errors.CommandTimeoutError: 106 logger.exception('Timed out while waiting for %s.', str(device)) 107 if blacklist: 108 blacklist.Extend([device.adb.GetDeviceSerial()], 109 reason='reboot_timeout') 110 111 112def RecoverDevices(devices, blacklist, enable_usb_reset=False): 113 """Attempts to recover any inoperable devices in the provided list. 114 115 Args: 116 devices: The list of devices to attempt to recover. 117 blacklist: The current device blacklist, which will be used then 118 reset. 119 """ 120 121 statuses = device_status.DeviceStatus(devices, blacklist) 122 123 should_restart_usb = set( 124 status['serial'] for status in statuses 125 if (not status['usb_status'] 126 or status['adb_status'] in ('offline', 'missing'))) 127 should_restart_adb = should_restart_usb.union(set( 128 status['serial'] for status in statuses 129 if status['adb_status'] == 'unauthorized')) 130 should_reboot_device = should_restart_adb.union(set( 131 status['serial'] for status in statuses 132 if status['blacklisted'])) 133 134 logger.debug('Should restart USB for:') 135 for d in should_restart_usb: 136 logger.debug(' %s', d) 137 logger.debug('Should restart ADB for:') 138 for d in should_restart_adb: 139 logger.debug(' %s', d) 140 logger.debug('Should reboot:') 141 for d in should_reboot_device: 142 logger.debug(' %s', d) 143 144 if blacklist: 145 blacklist.Reset() 146 147 if should_restart_adb: 148 KillAllAdb() 149 for serial in should_restart_usb: 150 try: 151 # TODO(crbug.com/642194): Resetting may be causing more harm 152 # (specifically, kernel panics) than it does good. 153 if enable_usb_reset: 154 reset_usb.reset_android_usb(serial) 155 else: 156 logger.warning('USB reset disabled for %s (crbug.com/642914)', 157 serial) 158 except IOError: 159 logger.exception('Unable to reset USB for %s.', serial) 160 if blacklist: 161 blacklist.Extend([serial], reason='USB failure') 162 except device_errors.DeviceUnreachableError: 163 logger.exception('Unable to reset USB for %s.', serial) 164 if blacklist: 165 blacklist.Extend([serial], reason='offline') 166 167 device_utils.DeviceUtils.parallel(devices).pMap( 168 RecoverDevice, blacklist, 169 should_reboot=lambda device: device.serial in should_reboot_device) 170 171 172def main(): 173 parser = argparse.ArgumentParser() 174 parser.add_argument('--adb-path', 175 help='Absolute path to the adb binary to use.') 176 parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') 177 parser.add_argument('--known-devices-file', action='append', default=[], 178 dest='known_devices_files', 179 help='Path to known device lists.') 180 parser.add_argument('--enable-usb-reset', action='store_true', 181 help='Reset USB if necessary.') 182 parser.add_argument('-v', '--verbose', action='count', default=1, 183 help='Log more information.') 184 185 args = parser.parse_args() 186 run_tests_helper.SetLogLevel(args.verbose) 187 188 devil_dynamic_config = devil_env.EmptyConfig() 189 if args.adb_path: 190 devil_dynamic_config['dependencies'].update( 191 devil_env.LocalConfigItem( 192 'adb', devil_env.GetPlatform(), args.adb_path)) 193 devil_env.config.Initialize(configs=[devil_dynamic_config]) 194 195 blacklist = (device_blacklist.Blacklist(args.blacklist_file) 196 if args.blacklist_file 197 else None) 198 199 expected_devices = device_status.GetExpectedDevices(args.known_devices_files) 200 usb_devices = set(lsusb.get_android_devices()) 201 devices = [device_utils.DeviceUtils(s) 202 for s in expected_devices.union(usb_devices)] 203 204 RecoverDevices(devices, blacklist, enable_usb_reset=args.enable_usb_reset) 205 206 207if __name__ == '__main__': 208 sys.exit(main()) 209