1#!/usr/bin/env python
2# Copyright 2016 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""A script to recover devices in a known bad state."""
7
8import argparse
9import logging
10import os
11import psutil
12import signal
13import sys
14
15if __name__ == '__main__':
16  sys.path.append(
17      os.path.abspath(os.path.join(os.path.dirname(__file__),
18                                   '..', '..', '..')))
19from devil import devil_env
20from devil.android import device_blacklist
21from devil.android import device_errors
22from devil.android import device_utils
23from devil.android.tools import device_status
24from devil.utils import lsusb
25# TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset.
26from devil.utils import reset_usb  # pylint: disable=unused-import
27from devil.utils import run_tests_helper
28
29logger = logging.getLogger(__name__)
30
31
32def KillAllAdb():
33  def get_all_adb():
34    for p in psutil.process_iter():
35      try:
36        if 'adb' in p.name:
37          yield p
38      except (psutil.NoSuchProcess, psutil.AccessDenied):
39        pass
40
41  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
42    for p in get_all_adb():
43      try:
44        logger.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
45                    ' '.join(p.cmdline))
46        p.send_signal(sig)
47      except (psutil.NoSuchProcess, psutil.AccessDenied):
48        pass
49  for p in get_all_adb():
50    try:
51      logger.error('Unable to kill %d (%s [%s])', p.pid, p.name,
52                   ' '.join(p.cmdline))
53    except (psutil.NoSuchProcess, psutil.AccessDenied):
54      pass
55
56
57def RecoverDevice(device, blacklist, should_reboot=lambda device: True):
58  if device_status.IsBlacklisted(device.adb.GetDeviceSerial(),
59                                 blacklist):
60    logger.debug('%s is blacklisted, skipping recovery.', str(device))
61    return
62
63  if should_reboot(device):
64    try:
65      device.WaitUntilFullyBooted(retries=0)
66    except (device_errors.CommandTimeoutError,
67            device_errors.CommandFailedError):
68      logger.exception('Failure while waiting for %s. '
69                       'Attempting to recover.', str(device))
70    try:
71      try:
72        device.Reboot(block=False, timeout=5, retries=0)
73      except device_errors.CommandTimeoutError:
74        logger.warning('Timed out while attempting to reboot %s normally.'
75                       'Attempting alternative reboot.', str(device))
76        # The device drops offline before we can grab the exit code, so
77        # we don't check for status.
78        try:
79          device.adb.Root()
80        finally:
81          # We are already in a failure mode, attempt to reboot regardless of
82          # what device.adb.Root() returns. If the sysrq reboot fails an
83          # exception willbe thrown at that level.
84          device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
85                           timeout=5, retries=0)
86    except device_errors.CommandFailedError:
87      logger.exception('Failed to reboot %s.', str(device))
88      if blacklist:
89        blacklist.Extend([device.adb.GetDeviceSerial()],
90                         reason='reboot_failure')
91    except device_errors.CommandTimeoutError:
92      logger.exception('Timed out while rebooting %s.', str(device))
93      if blacklist:
94        blacklist.Extend([device.adb.GetDeviceSerial()],
95                         reason='reboot_timeout')
96
97    try:
98      device.WaitUntilFullyBooted(
99          retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT)
100    except device_errors.CommandFailedError:
101      logger.exception('Failure while waiting for %s.', str(device))
102      if blacklist:
103        blacklist.Extend([device.adb.GetDeviceSerial()],
104                         reason='reboot_failure')
105    except device_errors.CommandTimeoutError:
106      logger.exception('Timed out while waiting for %s.', str(device))
107      if blacklist:
108        blacklist.Extend([device.adb.GetDeviceSerial()],
109                         reason='reboot_timeout')
110
111
112def RecoverDevices(devices, blacklist, enable_usb_reset=False):
113  """Attempts to recover any inoperable devices in the provided list.
114
115  Args:
116    devices: The list of devices to attempt to recover.
117    blacklist: The current device blacklist, which will be used then
118      reset.
119  """
120
121  statuses = device_status.DeviceStatus(devices, blacklist)
122
123  should_restart_usb = set(
124      status['serial'] for status in statuses
125      if (not status['usb_status']
126          or status['adb_status'] in ('offline', 'missing')))
127  should_restart_adb = should_restart_usb.union(set(
128      status['serial'] for status in statuses
129      if status['adb_status'] == 'unauthorized'))
130  should_reboot_device = should_restart_adb.union(set(
131      status['serial'] for status in statuses
132      if status['blacklisted']))
133
134  logger.debug('Should restart USB for:')
135  for d in should_restart_usb:
136    logger.debug('  %s', d)
137  logger.debug('Should restart ADB for:')
138  for d in should_restart_adb:
139    logger.debug('  %s', d)
140  logger.debug('Should reboot:')
141  for d in should_reboot_device:
142    logger.debug('  %s', d)
143
144  if blacklist:
145    blacklist.Reset()
146
147  if should_restart_adb:
148    KillAllAdb()
149  for serial in should_restart_usb:
150    try:
151      # TODO(crbug.com/642194): Resetting may be causing more harm
152      # (specifically, kernel panics) than it does good.
153      if enable_usb_reset:
154        reset_usb.reset_android_usb(serial)
155      else:
156        logger.warning('USB reset disabled for %s (crbug.com/642914)',
157                       serial)
158    except IOError:
159      logger.exception('Unable to reset USB for %s.', serial)
160      if blacklist:
161        blacklist.Extend([serial], reason='USB failure')
162    except device_errors.DeviceUnreachableError:
163      logger.exception('Unable to reset USB for %s.', serial)
164      if blacklist:
165        blacklist.Extend([serial], reason='offline')
166
167  device_utils.DeviceUtils.parallel(devices).pMap(
168      RecoverDevice, blacklist,
169      should_reboot=lambda device: device.serial in should_reboot_device)
170
171
172def main():
173  parser = argparse.ArgumentParser()
174  parser.add_argument('--adb-path',
175                      help='Absolute path to the adb binary to use.')
176  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
177  parser.add_argument('--known-devices-file', action='append', default=[],
178                      dest='known_devices_files',
179                      help='Path to known device lists.')
180  parser.add_argument('--enable-usb-reset', action='store_true',
181                      help='Reset USB if necessary.')
182  parser.add_argument('-v', '--verbose', action='count', default=1,
183                      help='Log more information.')
184
185  args = parser.parse_args()
186  run_tests_helper.SetLogLevel(args.verbose)
187
188  devil_dynamic_config = devil_env.EmptyConfig()
189  if args.adb_path:
190    devil_dynamic_config['dependencies'].update(
191        devil_env.LocalConfigItem(
192            'adb', devil_env.GetPlatform(), args.adb_path))
193  devil_env.config.Initialize(configs=[devil_dynamic_config])
194
195  blacklist = (device_blacklist.Blacklist(args.blacklist_file)
196               if args.blacklist_file
197               else None)
198
199  expected_devices = device_status.GetExpectedDevices(args.known_devices_files)
200  usb_devices = set(lsusb.get_android_devices())
201  devices = [device_utils.DeviceUtils(s)
202             for s in expected_devices.union(usb_devices)]
203
204  RecoverDevices(devices, blacklist, enable_usb_reset=args.enable_usb_reset)
205
206
207if __name__ == '__main__':
208  sys.exit(main())
209