1#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""A class to keep track of devices across builds and report state."""
8import json
9import logging
10import optparse
11import os
12import psutil
13import re
14import signal
15import smtplib
16import subprocess
17import sys
18import time
19import urllib
20
21import bb_annotations
22import bb_utils
23
24sys.path.append(os.path.join(os.path.dirname(__file__),
25                             os.pardir, os.pardir, 'util', 'lib',
26                             'common'))
27import perf_tests_results_helper  # pylint: disable=F0401
28
29sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
30from pylib import android_commands
31from pylib import constants
32from pylib.cmd_helper import GetCmdOutput
33from pylib.device import device_blacklist
34from pylib.device import device_errors
35from pylib.device import device_list
36from pylib.device import device_utils
37
38def DeviceInfo(serial, options):
39  """Gathers info on a device via various adb calls.
40
41  Args:
42    serial: The serial of the attached device to construct info about.
43
44  Returns:
45    Tuple of device type, build id, report as a string, error messages, and
46    boolean indicating whether or not device can be used for testing.
47  """
48
49  device_adb = device_utils.DeviceUtils(serial)
50  device_type = device_adb.GetProp('ro.build.product')
51  device_build = device_adb.GetProp('ro.build.id')
52  device_build_type = device_adb.GetProp('ro.build.type')
53  device_product_name = device_adb.GetProp('ro.product.name')
54
55  try:
56    battery_info = device_adb.old_interface.GetBatteryInfo()
57  except Exception as e:
58    battery_info = {}
59    logging.error('Unable to obtain battery info for %s, %s', serial, e)
60
61  def _GetData(re_expression, line, lambda_function=lambda x:x):
62    if not line:
63      return 'Unknown'
64    found = re.findall(re_expression, line)
65    if found and len(found):
66      return lambda_function(found[0])
67    return 'Unknown'
68
69  battery_level = int(battery_info.get('level', 100))
70  imei_slice = _GetData('Device ID = (\d+)',
71                        device_adb.old_interface.GetSubscriberInfo(),
72                        lambda x: x[-6:])
73  report = ['Device %s (%s)' % (serial, device_type),
74            '  Build: %s (%s)' %
75              (device_build, device_adb.GetProp('ro.build.fingerprint')),
76            '  Current Battery Service state: ',
77            '\n'.join(['    %s: %s' % (k, v)
78                       for k, v in battery_info.iteritems()]),
79            '  IMEI slice: %s' % imei_slice,
80            '  Wifi IP: %s' % device_adb.GetProp('dhcp.wlan0.ipaddress'),
81            '']
82
83  errors = []
84  dev_good = True
85  if battery_level < 15:
86    errors += ['Device critically low in battery. Turning off device.']
87    dev_good = False
88  if not options.no_provisioning_check:
89    setup_wizard_disabled = (
90        device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED')
91    if not setup_wizard_disabled and device_build_type != 'user':
92      errors += ['Setup wizard not disabled. Was it provisioned correctly?']
93  if (device_product_name == 'mantaray' and
94      battery_info.get('AC powered', None) != 'true'):
95    errors += ['Mantaray device not connected to AC power.']
96
97  # Turn off devices with low battery.
98  if battery_level < 15:
99    try:
100      device_adb.EnableRoot()
101    except device_errors.CommandFailedError as e:
102      # Attempt shutdown anyway.
103      # TODO(jbudorick) Handle this exception appropriately after interface
104      #                 conversions are finished.
105      logging.error(str(e))
106    device_adb.old_interface.Shutdown()
107  full_report = '\n'.join(report)
108  return device_type, device_build, battery_level, full_report, errors, dev_good
109
110
111def CheckForMissingDevices(options, adb_online_devs):
112  """Uses file of previous online devices to detect broken phones.
113
114  Args:
115    options: out_dir parameter of options argument is used as the base
116             directory to load and update the cache file.
117    adb_online_devs: A list of serial numbers of the currently visible
118                     and online attached devices.
119  """
120  # TODO(navabi): remove this once the bug that causes different number
121  # of devices to be detected between calls is fixed.
122  logger = logging.getLogger()
123  logger.setLevel(logging.INFO)
124
125  out_dir = os.path.abspath(options.out_dir)
126
127  # last_devices denotes all known devices prior to this run
128  last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME)
129  last_missing_devices_path = os.path.join(out_dir,
130      device_list.LAST_MISSING_DEVICES_FILENAME)
131  try:
132    last_devices = device_list.GetPersistentDeviceList(last_devices_path)
133  except IOError:
134    # Ignore error, file might not exist
135    last_devices = []
136
137  try:
138    last_missing_devices = device_list.GetPersistentDeviceList(
139        last_missing_devices_path)
140  except IOError:
141    last_missing_devices = []
142
143  missing_devs = list(set(last_devices) - set(adb_online_devs))
144  new_missing_devs = list(set(missing_devs) - set(last_missing_devices))
145
146  if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'):
147    logging.info('new_missing_devs %s' % new_missing_devs)
148    devices_missing_msg = '%d devices not detected.' % len(missing_devs)
149    bb_annotations.PrintSummaryText(devices_missing_msg)
150
151    from_address = 'chrome-bot@chromium.org'
152    to_addresses = ['chrome-labs-tech-ticket@google.com',
153                    'chrome-android-device-alert@google.com']
154    cc_addresses = ['chrome-android-device-alert@google.com']
155    subject = 'Devices offline on %s, %s, %s' % (
156      os.environ.get('BUILDBOT_SLAVENAME'),
157      os.environ.get('BUILDBOT_BUILDERNAME'),
158      os.environ.get('BUILDBOT_BUILDNUMBER'))
159    msg = ('Please reboot the following devices:\n%s' %
160           '\n'.join(map(str,new_missing_devs)))
161    SendEmail(from_address, to_addresses, cc_addresses, subject, msg)
162
163  all_known_devices = list(set(adb_online_devs) | set(last_devices))
164  device_list.WritePersistentDeviceList(last_devices_path, all_known_devices)
165  device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs)
166
167  if not all_known_devices:
168    # This can happen if for some reason the .last_devices file is not
169    # present or if it was empty.
170    return ['No online devices. Have any devices been plugged in?']
171  if missing_devs:
172    devices_missing_msg = '%d devices not detected.' % len(missing_devs)
173    bb_annotations.PrintSummaryText(devices_missing_msg)
174
175    # TODO(navabi): Debug by printing both output from GetCmdOutput and
176    # GetAttachedDevices to compare results.
177    crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary='
178                  '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' %
179                  (urllib.quote('Device Offline'),
180                   urllib.quote('Buildbot: %s %s\n'
181                                'Build: %s\n'
182                                '(please don\'t change any labels)' %
183                                (os.environ.get('BUILDBOT_BUILDERNAME'),
184                                 os.environ.get('BUILDBOT_SLAVENAME'),
185                                 os.environ.get('BUILDBOT_BUILDNUMBER')))))
186    return ['Current online devices: %s' % adb_online_devs,
187            '%s are no longer visible. Were they removed?\n' % missing_devs,
188            'SHERIFF:\n',
189            '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link,
190            'Cache file: %s\n\n' % last_devices_path,
191            'adb devices: %s' % GetCmdOutput(['adb', 'devices']),
192            'adb devices(GetAttachedDevices): %s' % adb_online_devs]
193  else:
194    new_devs = set(adb_online_devs) - set(last_devices)
195    if new_devs and os.path.exists(last_devices_path):
196      bb_annotations.PrintWarning()
197      bb_annotations.PrintSummaryText(
198          '%d new devices detected' % len(new_devs))
199      print ('New devices detected %s. And now back to your '
200             'regularly scheduled program.' % list(new_devs))
201
202
203def SendEmail(from_address, to_addresses, cc_addresses, subject, msg):
204  msg_body = '\r\n'.join(['From: %s' % from_address,
205                          'To: %s' % ', '.join(to_addresses),
206                          'CC: %s' % ', '.join(cc_addresses),
207                          'Subject: %s' % subject, '', msg])
208  try:
209    server = smtplib.SMTP('localhost')
210    server.sendmail(from_address, to_addresses, msg_body)
211    server.quit()
212  except Exception as e:
213    print 'Failed to send alert email. Error: %s' % e
214
215
216def RestartUsb():
217  if not os.path.isfile('/usr/bin/restart_usb'):
218    print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
219           'on host (see BUG=305769).')
220    return False
221
222  lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
223  lsusb_output, _ = lsusb_proc.communicate()
224  if lsusb_proc.returncode:
225    print ('Error: Could not get list of USB ports (i.e. lsusb).')
226    return lsusb_proc.returncode
227
228  usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
229                 for lsusb_line in lsusb_output.strip().split('\n')]
230
231  all_restarted = True
232  # Walk USB devices from leaves up (i.e reverse sorted) restarting the
233  # connection. If a parent node (e.g. usb hub) is restarted before the
234  # devices connected to it, the (bus, dev) for the hub can change, making the
235  # output we have wrong. This way we restart the devices before the hub.
236  for (bus, dev) in reversed(sorted(usb_devices)):
237    # Can not restart root usb connections
238    if dev != '001':
239      return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
240      if return_code:
241        print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
242        all_restarted = False
243      else:
244        print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
245
246  return all_restarted
247
248
249def KillAllAdb():
250  def GetAllAdb():
251    for p in psutil.process_iter():
252      try:
253        if 'adb' in p.name:
254          yield p
255      except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
256        pass
257
258  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
259    for p in GetAllAdb():
260      try:
261        print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name,
262            ' '.join(p.cmdline))
263        p.send_signal(sig)
264      except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
265        pass
266  for p in GetAllAdb():
267    try:
268      print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline))
269    except (psutil.error.NoSuchProcess, psutil.error.AccessDenied):
270      pass
271
272
273def main():
274  parser = optparse.OptionParser()
275  parser.add_option('', '--out-dir',
276                    help='Directory where the device path is stored',
277                    default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
278  parser.add_option('--no-provisioning-check', action='store_true',
279                    help='Will not check if devices are provisioned properly.')
280  parser.add_option('--device-status-dashboard', action='store_true',
281                    help='Output device status data for dashboard.')
282  parser.add_option('--restart-usb', action='store_true',
283                    help='Restart USB ports before running device check.')
284  parser.add_option('--json-output',
285                    help='Output JSON information into a specified file.')
286
287  options, args = parser.parse_args()
288  if args:
289    parser.error('Unknown options %s' % args)
290
291  # Remove the last build's "bad devices" before checking device statuses.
292  device_blacklist.ResetBlacklist()
293
294  try:
295    expected_devices = device_list.GetPersistentDeviceList(
296        os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME))
297  except IOError:
298    expected_devices = []
299  devices = android_commands.GetAttachedDevices()
300  # Only restart usb if devices are missing.
301  if set(expected_devices) != set(devices):
302    print 'expected_devices: %s, devices: %s' % (expected_devices, devices)
303    KillAllAdb()
304    retries = 5
305    usb_restarted = True
306    if options.restart_usb:
307      if not RestartUsb():
308        usb_restarted = False
309        bb_annotations.PrintWarning()
310        print 'USB reset stage failed, wait for any device to come back.'
311    while retries:
312      print 'retry adb devices...'
313      time.sleep(1)
314      devices = android_commands.GetAttachedDevices()
315      if set(expected_devices) == set(devices):
316        # All devices are online, keep going.
317        break
318      if not usb_restarted and devices:
319        # The USB wasn't restarted, but there's at least one device online.
320        # No point in trying to wait for all devices.
321        break
322      retries -= 1
323
324  # TODO(navabi): Test to make sure this fails and then fix call
325  offline_devices = android_commands.GetAttachedDevices(
326      hardware=False, emulator=False, offline=True)
327
328  types, builds, batteries, reports, errors = [], [], [], [], []
329  fail_step_lst = []
330  if devices:
331    types, builds, batteries, reports, errors, fail_step_lst = (
332        zip(*[DeviceInfo(dev, options) for dev in devices]))
333
334  err_msg = CheckForMissingDevices(options, devices) or []
335
336  unique_types = list(set(types))
337  unique_builds = list(set(builds))
338
339  bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s'
340                           % (len(devices), unique_types, unique_builds))
341  print '\n'.join(reports)
342
343  for serial, dev_errors in zip(devices, errors):
344    if dev_errors:
345      err_msg += ['%s errors:' % serial]
346      err_msg += ['    %s' % error for error in dev_errors]
347
348  if err_msg:
349    bb_annotations.PrintWarning()
350    msg = '\n'.join(err_msg)
351    print msg
352    from_address = 'buildbot@chromium.org'
353    to_addresses = ['chromium-android-device-alerts@google.com']
354    bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
355    slave_name = os.environ.get('BUILDBOT_SLAVENAME')
356    subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name)
357    SendEmail(from_address, to_addresses, [], subject, msg)
358
359  if options.device_status_dashboard:
360    perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices',
361                                              [len(devices)], 'devices')
362    perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices',
363                                              [len(offline_devices)], 'devices',
364                                              'unimportant')
365    for serial, battery in zip(devices, batteries):
366      perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial,
367                                                [battery], '%',
368                                                'unimportant')
369
370  if options.json_output:
371    with open(options.json_output, 'wb') as f:
372      f.write(json.dumps({
373        'online_devices': devices,
374        'offline_devices': offline_devices,
375        'expected_devices': expected_devices,
376        'unique_types': unique_types,
377        'unique_builds': unique_builds,
378      }))
379
380  if False in fail_step_lst:
381    # TODO(navabi): Build fails on device status check step if there exists any
382    # devices with critically low battery. Remove those devices from testing,
383    # allowing build to continue with good devices.
384    return 2
385
386  if not devices:
387    return 1
388
389
390if __name__ == '__main__':
391  sys.exit(main())
392