17dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch#!/usr/bin/env python 27dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch# 37dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch# Copyright 2013 The Chromium Authors. All rights reserved. 47dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch# Use of this source code is governed by a BSD-style license that can be 57dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch# found in the LICENSE file. 67dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 77dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch"""A class to keep track of devices across builds and report state.""" 8116680a4aac90f2aa7413d9095a592090648e557Ben Murdochimport json 97dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport logging 107dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport optparse 117dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport os 121e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)import psutil 131e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)import re 141e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)import signal 157dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport smtplib 168bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)import subprocess 177dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport sys 181e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)import time 19bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdochimport urllib 207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 217dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport bb_annotations 228bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)import bb_utils 237dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)sys.path.append(os.path.join(os.path.dirname(__file__), 254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) os.pardir, os.pardir, 'util', 'lib', 264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'common')) 274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import perf_tests_results_helper # pylint: disable=F0401 284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 297dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochsys.path.append(os.path.join(os.path.dirname(__file__), '..')) 307dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochfrom pylib import android_commands 317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochfrom pylib import constants 327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochfrom pylib.cmd_helper import GetCmdOutput 33e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochfrom pylib.device import device_blacklist 34cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)from pylib.device import device_errors 3546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles)from pylib.device import device_list 36a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochfrom pylib.device import device_utils 377dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 387dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochdef DeviceInfo(serial, options): 397dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Gathers info on a device via various adb calls. 407dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 417dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 427dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch serial: The serial of the attached device to construct info about. 437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 447dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Returns: 457dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Tuple of device type, build id, report as a string, error messages, and 467dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch boolean indicating whether or not device can be used for testing. 477dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 487dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 49a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch device_adb = device_utils.DeviceUtils(serial) 50116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch device_type = device_adb.GetProp('ro.build.product') 51116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch device_build = device_adb.GetProp('ro.build.id') 52116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch device_build_type = device_adb.GetProp('ro.build.type') 53116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch device_product_name = device_adb.GetProp('ro.product.name') 54a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch 55a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) try: 56f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) battery_info = device_adb.old_interface.GetBatteryInfo() 57a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) except Exception as e: 58f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) battery_info = {} 59a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) logging.error('Unable to obtain battery info for %s, %s', serial, e) 6058e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch 6158e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch def _GetData(re_expression, line, lambda_function=lambda x:x): 6258e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch if not line: 6358e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch return 'Unknown' 6458e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch found = re.findall(re_expression, line) 6558e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch if found and len(found): 6658e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch return lambda_function(found[0]) 6758e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch return 'Unknown' 6858e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch 69f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) battery_level = int(battery_info.get('level', 100)) 7058e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch imei_slice = _GetData('Device ID = (\d+)', 71a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch device_adb.old_interface.GetSubscriberInfo(), 7258e6fbe4ee35d65e14b626c557d37565bf8ad179Ben Murdoch lambda x: x[-6:]) 737dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch report = ['Device %s (%s)' % (serial, device_type), 74a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch ' Build: %s (%s)' % 75116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch (device_build, device_adb.GetProp('ro.build.fingerprint')), 76f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) ' Current Battery Service state: ', 77f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) '\n'.join([' %s: %s' % (k, v) 78f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for k, v in battery_info.iteritems()]), 79a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch ' IMEI slice: %s' % imei_slice, 80116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ' Wifi IP: %s' % device_adb.GetProp('dhcp.wlan0.ipaddress'), 817dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch ''] 827dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 837dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch errors = [] 846e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) dev_good = True 857dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if battery_level < 15: 867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch errors += ['Device critically low in battery. Turning off device.'] 876e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) dev_good = False 8858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if not options.no_provisioning_check: 89a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch setup_wizard_disabled = ( 90116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED') 9158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if not setup_wizard_disabled and device_build_type != 'user': 9258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) errors += ['Setup wizard not disabled. Was it provisioned correctly?'] 93f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if (device_product_name == 'mantaray' and 94f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) battery_info.get('AC powered', None) != 'true'): 957dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch errors += ['Mantaray device not connected to AC power.'] 96effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 97effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Turn off devices with low battery. 987dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if battery_level < 15: 99cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) try: 100cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) device_adb.EnableRoot() 101cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) except device_errors.CommandFailedError as e: 102cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Attempt shutdown anyway. 103cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # TODO(jbudorick) Handle this exception appropriately after interface 104cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # conversions are finished. 105cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) logging.error(str(e)) 106a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch device_adb.old_interface.Shutdown() 107a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch full_report = '\n'.join(report) 1086e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) return device_type, device_build, battery_level, full_report, errors, dev_good 1097dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1107dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1117dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochdef CheckForMissingDevices(options, adb_online_devs): 1127dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Uses file of previous online devices to detect broken phones. 1137dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1147dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch Args: 1157dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch options: out_dir parameter of options argument is used as the base 1167dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch directory to load and update the cache file. 1177dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch adb_online_devs: A list of serial numbers of the currently visible 1187dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch and online attached devices. 1197dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """ 1207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # TODO(navabi): remove this once the bug that causes different number 1217dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # of devices to be detected between calls is fixed. 1227dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logger = logging.getLogger() 1237dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch logger.setLevel(logging.INFO) 1247dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1257dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch out_dir = os.path.abspath(options.out_dir) 1267dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 127f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # last_devices denotes all known devices prior to this run 12846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME) 129f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) last_missing_devices_path = os.path.join(out_dir, 130f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) device_list.LAST_MISSING_DEVICES_FILENAME) 13146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) try: 13246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) last_devices = device_list.GetPersistentDeviceList(last_devices_path) 13346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) except IOError: 13446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # Ignore error, file might not exist 13546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) last_devices = [] 136f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 137f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) try: 138f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) last_missing_devices = device_list.GetPersistentDeviceList( 139f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) last_missing_devices_path) 140f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) except IOError: 141f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) last_missing_devices = [] 142f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 1437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch missing_devs = list(set(last_devices) - set(adb_online_devs)) 144f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) new_missing_devs = list(set(missing_devs) - set(last_missing_devices)) 145f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 146116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'): 147f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) logging.info('new_missing_devs %s' % new_missing_devs) 148f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) devices_missing_msg = '%d devices not detected.' % len(missing_devs) 149f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) bb_annotations.PrintSummaryText(devices_missing_msg) 150f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 1516d86b77056ed63eb6871182f42a9fd5f07550f90Torne (Richard Coles) from_address = 'chrome-bot@chromium.org' 1521320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci to_addresses = ['chrome-labs-tech-ticket@google.com', 1531320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'chrome-android-device-alert@google.com'] 1541320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci cc_addresses = ['chrome-android-device-alert@google.com'] 155116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch subject = 'Devices offline on %s, %s, %s' % ( 156116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch os.environ.get('BUILDBOT_SLAVENAME'), 157116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch os.environ.get('BUILDBOT_BUILDERNAME'), 158116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch os.environ.get('BUILDBOT_BUILDNUMBER')) 159f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) msg = ('Please reboot the following devices:\n%s' % 160f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) '\n'.join(map(str,new_missing_devs))) 1611320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci SendEmail(from_address, to_addresses, cc_addresses, subject, msg) 1627dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1637dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch all_known_devices = list(set(adb_online_devs) | set(last_devices)) 16446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) device_list.WritePersistentDeviceList(last_devices_path, all_known_devices) 165f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs) 1667dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1677dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if not all_known_devices: 1687dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # This can happen if for some reason the .last_devices file is not 1697dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # present or if it was empty. 1707dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return ['No online devices. Have any devices been plugged in?'] 1717dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if missing_devs: 1727dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch devices_missing_msg = '%d devices not detected.' % len(missing_devs) 1737dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintSummaryText(devices_missing_msg) 1747dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 1757dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # TODO(navabi): Debug by printing both output from GetCmdOutput and 1767dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # GetAttachedDevices to compare results. 177bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary=' 178bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' % 179bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch (urllib.quote('Device Offline'), 180bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch urllib.quote('Buildbot: %s %s\n' 181bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 'Build: %s\n' 182bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '(please don\'t change any labels)' % 183bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch (os.environ.get('BUILDBOT_BUILDERNAME'), 184bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch os.environ.get('BUILDBOT_SLAVENAME'), 185bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch os.environ.get('BUILDBOT_BUILDNUMBER'))))) 1867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return ['Current online devices: %s' % adb_online_devs, 1877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch '%s are no longer visible. Were they removed?\n' % missing_devs, 188bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch 'SHERIFF:\n', 189bb1529ce867d8845a77ec7cdf3e3003ef1771a40Ben Murdoch '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link, 1907dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'Cache file: %s\n\n' % last_devices_path, 1917dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'adb devices: %s' % GetCmdOutput(['adb', 'devices']), 192116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'adb devices(GetAttachedDevices): %s' % adb_online_devs] 1937dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch else: 1947dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch new_devs = set(adb_online_devs) - set(last_devices) 1957dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if new_devs and os.path.exists(last_devices_path): 1967dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintWarning() 1977dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintSummaryText( 1987dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch '%d new devices detected' % len(new_devs)) 1997dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch print ('New devices detected %s. And now back to your ' 2007dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'regularly scheduled program.' % list(new_devs)) 2017dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 2027dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 2031320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef SendEmail(from_address, to_addresses, cc_addresses, subject, msg): 204116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch msg_body = '\r\n'.join(['From: %s' % from_address, 205116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'To: %s' % ', '.join(to_addresses), 2061320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'CC: %s' % ', '.join(cc_addresses), 2077dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 'Subject: %s' % subject, '', msg]) 2087dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch try: 2097dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch server = smtplib.SMTP('localhost') 210116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch server.sendmail(from_address, to_addresses, msg_body) 2117dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch server.quit() 2127dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch except Exception as e: 2137dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch print 'Failed to send alert email. Error: %s' % e 2147dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 2157dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 2168bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)def RestartUsb(): 2178bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if not os.path.isfile('/usr/bin/restart_usb'): 2188bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed ' 2198bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 'on host (see BUG=305769).') 2205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return False 2218bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2228bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE) 2238bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) lsusb_output, _ = lsusb_proc.communicate() 2248bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if lsusb_proc.returncode: 2258bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print ('Error: Could not get list of USB ports (i.e. lsusb).') 2268bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return lsusb_proc.returncode 2278bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2288bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0] 2298bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) for lsusb_line in lsusb_output.strip().split('\n')] 2308bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) all_restarted = True 2328bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # Walk USB devices from leaves up (i.e reverse sorted) restarting the 2338bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # connection. If a parent node (e.g. usb hub) is restarted before the 2348bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # devices connected to it, the (bus, dev) for the hub can change, making the 2358bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # output we have wrong. This way we restart the devices before the hub. 2368bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) for (bus, dev) in reversed(sorted(usb_devices)): 2378bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) # Can not restart root usb connections 2388bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if dev != '001': 2398bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev]) 2408bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if return_code: 2418bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev) 2425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) all_restarted = False 2438bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) else: 2448bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev) 2458bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return all_restarted 2478bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2488bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 2491e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)def KillAllAdb(): 2501e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) def GetAllAdb(): 2511e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for p in psutil.process_iter(): 2520f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) try: 2530f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) if 'adb' in p.name: 2540f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) yield p 255116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch except (psutil.error.NoSuchProcess, psutil.error.AccessDenied): 2560f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) pass 2571e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2581e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: 2591e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for p in GetAllAdb(): 2601e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) try: 2611e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name, 2621e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) ' '.join(p.cmdline)) 2631e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) p.send_signal(sig) 264116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch except (psutil.error.NoSuchProcess, psutil.error.AccessDenied): 2651e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) pass 2661e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) for p in GetAllAdb(): 2670f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) try: 2680f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline)) 269116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch except (psutil.error.NoSuchProcess, psutil.error.AccessDenied): 2700f1bc08d4cfcc34181b0b5cbf065c40f687bf740Torne (Richard Coles) pass 2711e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2721e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 2737dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochdef main(): 2747dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch parser = optparse.OptionParser() 2757dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch parser.add_option('', '--out-dir', 2767dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch help='Directory where the device path is stored', 277ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) 27858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) parser.add_option('--no-provisioning-check', action='store_true', 2797dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch help='Will not check if devices are provisioned properly.') 28058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) parser.add_option('--device-status-dashboard', action='store_true', 281a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch help='Output device status data for dashboard.') 2828bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) parser.add_option('--restart-usb', action='store_true', 2838bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) help='Restart USB ports before running device check.') 284116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch parser.add_option('--json-output', 285116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch help='Output JSON information into a specified file.') 286116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 2877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch options, args = parser.parse_args() 2887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if args: 2897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch parser.error('Unknown options %s' % args) 2908bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 291e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch # Remove the last build's "bad devices" before checking device statuses. 292e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch device_blacklist.ResetBlacklist() 293a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 294116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch try: 295116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch expected_devices = device_list.GetPersistentDeviceList( 296116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)) 297116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch except IOError: 298116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch expected_devices = [] 299116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch devices = android_commands.GetAttachedDevices() 300116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # Only restart usb if devices are missing. 301116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if set(expected_devices) != set(devices): 302116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch print 'expected_devices: %s, devices: %s' % (expected_devices, devices) 303116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch KillAllAdb() 304116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch retries = 5 305116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch usb_restarted = True 306116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if options.restart_usb: 3075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not RestartUsb(): 3085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) usb_restarted = False 3095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bb_annotations.PrintWarning() 3105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) print 'USB reset stage failed, wait for any device to come back.' 311116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch while retries: 312116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch print 'retry adb devices...' 313116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch time.sleep(1) 314116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch devices = android_commands.GetAttachedDevices() 315116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if set(expected_devices) == set(devices): 316116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # All devices are online, keep going. 317116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch break 318116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if not usb_restarted and devices: 319116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # The USB wasn't restarted, but there's at least one device online. 320116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # No point in trying to wait for all devices. 321116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch break 322116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch retries -= 1 3238bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 324a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch # TODO(navabi): Test to make sure this fails and then fix call 325a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch offline_devices = android_commands.GetAttachedDevices( 326a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch hardware=False, emulator=False, offline=True) 327a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch 328a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch types, builds, batteries, reports, errors = [], [], [], [], [] 3297dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch fail_step_lst = [] 3307dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if devices: 331a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch types, builds, batteries, reports, errors, fail_step_lst = ( 3327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch zip(*[DeviceInfo(dev, options) for dev in devices])) 3337dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3347dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch err_msg = CheckForMissingDevices(options, devices) or [] 3357dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3367dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch unique_types = list(set(types)) 3377dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch unique_builds = list(set(builds)) 3387dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3397dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' 3407dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch % (len(devices), unique_types, unique_builds)) 3417dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch print '\n'.join(reports) 3427dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch for serial, dev_errors in zip(devices, errors): 3447dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if dev_errors: 3457dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch err_msg += ['%s errors:' % serial] 3467dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch err_msg += [' %s' % error for error in dev_errors] 3477dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3487dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if err_msg: 3497dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintWarning() 3507dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch msg = '\n'.join(err_msg) 3517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch print msg 352f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) from_address = 'buildbot@chromium.org' 353116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch to_addresses = ['chromium-android-device-alerts@google.com'] 354f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) bot_name = os.environ.get('BUILDBOT_BUILDERNAME') 355f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) slave_name = os.environ.get('BUILDBOT_SLAVENAME') 356f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) 3571320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci SendEmail(from_address, to_addresses, [], subject, msg) 3587dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 359a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch if options.device_status_dashboard: 3604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices', 3614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) [len(devices)], 'devices') 3624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices', 3634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) [len(offline_devices)], 'devices', 3644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'unimportant') 365a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch for serial, battery in zip(devices, batteries): 3664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial, 3674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) [battery], '%', 3684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'unimportant') 369a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch 370116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if options.json_output: 371116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch with open(options.json_output, 'wb') as f: 372116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch f.write(json.dumps({ 373116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'online_devices': devices, 374116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'offline_devices': offline_devices, 375116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'expected_devices': expected_devices, 376116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'unique_types': unique_types, 377116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'unique_builds': unique_builds, 378116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch })) 379116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 3807dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if False in fail_step_lst: 3817dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch # TODO(navabi): Build fails on device status check step if there exists any 382effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # devices with critically low battery. Remove those devices from testing, 383effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # allowing build to continue with good devices. 384f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return 2 3857dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3867dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if not devices: 3877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch return 1 3887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 3907dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochif __name__ == '__main__': 3917dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch sys.exit(main()) 392