1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Provisions Android devices with settings required for bots.
8
9Usage:
10  ./provision_devices.py [-d <device serial number>]
11"""
12
13import argparse
14import datetime
15import json
16import logging
17import os
18import posixpath
19import re
20import subprocess
21import sys
22import time
23
24# Import _strptime before threaded code. datetime.datetime.strptime is
25# threadsafe except for the initial import of the _strptime module.
26# See crbug.com/584730 and https://bugs.python.org/issue7980.
27import _strptime  # pylint: disable=unused-import
28
29import devil_chromium
30from devil import devil_env
31from devil.android import battery_utils
32from devil.android import device_blacklist
33from devil.android import device_errors
34from devil.android import device_temp_file
35from devil.android import device_utils
36from devil.android.sdk import keyevent
37from devil.android.sdk import version_codes
38from devil.constants import exit_codes
39from devil.utils import run_tests_helper
40from devil.utils import timeout_retry
41from pylib import constants
42from pylib import device_settings
43from pylib.constants import host_paths
44
45_SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
46_CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
47_TOMBSTONE_REGEX = re.compile('tombstone.*')
48
49
50class _DEFAULT_TIMEOUTS(object):
51  # L can take a while to reboot after a wipe.
52  LOLLIPOP = 600
53  PRE_LOLLIPOP = 180
54
55  HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
56
57
58class _PHASES(object):
59  WIPE = 'wipe'
60  PROPERTIES = 'properties'
61  FINISH = 'finish'
62
63  ALL = [WIPE, PROPERTIES, FINISH]
64
65
66def ProvisionDevices(args):
67  blacklist = (device_blacklist.Blacklist(args.blacklist_file)
68               if args.blacklist_file
69               else None)
70  devices = [d for d in device_utils.DeviceUtils.HealthyDevices(blacklist)
71             if not args.emulators or d.adb.is_emulator]
72  if args.device:
73    devices = [d for d in devices if d == args.device]
74  if not devices:
75    raise device_errors.DeviceUnreachableError(args.device)
76  parallel_devices = device_utils.DeviceUtils.parallel(devices)
77  if args.emulators:
78    parallel_devices.pMap(SetProperties, args)
79  else:
80    parallel_devices.pMap(ProvisionDevice, blacklist, args)
81  if args.auto_reconnect:
82    _LaunchHostHeartbeat()
83  blacklisted_devices = blacklist.Read() if blacklist else []
84  if args.output_device_blacklist:
85    with open(args.output_device_blacklist, 'w') as f:
86      json.dump(blacklisted_devices, f)
87  if all(d in blacklisted_devices for d in devices):
88    raise device_errors.NoDevicesError
89  return 0
90
91
92def ProvisionDevice(device, blacklist, options):
93  if options.reboot_timeout:
94    reboot_timeout = options.reboot_timeout
95  elif device.build_version_sdk >= version_codes.LOLLIPOP:
96    reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
97  else:
98    reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
99
100  def should_run_phase(phase_name):
101    return not options.phases or phase_name in options.phases
102
103  def run_phase(phase_func, reboot=True):
104    try:
105      device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
106    except device_errors.CommandTimeoutError:
107      logging.error('Device did not finish booting. Will try to reboot.')
108      device.Reboot(timeout=reboot_timeout)
109    phase_func(device, options)
110    if reboot:
111      device.Reboot(False, retries=0)
112      device.adb.WaitForDevice()
113
114  try:
115    if should_run_phase(_PHASES.WIPE):
116      if (options.chrome_specific_wipe or device.IsUserBuild() or
117          device.build_version_sdk >= version_codes.MARSHMALLOW):
118        run_phase(WipeChromeData)
119      else:
120        run_phase(WipeDevice)
121
122    if should_run_phase(_PHASES.PROPERTIES):
123      run_phase(SetProperties)
124
125    if should_run_phase(_PHASES.FINISH):
126      run_phase(FinishProvisioning, reboot=False)
127
128    if options.chrome_specific_wipe:
129      package = "com.google.android.gms"
130      version_name = device.GetApplicationVersion(package)
131      logging.info("Version name for %s is %s", package, version_name)
132
133    CheckExternalStorage(device)
134
135  except device_errors.CommandTimeoutError:
136    logging.exception('Timed out waiting for device %s. Adding to blacklist.',
137                      str(device))
138    if blacklist:
139      blacklist.Extend([str(device)], reason='provision_timeout')
140
141  except device_errors.CommandFailedError:
142    logging.exception('Failed to provision device %s. Adding to blacklist.',
143                      str(device))
144    if blacklist:
145      blacklist.Extend([str(device)], reason='provision_failure')
146
147def CheckExternalStorage(device):
148  """Checks that storage is writable and if not makes it writable.
149
150  Arguments:
151    device: The device to check.
152  """
153  try:
154    with device_temp_file.DeviceTempFile(
155        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
156      device.WriteFile(f.name, 'test')
157  except device_errors.CommandFailedError:
158    logging.info('External storage not writable. Remounting / as RW')
159    device.RunShellCommand(['mount', '-o', 'remount,rw', '/'],
160                           check_return=True, as_root=True)
161    device.EnableRoot()
162    with device_temp_file.DeviceTempFile(
163        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
164      device.WriteFile(f.name, 'test')
165
166def WipeChromeData(device, options):
167  """Wipes chrome specific data from device
168
169  (1) uninstall any app whose name matches *chrom*, except
170      com.android.chrome, which is the chrome stable package. Doing so also
171      removes the corresponding dirs under /data/data/ and /data/app/
172  (2) remove any dir under /data/app-lib/ whose name matches *chrom*
173  (3) remove any files under /data/tombstones/ whose name matches "tombstone*"
174  (4) remove /data/local.prop if there is any
175  (5) remove /data/local/chrome-command-line if there is any
176  (6) remove anything under /data/local/.config/ if the dir exists
177      (this is telemetry related)
178  (7) remove anything under /data/local/tmp/
179
180  Arguments:
181    device: the device to wipe
182  """
183  if options.skip_wipe:
184    return
185
186  try:
187    if device.IsUserBuild():
188      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX,
189                        constants.PACKAGE_INFO['chrome_stable'].package)
190      device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
191                             check_return=True)
192      device.RunShellCommand('rm -rf /data/local/tmp/*', check_return=True)
193    else:
194      device.EnableRoot()
195      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX,
196                        constants.PACKAGE_INFO['chrome_stable'].package)
197      _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
198      _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
199
200      _WipeFileOrDir(device, '/data/local.prop')
201      _WipeFileOrDir(device, '/data/local/chrome-command-line')
202      _WipeFileOrDir(device, '/data/local/.config/')
203      _WipeFileOrDir(device, '/data/local/tmp/')
204      device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
205                             check_return=True)
206  except device_errors.CommandFailedError:
207    logging.exception('Possible failure while wiping the device. '
208                      'Attempting to continue.')
209
210
211def WipeDevice(device, options):
212  """Wipes data from device, keeping only the adb_keys for authorization.
213
214  After wiping data on a device that has been authorized, adb can still
215  communicate with the device, but after reboot the device will need to be
216  re-authorized because the adb keys file is stored in /data/misc/adb/.
217  Thus, adb_keys file is rewritten so the device does not need to be
218  re-authorized.
219
220  Arguments:
221    device: the device to wipe
222  """
223  if options.skip_wipe:
224    return
225
226  try:
227    device.EnableRoot()
228    device_authorized = device.FileExists(constants.ADB_KEYS_FILE)
229    if device_authorized:
230      adb_keys = device.ReadFile(constants.ADB_KEYS_FILE,
231                                 as_root=True).splitlines()
232    device.RunShellCommand(['wipe', 'data'],
233                           as_root=True, check_return=True)
234    device.adb.WaitForDevice()
235
236    if device_authorized:
237      adb_keys_set = set(adb_keys)
238      for adb_key_file in options.adb_key_files or []:
239        try:
240          with open(adb_key_file, 'r') as f:
241            adb_public_keys = f.readlines()
242          adb_keys_set.update(adb_public_keys)
243        except IOError:
244          logging.warning('Unable to find adb keys file %s.', adb_key_file)
245      _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
246  except device_errors.CommandFailedError:
247    logging.exception('Possible failure while wiping the device. '
248                      'Attempting to continue.')
249
250
251def _WriteAdbKeysFile(device, adb_keys_string):
252  dir_path = posixpath.dirname(constants.ADB_KEYS_FILE)
253  device.RunShellCommand(['mkdir', '-p', dir_path],
254                         as_root=True, check_return=True)
255  device.RunShellCommand(['restorecon', dir_path],
256                         as_root=True, check_return=True)
257  device.WriteFile(constants.ADB_KEYS_FILE, adb_keys_string, as_root=True)
258  device.RunShellCommand(['restorecon', constants.ADB_KEYS_FILE],
259                         as_root=True, check_return=True)
260
261
262def SetProperties(device, options):
263  try:
264    device.EnableRoot()
265  except device_errors.CommandFailedError as e:
266    logging.warning(str(e))
267
268  if not device.IsUserBuild():
269    _ConfigureLocalProperties(device, options.enable_java_debug)
270  else:
271    logging.warning('Cannot configure properties in user builds.')
272  device_settings.ConfigureContentSettings(
273      device, device_settings.DETERMINISTIC_DEVICE_SETTINGS)
274  if options.disable_location:
275    device_settings.ConfigureContentSettings(
276        device, device_settings.DISABLE_LOCATION_SETTINGS)
277  else:
278    device_settings.ConfigureContentSettings(
279        device, device_settings.ENABLE_LOCATION_SETTINGS)
280
281  if options.disable_mock_location:
282    device_settings.ConfigureContentSettings(
283        device, device_settings.DISABLE_MOCK_LOCATION_SETTINGS)
284  else:
285    device_settings.ConfigureContentSettings(
286        device, device_settings.ENABLE_MOCK_LOCATION_SETTINGS)
287
288  device_settings.SetLockScreenSettings(device)
289  if options.disable_network:
290    device_settings.ConfigureContentSettings(
291        device, device_settings.NETWORK_DISABLED_SETTINGS)
292
293  if options.disable_system_chrome:
294    # The system chrome version on the device interferes with some tests.
295    device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
296                           check_return=True)
297
298  if options.remove_system_webview:
299    if any(device.PathExists(p) for p in _SYSTEM_WEBVIEW_PATHS):
300      logging.info('System WebView exists and needs to be removed')
301      if device.HasRoot():
302        # Disabled Marshmallow's Verity security feature
303        if device.build_version_sdk >= version_codes.MARSHMALLOW:
304          device.adb.DisableVerity()
305          device.Reboot()
306          device.WaitUntilFullyBooted()
307          device.EnableRoot()
308
309        # This is required, e.g., to replace the system webview on a device.
310        device.adb.Remount()
311        device.RunShellCommand(['stop'], check_return=True)
312        device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
313                               check_return=True)
314        device.RunShellCommand(['start'], check_return=True)
315      else:
316        logging.warning('Cannot remove system webview from a non-rooted device')
317    else:
318      logging.info('System WebView already removed')
319
320  # Some device types can momentarily disappear after setting properties.
321  device.adb.WaitForDevice()
322
323
324def _ConfigureLocalProperties(device, java_debug=True):
325  """Set standard readonly testing device properties prior to reboot."""
326  local_props = [
327      'persist.sys.usb.config=adb',
328      'ro.monkey=1',
329      'ro.test_harness=1',
330      'ro.audio.silent=1',
331      'ro.setupwizard.mode=DISABLED',
332      ]
333  if java_debug:
334    local_props.append(
335        '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
336    local_props.append('debug.checkjni=1')
337  try:
338    device.WriteFile(
339        device.LOCAL_PROPERTIES_PATH,
340        '\n'.join(local_props), as_root=True)
341    # Android will not respect the local props file if it is world writable.
342    device.RunShellCommand(
343        ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
344        as_root=True, check_return=True)
345  except device_errors.CommandFailedError:
346    logging.exception('Failed to configure local properties.')
347
348
349def FinishProvisioning(device, options):
350  # The lockscreen can't be disabled on user builds, so send a keyevent
351  # to unlock it.
352  if device.IsUserBuild():
353    device.SendKeyEvent(keyevent.KEYCODE_MENU)
354
355  if options.min_battery_level is not None:
356    battery = battery_utils.BatteryUtils(device)
357    try:
358      battery.ChargeDeviceToLevel(options.min_battery_level)
359    except device_errors.DeviceChargingError:
360      device.Reboot()
361      battery.ChargeDeviceToLevel(options.min_battery_level)
362
363  if options.max_battery_temp is not None:
364    try:
365      battery = battery_utils.BatteryUtils(device)
366      battery.LetBatteryCoolToTemperature(options.max_battery_temp)
367    except device_errors.CommandFailedError:
368      logging.exception('Unable to let battery cool to specified temperature.')
369
370  def _set_and_verify_date():
371    if device.build_version_sdk >= version_codes.MARSHMALLOW:
372      date_format = '%m%d%H%M%Y.%S'
373      set_date_command = ['date', '-u']
374      get_date_command = ['date', '-u']
375    else:
376      date_format = '%Y%m%d.%H%M%S'
377      set_date_command = ['date', '-s']
378      get_date_command = ['date']
379
380    # TODO(jbudorick): This is wrong on pre-M devices -- get/set are
381    # dealing in local time, but we're setting based on GMT.
382    strgmtime = time.strftime(date_format, time.gmtime())
383    set_date_command.append(strgmtime)
384    device.RunShellCommand(set_date_command, as_root=True, check_return=True)
385
386    get_date_command.append('+"%Y%m%d.%H%M%S"')
387    device_time = device.RunShellCommand(
388        get_date_command, as_root=True, single_line=True).replace('"', '')
389    device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
390    correct_time = datetime.datetime.strptime(strgmtime, date_format)
391    tdelta = (correct_time - device_time).seconds
392    if tdelta <= 1:
393      logging.info('Date/time successfully set on %s', device)
394      return True
395    else:
396      logging.error('Date mismatch. Device: %s Correct: %s',
397                    device_time.isoformat(), correct_time.isoformat())
398      return False
399
400  # Sometimes the date is not set correctly on the devices. Retry on failure.
401  if device.IsUserBuild():
402    # TODO(bpastene): Figure out how to set the date & time on user builds.
403    pass
404  else:
405    if not timeout_retry.WaitFor(
406        _set_and_verify_date, wait_period=1, max_tries=2):
407      raise device_errors.CommandFailedError(
408          'Failed to set date & time.', device_serial=str(device))
409
410  props = device.RunShellCommand('getprop', check_return=True)
411  for prop in props:
412    logging.info('  %s', prop)
413  if options.auto_reconnect:
414    _PushAndLaunchAdbReboot(device, options.target)
415
416
417def _UninstallIfMatch(device, pattern, app_to_keep):
418  installed_packages = device.RunShellCommand(['pm', 'list', 'packages'])
419  installed_system_packages = [
420      pkg.split(':')[1] for pkg in device.RunShellCommand(['pm', 'list',
421                                                           'packages', '-s'])]
422  for package_output in installed_packages:
423    package = package_output.split(":")[1]
424    if pattern.match(package) and not package == app_to_keep:
425      if not device.IsUserBuild() or package not in installed_system_packages:
426        device.Uninstall(package)
427
428
429def _WipeUnderDirIfMatch(device, path, pattern):
430  ls_result = device.Ls(path)
431  for (content, _) in ls_result:
432    if pattern.match(content):
433      _WipeFileOrDir(device, path + content)
434
435
436def _WipeFileOrDir(device, path):
437  if device.PathExists(path):
438    device.RunShellCommand(['rm', '-rf', path], check_return=True)
439
440
441def _PushAndLaunchAdbReboot(device, target):
442  """Pushes and launches the adb_reboot binary on the device.
443
444  Arguments:
445    device: The DeviceUtils instance for the device to which the adb_reboot
446            binary should be pushed.
447    target: The build target (example, Debug or Release) which helps in
448            locating the adb_reboot binary.
449  """
450  logging.info('Will push and launch adb_reboot on %s', str(device))
451  # Kill if adb_reboot is already running.
452  device.KillAll('adb_reboot', blocking=True, timeout=2, quiet=True)
453  # Push adb_reboot
454  logging.info('  Pushing adb_reboot ...')
455  adb_reboot = os.path.join(host_paths.DIR_SOURCE_ROOT,
456                            'out/%s/adb_reboot' % target)
457  device.PushChangedFiles([(adb_reboot, '/data/local/tmp/')])
458  # Launch adb_reboot
459  logging.info('  Launching adb_reboot ...')
460  device.RunShellCommand(
461      ['/data/local/tmp/adb_reboot'],
462      check_return=True)
463
464
465def _LaunchHostHeartbeat():
466  # Kill if existing host_heartbeat
467  KillHostHeartbeat()
468  # Launch a new host_heartbeat
469  logging.info('Spawning host heartbeat...')
470  subprocess.Popen([os.path.join(host_paths.DIR_SOURCE_ROOT,
471                                 'build/android/host_heartbeat.py')])
472
473def KillHostHeartbeat():
474  ps = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
475  stdout, _ = ps.communicate()
476  matches = re.findall('\\n.*host_heartbeat.*', stdout)
477  for match in matches:
478    logging.info('An instance of host heart beart running... will kill')
479    pid = re.findall(r'(\S+)', match)[1]
480    subprocess.call(['kill', str(pid)])
481
482def main():
483  # Recommended options on perf bots:
484  # --disable-network
485  #     TODO(tonyg): We eventually want network on. However, currently radios
486  #     can cause perfbots to drain faster than they charge.
487  # --min-battery-level 95
488  #     Some perf bots run benchmarks with USB charging disabled which leads
489  #     to gradual draining of the battery. We must wait for a full charge
490  #     before starting a run in order to keep the devices online.
491
492  parser = argparse.ArgumentParser(
493      description='Provision Android devices with settings required for bots.')
494  parser.add_argument('-d', '--device', metavar='SERIAL',
495                      help='the serial number of the device to be provisioned'
496                      ' (the default is to provision all devices attached)')
497  parser.add_argument('--adb-path',
498                      help='Absolute path to the adb binary to use.')
499  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
500  parser.add_argument('--phase', action='append', choices=_PHASES.ALL,
501                      dest='phases',
502                      help='Phases of provisioning to run. '
503                           '(If omitted, all phases will be run.)')
504  parser.add_argument('--skip-wipe', action='store_true', default=False,
505                      help="don't wipe device data during provisioning")
506  parser.add_argument('--reboot-timeout', metavar='SECS', type=int,
507                      help='when wiping the device, max number of seconds to'
508                      ' wait after each reboot '
509                      '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
510  parser.add_argument('--min-battery-level', type=int, metavar='NUM',
511                      help='wait for the device to reach this minimum battery'
512                      ' level before trying to continue')
513  parser.add_argument('--disable-location', action='store_true',
514                      help='disable Google location services on devices')
515  parser.add_argument('--disable-mock-location', action='store_true',
516                      default=False, help='Set ALLOW_MOCK_LOCATION to false')
517  parser.add_argument('--disable-network', action='store_true',
518                      help='disable network access on devices')
519  parser.add_argument('--disable-java-debug', action='store_false',
520                      dest='enable_java_debug', default=True,
521                      help='disable Java property asserts and JNI checking')
522  parser.add_argument('--disable-system-chrome', action='store_true',
523                      help='Disable the system chrome from devices.')
524  parser.add_argument('--remove-system-webview', action='store_true',
525                      help='Remove the system webview from devices.')
526  parser.add_argument('-t', '--target', default='Debug',
527                      help='the build target (default: %(default)s)')
528  parser.add_argument('-r', '--auto-reconnect', action='store_true',
529                      help='push binary which will reboot the device on adb'
530                      ' disconnections')
531  parser.add_argument('--adb-key-files', type=str, nargs='+',
532                      help='list of adb keys to push to device')
533  parser.add_argument('-v', '--verbose', action='count', default=1,
534                      help='Log more information.')
535  parser.add_argument('--max-battery-temp', type=int, metavar='NUM',
536                      help='Wait for the battery to have this temp or lower.')
537  parser.add_argument('--output-device-blacklist',
538                      help='Json file to output the device blacklist.')
539  parser.add_argument('--chrome-specific-wipe', action='store_true',
540                      help='only wipe chrome specific data during provisioning')
541  parser.add_argument('--emulators', action='store_true',
542                      help='provision only emulators and ignore usb devices')
543  args = parser.parse_args()
544  constants.SetBuildType(args.target)
545
546  run_tests_helper.SetLogLevel(args.verbose)
547
548  devil_custom_deps = None
549  if args.adb_path:
550    devil_custom_deps = {
551      'adb': {
552        devil_env.GetPlatform(): [args.adb_path],
553      },
554    }
555
556  devil_chromium.Initialize(custom_deps=devil_custom_deps)
557
558  try:
559    return ProvisionDevices(args)
560  except (device_errors.DeviceUnreachableError, device_errors.NoDevicesError):
561    return exit_codes.INFRA
562
563
564if __name__ == '__main__':
565  sys.exit(main())
566