1b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang# Copyright 2017 The Chromium Authors. All rights reserved.
2b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang# Use of this source code is governed by a BSD-style license that can be
3b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang# found in the LICENSE file.
4b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
5b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
6b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wangimport logging
7b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
8b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wangfrom devil import base_error
9b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wangfrom devil.android import device_errors
10b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
11b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wanglogger = logging.getLogger(__name__)
12b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
13b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
14b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wangdef RetryOnSystemCrash(f, device, retries=3):
15b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  """Retries the given function on a device crash.
16b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
17b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  If the provided function fails with a DeviceUnreachableError, this will wait
18b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  for the device to come back online, then retry the function.
19b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
20b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  Note that this uses the same retry scheme as timeout_retry.Run.
21b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang
22b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  Args:
23b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    f: a unary callable that takes an instance of device_utils.DeviceUtils.
24b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    device: an instance of device_utils.DeviceUtils.
25b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    retries: the number of retries.
26b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  Returns:
27b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    Whatever f returns.
28b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  """
29b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  num_try = 1
30b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang  while True:
31b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    try:
32b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang      return f(device)
33b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    except device_errors.DeviceUnreachableError:
34b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang      if num_try > retries:
35b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang        logger.error('%d consecutive device crashes. No longer retrying.',
36b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang                      num_try)
37b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang        raise
38b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang      try:
39b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang        logger.warning('Device is unreachable. Waiting for recovery...')
40b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang        device.WaitUntilFullyBooted()
41b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang      except base_error.BaseError:
42b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang        logger.exception('Device never recovered. X(')
43b2cf025c7d5cebd43084f38c6c7ff9cc17da428aWei Wang    num_try += 1
44