xref: /external/chromium_org/tools/telemetry/telemetry/page/page_runner.py

# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import optparse
import os
import random
import sys
import tempfile
import time

from telemetry import decorators
from telemetry.core import browser_finder
from telemetry.core import browser_info
from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core import wpr_modes
from telemetry.core.platform.profiler import profiler_finder
from telemetry.page import page_filter
from telemetry.page import page_test
from telemetry.page.actions import navigate
from telemetry.page.actions import page_action
from telemetry.results import results_options
from telemetry.util import cloud_storage
from telemetry.util import exception_formatter
from telemetry.value import failure
from telemetry.value import skip


class _RunState(object):
  def __init__(self):
    self.browser = None

    self._append_to_existing_wpr = False
    self._last_archive_path = None
    self._first_browser = True
    self.profiler_dir = None

  def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
                           credentials_path, archive_path, finder_options):
    started_browser = not self.browser
    # Create a browser.
    if not self.browser:
      test.CustomizeBrowserOptionsForSinglePage(page, finder_options)
      possible_browser.SetReplayArchivePath(archive_path,
                                        self._append_to_existing_wpr,
                                        page_set.make_javascript_deterministic)
      possible_browser.SetCredentialsPath(credentials_path)
      self._last_archive_path = page.archive_path

      test.WillStartBrowser(possible_browser.platform)
      self.browser = possible_browser.Create()
      test.DidStartBrowser(self.browser)

      if self._first_browser:
        self._first_browser = False
        self.browser.credentials.WarnIfMissingCredentials(page_set)
        logging.info('OS: %s %s',
                     self.browser.platform.GetOSName(),
                     self.browser.platform.GetOSVersionName())
        if self.browser.supports_system_info:
          system_info = self.browser.GetSystemInfo()
          if system_info.model_name:
            logging.info('Model: %s', system_info.model_name)
          if system_info.gpu:
            for i, device in enumerate(system_info.gpu.devices):
              logging.info('GPU device %d: %s', i, device)
            if system_info.gpu.aux_attributes:
              logging.info('GPU Attributes:')
              for k, v in sorted(system_info.gpu.aux_attributes.iteritems()):
                logging.info('  %-20s: %s', k, v)
            if system_info.gpu.feature_status:
              logging.info('Feature Status:')
              for k, v in sorted(system_info.gpu.feature_status.iteritems()):
                logging.info('  %-20s: %s', k, v)
            if system_info.gpu.driver_bug_workarounds:
              logging.info('Driver Bug Workarounds:')
              for workaround in system_info.gpu.driver_bug_workarounds:
                logging.info('  %s', workaround)
          else:
            logging.info('No GPU devices')
        else:
          logging.warning('System info not supported')
    else:
      # Set up WPR path if it changed.
      if page.archive_path and self._last_archive_path != page.archive_path:
        self.browser.SetReplayArchivePath(
            page.archive_path,
            self._append_to_existing_wpr,
            page_set.make_javascript_deterministic)
        self._last_archive_path = page.archive_path

    if self.browser.supports_tab_control and test.close_tabs_before_run:
      # Create a tab if there's none.
      if len(self.browser.tabs) == 0:
        self.browser.tabs.New()

      # Ensure only one tab is open, unless the test is a multi-tab test.
      if not test.is_multi_tab_test:
        while len(self.browser.tabs) > 1:
          self.browser.tabs[-1].Close()

      # Must wait for tab to commit otherwise it can commit after the next
      # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame()
      # will cancel the next navigation because it's pending. This manifests as
      # the first navigation in a PageSet freezing indefinitly because the
      # navigation was silently cancelled when |self.browser.tabs[0]| was
      # committed. Only do this when we just started the browser, otherwise
      # there are cases where previous pages in a PageSet never complete
      # loading so we'll wait forever.
      if started_browser:
        self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete()

  def StopBrowser(self):
    if self.browser:
      self.browser.Close()
      self.browser = None

      # Restarting the state will also restart the wpr server. If we're
      # recording, we need to continue adding into the same wpr archive,
      # not overwrite it.
      self._append_to_existing_wpr = True

  def StartProfiling(self, page, finder_options):
    if not self.profiler_dir:
      self.profiler_dir = tempfile.mkdtemp()
    output_file = os.path.join(self.profiler_dir, page.file_safe_name)
    is_repeating = (finder_options.page_repeat != 1 or
                    finder_options.pageset_repeat != 1)
    if is_repeating:
      output_file = util.GetSequentialFileName(output_file)
    self.browser.platform.profiling_controller.Start(
        finder_options.profiler, output_file)

  def StopProfiling(self):
    if self.browser:
      self.browser.platform.profiling_controller.Stop()


class PageState(object):
  def __init__(self, page, tab):
    self.page = page
    self.tab = tab

    self._did_login = False

  def PreparePage(self, test=None):
    if self.page.is_file:
      self.tab.browser.SetHTTPServerDirectories(
          self.page.page_set.serving_dirs | set([self.page.serving_dir]))

    if self.page.credentials:
      if not self.tab.browser.credentials.LoginNeeded(
          self.tab, self.page.credentials):
        raise page_test.Failure('Login as ' + self.page.credentials + ' failed')
      self._did_login = True

    if test:
      if test.clear_cache_before_each_run:
        self.tab.ClearCache(force=True)

  def ImplicitPageNavigation(self, test=None):
    """Executes the implicit navigation that occurs for every page iteration.

    This function will be called once per page before any actions are executed.
    """
    if test:
      test.WillNavigateToPage(self.page, self.tab)
      test.RunNavigateSteps(self.page, self.tab)
      test.DidNavigateToPage(self.page, self.tab)
    else:
      i = navigate.NavigateAction()
      i.RunAction(self.page, self.tab, None)

  def CleanUpPage(self, test):
    test.CleanUpAfterPage(self.page, self.tab)
    if self.page.credentials and self._did_login:
      self.tab.browser.credentials.LoginNoLongerNeeded(
          self.tab, self.page.credentials)


def AddCommandLineArgs(parser):
  page_filter.PageFilter.AddCommandLineArgs(parser)
  results_options.AddResultsOptions(parser)

  # Page set options
  group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
  group.add_option('--pageset-shuffle', action='store_true',
      dest='pageset_shuffle',
      help='Shuffle the order of pages within a pageset.')
  group.add_option('--pageset-shuffle-order-file',
      dest='pageset_shuffle_order_file', default=None,
      help='Filename of an output of a previously run test on the current '
      'pageset. The tests will run in the same order again, overriding '
      'what is specified by --page-repeat and --pageset-repeat.')
  group.add_option('--page-repeat', default=1, type='int',
                   help='Number of times to repeat each individual page '
                   'before proceeding with the next page in the pageset.')
  group.add_option('--pageset-repeat', default=1, type='int',
                   help='Number of times to repeat the entire pageset.')
  group.add_option('--max-failures', default=None, type='int',
                   help='Maximum number of test failures before aborting '
                   'the run. Defaults to the number specified by the '
                   'PageTest.')
  parser.add_option_group(group)

  # WPR options
  group = optparse.OptionGroup(parser, 'Web Page Replay options')
  group.add_option('--use-live-sites',
      dest='use_live_sites', action='store_true',
      help='Run against live sites and ignore the Web Page Replay archives.')
  parser.add_option_group(group)

  parser.add_option('-d', '--also-run-disabled-tests',
                    dest='run_disabled_tests',
                    action='store_true', default=False,
                    help='Ignore @Disabled and @Enabled restrictions.')

def ProcessCommandLineArgs(parser, args):
  page_filter.PageFilter.ProcessCommandLineArgs(parser, args)

  # Page set options
  if args.pageset_shuffle_order_file and not args.pageset_shuffle:
    parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')

  if args.page_repeat < 1:
    parser.error('--page-repeat must be a positive integer.')
  if args.pageset_repeat < 1:
    parser.error('--pageset-repeat must be a positive integer.')


def _PrepareAndRunPage(test, page_set, expectations, finder_options,
                       browser_options, page, credentials_path,
                       possible_browser, results, state):
  if finder_options.use_live_sites:
    browser_options.wpr_mode = wpr_modes.WPR_OFF
  elif browser_options.wpr_mode != wpr_modes.WPR_RECORD:
    browser_options.wpr_mode = (
        wpr_modes.WPR_REPLAY
        if page.archive_path and os.path.isfile(page.archive_path)
        else wpr_modes.WPR_OFF)

  max_attempts = test.attempts
  attempt_num = 0
  while attempt_num < max_attempts:
    attempt_num += 1
    try:
      results.WillAttemptPageRun(attempt_num, max_attempts)

      if test.RestartBrowserBeforeEachPage() or page.startup_url:
        state.StopBrowser()
        # If we are restarting the browser for each page customize the per page
        # options for just the current page before starting the browser.
      state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
                                 credentials_path, page.archive_path,
                                 finder_options)
      if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)):
        logging.info('Skip test for page %s because browser is not supported.'
                     % page.url)
        return

      expectation = expectations.GetExpectationForPage(state.browser, page)

      _WaitForThermalThrottlingIfNeeded(state.browser.platform)

      if finder_options.profiler:
        state.StartProfiling(page, finder_options)

      try:
        _RunPage(test, page, state, expectation, results)
        _CheckThermalThrottling(state.browser.platform)
      except exceptions.TabCrashException as e:
        if test.is_multi_tab_test:
          logging.error('Aborting multi-tab test after tab %s crashed',
                        page.url)
          raise
        logging.warning(str(e))
        state.StopBrowser()

      if finder_options.profiler:
        state.StopProfiling()

      if (test.StopBrowserAfterPage(state.browser, page)):
        state.StopBrowser()

      return
    except exceptions.BrowserGoneException as e:
      state.StopBrowser()
      if attempt_num == max_attempts:
        logging.error('Aborting after too many retries')
        raise
      if test.is_multi_tab_test:
        logging.error('Aborting multi-tab test after browser crashed')
        raise
      logging.warning(str(e))


@decorators.Cache
def _UpdateCredentials(page_set):
  # Attempt to download the credentials file.
  if page_set.credentials_path:
    try:
      cloud_storage.GetIfChanged(
          os.path.join(page_set.base_dir, page_set.credentials_path))
    except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
            cloud_storage.CloudStorageError) as e:
      logging.warning('Cannot retrieve credential file %s due to cloud storage '
                      'error %s', page_set.credentials_path, str(e))


@decorators.Cache
def _UpdatePageSetArchivesIfChanged(page_set):
  # Scan every serving directory for .sha1 files
  # and download them from Cloud Storage. Assume all data is public.
  all_serving_dirs = page_set.serving_dirs.copy()
  # Add individual page dirs to all serving dirs.
  for page in page_set:
    if page.is_file:
      all_serving_dirs.add(page.serving_dir)
  # Scan all serving dirs.
  for serving_dir in all_serving_dirs:
    if os.path.splitdrive(serving_dir)[1] == '/':
      raise ValueError('Trying to serve root directory from HTTP server.')
    for dirpath, _, filenames in os.walk(serving_dir):
      for filename in filenames:
        path, extension = os.path.splitext(
            os.path.join(dirpath, filename))
        if extension != '.sha1':
          continue
        cloud_storage.GetIfChanged(path, page_set.bucket)


def Run(test, page_set, expectations, finder_options, results):
  """Runs a given test against a given page_set with the given options."""
  test.ValidatePageSet(page_set)

  # Create a possible_browser with the given options.
  try:
    possible_browser = browser_finder.FindBrowser(finder_options)
  except browser_finder.BrowserTypeRequiredException, e:
    sys.stderr.write(str(e) + '\n')
    sys.exit(-1)
  if not possible_browser:
    sys.stderr.write(
        'No browser found. Available browsers:\n%s\n' %
        '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
    sys.exit(-1)

  browser_options = possible_browser.finder_options.browser_options
  browser_options.browser_type = possible_browser.browser_type
  test.CustomizeBrowserOptions(browser_options)

  if (not decorators.IsEnabled(test, possible_browser) and
      not finder_options.run_disabled_tests):
    logging.warning('You are trying to run a disabled test.')
    logging.warning('Pass --also-run-disabled-tests to squelch this message.')
    return

  if possible_browser.IsRemote():
    possible_browser.RunRemote()
    sys.exit(0)

  # Reorder page set based on options.
  pages = _ShuffleAndFilterPageSet(page_set, finder_options)

  if not finder_options.use_live_sites:
    _UpdateCredentials(page_set)
    if browser_options.wpr_mode != wpr_modes.WPR_RECORD:
      _UpdatePageSetArchivesIfChanged(page_set)
      pages = _CheckArchives(page_set, pages, results)

  # Verify credentials path.
  credentials_path = None
  if page_set.credentials_path:
    credentials_path = os.path.join(os.path.dirname(page_set.file_path),
                                    page_set.credentials_path)
    if not os.path.exists(credentials_path):
      credentials_path = None

  # Set up user agent.
  browser_options.browser_user_agent_type = page_set.user_agent_type or None

  if finder_options.profiler:
    profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
    profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
                                           finder_options)

  for page in list(pages):
    if not test.CanRunForPage(page):
      results.WillRunPage(page)
      logging.debug('Skipping test: it cannot run for %s', page.url)
      results.AddValue(skip.SkipValue(page, 'Test cannot run'))
      results.DidRunPage(page)
      pages.remove(page)

  if not pages:
    return

  state = _RunState()
  pages_with_discarded_first_result = set()
  max_failures = finder_options.max_failures  # command-line gets priority
  if max_failures is None:
    max_failures = test.max_failures  # may be None

  try:
    test.WillRunTest(finder_options)
    for _ in xrange(finder_options.pageset_repeat):
      for page in pages:
        if test.IsExiting():
          break
        for _ in xrange(finder_options.page_repeat):
          results.WillRunPage(page)
          try:
            _PrepareAndRunPage(
                test, page_set, expectations, finder_options, browser_options,
                page, credentials_path, possible_browser, results, state)
          finally:
            discard_run = (test.discard_first_result and
                           page not in pages_with_discarded_first_result)
            if discard_run:
              pages_with_discarded_first_result.add(page)
            results.DidRunPage(page, discard_run=discard_run)
        if max_failures is not None and len(results.failures) > max_failures:
          logging.error('Too many failures. Aborting.')
          test.RequestExit()
  finally:
    test.DidRunTest(state.browser, results)
    state.StopBrowser()


def _ShuffleAndFilterPageSet(page_set, finder_options):
  if finder_options.pageset_shuffle_order_file:
    return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)
  pages = [page for page in page_set.pages[:]
           if not page.disabled and page_filter.PageFilter.IsSelected(page)]
  if finder_options.pageset_shuffle:
    random.shuffle(pages)
  return pages


def _CheckArchives(page_set, pages, results):
  """Returns a subset of pages that are local or have WPR archives.

  Logs warnings if any are missing.
  """
  # Warn of any problems with the entire page set.
  if any(not p.is_local for p in pages):
    if not page_set.archive_data_file:
      logging.warning('The page set is missing an "archive_data_file" '
                      'property. Skipping any live sites. To include them, '
                      'pass the flag --use-live-sites.')
    if not page_set.wpr_archive_info:
      logging.warning('The archive info file is missing. '
                      'To fix this, either add svn-internal to your '
                      '.gclient using http://goto/read-src-internal, '
                      'or create a new archive using record_wpr.')

  # Warn of any problems with individual pages and return valid pages.
  pages_missing_archive_path = []
  pages_missing_archive_data = []
  valid_pages = []
  for page in pages:
    if not page.is_local and not page.archive_path:
      pages_missing_archive_path.append(page)
    elif not page.is_local and not os.path.isfile(page.archive_path):
      pages_missing_archive_data.append(page)
    else:
      valid_pages.append(page)
  if pages_missing_archive_path:
    logging.warning('The page set archives for some pages do not exist. '
                    'Skipping those pages. To fix this, record those pages '
                    'using record_wpr. To ignore this warning and run '
                    'against live sites, pass the flag --use-live-sites.')
  if pages_missing_archive_data:
    logging.warning('The page set archives for some pages are missing. '
                    'Someone forgot to check them in, or they were deleted. '
                    'Skipping those pages. To fix this, record those pages '
                    'using record_wpr. To ignore this warning and run '
                    'against live sites, pass the flag --use-live-sites.')
  for page in pages_missing_archive_path + pages_missing_archive_data:
    results.WillRunPage(page)
    results.AddValue(failure.FailureValue.FromMessage(
        page, 'Page set archive doesn\'t exist.'))
    results.DidRunPage(page)
  return valid_pages


def _RunPage(test, page, state, expectation, results):
  if expectation == 'skip':
    logging.debug('Skipping test: Skip expectation for %s', page.url)
    results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
    return

  page_state = PageState(page, test.TabForPage(page, state.browser))

  def ProcessError():
    if expectation == 'fail':
      msg = 'Expected exception while running %s' % page.url
    else:
      msg = 'Exception while running %s' % page.url
      results.AddValue(failure.FailureValue(page, sys.exc_info()))
    exception_formatter.PrintFormattedException(msg=msg)

  try:
    page_state.PreparePage(test)
    page_state.ImplicitPageNavigation(test)
    test.RunPage(page, page_state.tab, results)
    util.CloseConnections(page_state.tab)
  except page_test.TestNotSupportedOnPlatformFailure:
    raise
  except page_test.Failure:
    if expectation == 'fail':
      exception_formatter.PrintFormattedException(
          msg='Expected failure while running %s' % page.url)
    else:
      exception_formatter.PrintFormattedException(
          msg='Failure while running %s' % page.url)
      results.AddValue(failure.FailureValue(page, sys.exc_info()))
  except (util.TimeoutException, exceptions.LoginException,
          exceptions.ProfilingException):
    ProcessError()
  except (exceptions.TabCrashException, exceptions.BrowserGoneException):
    ProcessError()
    # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
    raise
  except page_action.PageActionNotSupported as e:
    results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e))
  except Exception:
    exception_formatter.PrintFormattedException(
        msg='Unhandled exception while running %s' % page.url)
    results.AddValue(failure.FailureValue(page, sys.exc_info()))
  else:
    if expectation == 'fail':
      logging.warning('%s was expected to fail, but passed.\n', page.url)
  finally:
    page_state.CleanUpPage(test)


def _WaitForThermalThrottlingIfNeeded(platform):
  if not platform.CanMonitorThermalThrottling():
    return
  thermal_throttling_retry = 0
  while (platform.IsThermallyThrottled() and
         thermal_throttling_retry < 3):
    logging.warning('Thermally throttled, waiting (%d)...',
                    thermal_throttling_retry)
    thermal_throttling_retry += 1
    time.sleep(thermal_throttling_retry * 2)

  if thermal_throttling_retry and platform.IsThermallyThrottled():
    logging.warning('Device is thermally throttled before running '
                    'performance tests, results will vary.')


def _CheckThermalThrottling(platform):
  if not platform.CanMonitorThermalThrottling():
    return
  if platform.HasBeenThermallyThrottled():
    logging.warning('Device has been thermally throttled during '
                    'performance tests, results will vary.')