xref: /external/chromium_org/tools/telemetry/telemetry/page/page_runner.py

# Copyright 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import collections
import copy
import logging
import optparse
import os
import random
import sys
import tempfile
import time

from telemetry import decorators
from telemetry.core import browser_finder
from telemetry.core import browser_info
from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core import wpr_modes
from telemetry.core.platform.profiler import profiler_finder
from telemetry.page import page_filter
from telemetry.page import page_runner_repeat
from telemetry.page import page_test
from telemetry.page.actions import navigate
from telemetry.page.actions import page_action
from telemetry.results import results_options
from telemetry.util import cloud_storage
from telemetry.util import exception_formatter


class _RunState(object):
  def __init__(self):
    self.browser = None

    self._append_to_existing_wpr = False
    self._last_archive_path = None
    self._first_browser = True
    self.first_page = collections.defaultdict(lambda: True)
    self.profiler_dir = None
    self.repeat_state = None

  def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
                           credentials_path, archive_path, finder_options):
    started_browser = not self.browser
    # Create a browser.
    if not self.browser:
      test.CustomizeBrowserOptionsForSinglePage(page, finder_options)
      self.browser = possible_browser.Create()
      self.browser.credentials.credentials_path = credentials_path

      # Set up WPR path on the new browser.
      self.browser.SetReplayArchivePath(archive_path,
                                        self._append_to_existing_wpr,
                                        page_set.make_javascript_deterministic)
      self._last_archive_path = page.archive_path

      test.WillStartBrowser(self.browser)
      self.browser.Start()
      test.DidStartBrowser(self.browser)

      if self._first_browser:
        self._first_browser = False
        self.browser.credentials.WarnIfMissingCredentials(page_set)
        logging.info('OS: %s %s',
                     self.browser.platform.GetOSName(),
                     self.browser.platform.GetOSVersionName())
        if self.browser.supports_system_info:
          system_info = self.browser.GetSystemInfo()
          if system_info.model_name:
            logging.info('Model: %s', system_info.model_name)
          if system_info.gpu:
            for i, device in enumerate(system_info.gpu.devices):
              logging.info('GPU device %d: %s', i, device)
            if system_info.gpu.aux_attributes:
              logging.info('GPU Attributes:')
              for k, v in sorted(system_info.gpu.aux_attributes.iteritems()):
                logging.info('  %-20s: %s', k, v)
            if system_info.gpu.feature_status:
              logging.info('Feature Status:')
              for k, v in sorted(system_info.gpu.feature_status.iteritems()):
                logging.info('  %-20s: %s', k, v)
            if system_info.gpu.driver_bug_workarounds:
              logging.info('Driver Bug Workarounds:')
              for workaround in system_info.gpu.driver_bug_workarounds:
                logging.info('  %s', workaround)
          else:
            logging.info('No GPU devices')
    else:
      # Set up WPR path if it changed.
      if page.archive_path and self._last_archive_path != page.archive_path:
        self.browser.SetReplayArchivePath(
            page.archive_path,
            self._append_to_existing_wpr,
            page_set.make_javascript_deterministic)
        self._last_archive_path = page.archive_path

    if self.browser.supports_tab_control and test.close_tabs_before_run:
      # Create a tab if there's none.
      if len(self.browser.tabs) == 0:
        self.browser.tabs.New()

      # Ensure only one tab is open, unless the test is a multi-tab test.
      if not test.is_multi_tab_test:
        while len(self.browser.tabs) > 1:
          self.browser.tabs[-1].Close()

      # Must wait for tab to commit otherwise it can commit after the next
      # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame()
      # will cancel the next navigation because it's pending. This manifests as
      # the first navigation in a PageSet freezing indefinitly because the
      # navigation was silently cancelled when |self.browser.tabs[0]| was
      # committed. Only do this when we just started the browser, otherwise
      # there are cases where previous pages in a PageSet never complete
      # loading so we'll wait forever.
      if started_browser:
        self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete()

  def StopBrowser(self):
    if self.browser:
      self.browser.Close()
      self.browser = None

      # Restarting the state will also restart the wpr server. If we're
      # recording, we need to continue adding into the same wpr archive,
      # not overwrite it.
      self._append_to_existing_wpr = True

  def StartProfiling(self, page, finder_options):
    if not self.profiler_dir:
      self.profiler_dir = tempfile.mkdtemp()
    output_file = os.path.join(self.profiler_dir, page.file_safe_name)
    is_repeating = (finder_options.page_repeat != 1 or
                    finder_options.pageset_repeat != 1)
    if is_repeating:
      output_file = util.GetSequentialFileName(output_file)
    self.browser.StartProfiling(finder_options.profiler, output_file)

  def StopProfiling(self):
    if self.browser:
      self.browser.StopProfiling()


class PageState(object):
  def __init__(self, page, tab):
    self.page = page
    self.tab = tab

    self._did_login = False

  def PreparePage(self, test=None):
    if self.page.is_file:
      server_started = self.tab.browser.SetHTTPServerDirectories(
        self.page.page_set.serving_dirs | set([self.page.serving_dir]))
      if server_started and test:
        test.DidStartHTTPServer(self.tab)

    if self.page.credentials:
      if not self.tab.browser.credentials.LoginNeeded(
          self.tab, self.page.credentials):
        raise page_test.Failure('Login as ' + self.page.credentials + ' failed')
      self._did_login = True

    if test:
      if test.clear_cache_before_each_run:
        self.tab.ClearCache(force=True)

  def ImplicitPageNavigation(self, test=None):
    """Executes the implicit navigation that occurs for every page iteration.

    This function will be called once per page before any actions are executed.
    """
    if test:
      test.WillNavigateToPage(self.page, self.tab)
      test.RunNavigateSteps(self.page, self.tab)
      test.DidNavigateToPage(self.page, self.tab)
    else:
      i = navigate.NavigateAction()
      i.RunAction(self.page, self.tab, None)

  def CleanUpPage(self, test):
    test.CleanUpAfterPage(self.page, self.tab)
    if self.page.credentials and self._did_login:
      self.tab.browser.credentials.LoginNoLongerNeeded(
          self.tab, self.page.credentials)


def AddCommandLineArgs(parser):
  page_filter.PageFilter.AddCommandLineArgs(parser)
  results_options.AddResultsOptions(parser)

  # Page set options
  group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
  group.add_option('--pageset-shuffle', action='store_true',
      dest='pageset_shuffle',
      help='Shuffle the order of pages within a pageset.')
  group.add_option('--pageset-shuffle-order-file',
      dest='pageset_shuffle_order_file', default=None,
      help='Filename of an output of a previously run test on the current '
      'pageset. The tests will run in the same order again, overriding '
      'what is specified by --page-repeat and --pageset-repeat.')
  group.add_option('--page-repeat', default=1, type='int',
                   help='Number of times to repeat each individual page '
                   'before proceeding with the next page in the pageset.')
  group.add_option('--pageset-repeat', default=1, type='int',
                   help='Number of times to repeat the entire pageset.')
  parser.add_option_group(group)

  # WPR options
  group = optparse.OptionGroup(parser, 'Web Page Replay options')
  group.add_option('--use-live-sites',
      dest='use_live_sites', action='store_true',
      help='Run against live sites and ignore the Web Page Replay archives.')
  parser.add_option_group(group)

  parser.add_option('-d', '--also-run-disabled-tests',
                    dest='run_disabled_tests',
                    action='store_true', default=False,
                    help='Ignore @Disabled and @Enabled restrictions.')

def ProcessCommandLineArgs(parser, args):
  page_filter.PageFilter.ProcessCommandLineArgs(parser, args)

  # Page set options
  if args.pageset_shuffle_order_file and not args.pageset_shuffle:
    parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')

  if args.page_repeat < 1:
    parser.error('--page-repeat must be a positive integer.')
  if args.pageset_repeat < 1:
    parser.error('--pageset-repeat must be a positive integer.')


def _PrepareAndRunPage(test, page_set, expectations, finder_options,
                       browser_options, page, credentials_path,
                       possible_browser, results, state):
  if finder_options.use_live_sites:
    browser_options.wpr_mode = wpr_modes.WPR_OFF
  elif browser_options.wpr_mode != wpr_modes.WPR_RECORD:
    browser_options.wpr_mode = (
        wpr_modes.WPR_REPLAY
        if page.archive_path and os.path.isfile(page.archive_path)
        else wpr_modes.WPR_OFF)

  tries = test.attempts
  while tries:
    tries -= 1
    try:
      results_for_current_run = copy.copy(results)
      if test.RestartBrowserBeforeEachPage() or page.startup_url:
        state.StopBrowser()
        # If we are restarting the browser for each page customize the per page
        # options for just the current page before starting the browser.
      state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
                                 credentials_path, page.archive_path,
                                 finder_options)
      if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)):
        logging.info('Skip test for page %s because browser is not supported.'
                     % page.url)
        return results

      expectation = expectations.GetExpectationForPage(state.browser, page)

      _WaitForThermalThrottlingIfNeeded(state.browser.platform)

      if finder_options.profiler:
        state.StartProfiling(page, finder_options)

      try:
        _RunPage(test, page, state, expectation,
                 results_for_current_run, finder_options)
        _CheckThermalThrottling(state.browser.platform)
      except exceptions.TabCrashException as e:
        if test.is_multi_tab_test:
          logging.error('Aborting multi-tab test after tab %s crashed',
                        page.url)
          raise
        logging.warning(str(e))
        state.StopBrowser()

      if finder_options.profiler:
        state.StopProfiling()

      if (test.StopBrowserAfterPage(state.browser, page)):
        state.StopBrowser()

      if state.first_page[page]:
        state.first_page[page] = False
        if test.discard_first_result:
          return results
      return results_for_current_run
    except exceptions.BrowserGoneException as e:
      state.StopBrowser()
      if not tries:
        logging.error('Aborting after too many retries')
        raise
      if test.is_multi_tab_test:
        logging.error('Aborting multi-tab test after browser crashed')
        raise
      logging.warning(str(e))


def _UpdatePageSetArchivesIfChanged(page_set):
  # Attempt to download the credentials file.
  if page_set.credentials_path:
    try:
      cloud_storage.GetIfChanged(
          os.path.join(page_set.base_dir, page_set.credentials_path))
    except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
            cloud_storage.CloudStorageError) as e:
      logging.warning('Cannot retrieve credential file %s due to cloud storage '
                      'error %s', page_set.credentials_path, str(e))

  # Scan every serving directory for .sha1 files
  # and download them from Cloud Storage. Assume all data is public.
  all_serving_dirs = page_set.serving_dirs.copy()
  # Add individual page dirs to all serving dirs.
  for page in page_set:
    if page.is_file:
      all_serving_dirs.add(page.serving_dir)
  # Scan all serving dirs.
  for serving_dir in all_serving_dirs:
    if os.path.splitdrive(serving_dir)[1] == '/':
      raise ValueError('Trying to serve root directory from HTTP server.')
    for dirpath, _, filenames in os.walk(serving_dir):
      for filename in filenames:
        path, extension = os.path.splitext(
            os.path.join(dirpath, filename))
        if extension != '.sha1':
          continue
        cloud_storage.GetIfChanged(path, page_set.bucket)


def Run(test, page_set, expectations, finder_options):
  """Runs a given test against a given page_set with the given options."""
  results = results_options.PrepareResults(test, finder_options)

  test.ValidatePageSet(page_set)

  # Create a possible_browser with the given options.
  try:
    possible_browser = browser_finder.FindBrowser(finder_options)
  except browser_finder.BrowserTypeRequiredException, e:
    sys.stderr.write(str(e) + '\n')
    sys.exit(-1)
  if not possible_browser:
    sys.stderr.write(
        'No browser found. Available browsers:\n' +
        '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)) +
        '\n')
    sys.exit(-1)

  browser_options = possible_browser.finder_options.browser_options
  browser_options.browser_type = possible_browser.browser_type
  test.CustomizeBrowserOptions(browser_options)

  should_run = decorators.IsEnabled(test, possible_browser)

  should_run = should_run or finder_options.run_disabled_tests

  if not should_run:
    logging.warning('You are trying to run a disabled test.')
    logging.warning('Pass --also-run-disabled-tests to squelch this message.')
    return results

  # Reorder page set based on options.
  pages = _ShuffleAndFilterPageSet(page_set, finder_options)

  if (not finder_options.use_live_sites and
      browser_options.wpr_mode != wpr_modes.WPR_RECORD):
    _UpdatePageSetArchivesIfChanged(page_set)
    pages = _CheckArchives(page_set, pages, results)

  # Verify credentials path.
  credentials_path = None
  if page_set.credentials_path:
    credentials_path = os.path.join(os.path.dirname(page_set.file_path),
                                    page_set.credentials_path)
    if not os.path.exists(credentials_path):
      credentials_path = None

  # Set up user agent.
  browser_options.browser_user_agent_type = page_set.user_agent_type or None

  if finder_options.profiler:
    profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
    profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
                                           finder_options)

  for page in list(pages):
    if not test.CanRunForPage(page):
      results.StartTest(page)
      logging.debug('Skipping test: it cannot run for %s', page.url)
      results.AddSkip(page, 'Test cannot run')
      results.StopTest(page)
      pages.remove(page)

  if not pages:
    return results

  state = _RunState()
  # TODO(dtu): Move results creation and results_for_current_run into RunState.

  try:
    test.WillRunTest(finder_options)
    state.repeat_state = page_runner_repeat.PageRunnerRepeatState(
        finder_options)

    state.repeat_state.WillRunPageSet()
    while state.repeat_state.ShouldRepeatPageSet() and not test.IsExiting():
      for page in pages:
        state.repeat_state.WillRunPage()
        test.WillRunPageRepeats(page)
        while state.repeat_state.ShouldRepeatPage():
          results.StartTest(page)
          try:
            results = _PrepareAndRunPage(
                test, page_set, expectations, finder_options, browser_options,
                page, credentials_path, possible_browser, results, state)
          finally:
            state.repeat_state.DidRunPage()
            results.StopTest(page)
        test.DidRunPageRepeats(page)
        if (not test.max_failures is None and
            len(results.failures) > test.max_failures):
          logging.error('Too many failures. Aborting.')
          test.RequestExit()
        if test.IsExiting():
          break
      state.repeat_state.DidRunPageSet()

    test.DidRunTest(state.browser, results)
  finally:
    state.StopBrowser()

  return results


def _ShuffleAndFilterPageSet(page_set, finder_options):
  if finder_options.pageset_shuffle_order_file:
    return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)

  pages = [page for page in page_set.pages[:]
           if not page.disabled and page_filter.PageFilter.IsSelected(page)]

  if finder_options.pageset_shuffle:
    random.Random().shuffle(pages)

  return pages


def _CheckArchives(page_set, pages, results):
  """Returns a subset of pages that are local or have WPR archives.

  Logs warnings if any are missing."""
  page_set_has_live_sites = False
  for page in pages:
    if not page.is_local:
      page_set_has_live_sites = True
      break

  # Potential problems with the entire page set.
  if page_set_has_live_sites:
    if not page_set.archive_data_file:
      logging.warning('The page set is missing an "archive_data_file" '
                      'property. Skipping any live sites. To include them, '
                      'pass the flag --use-live-sites.')
    if not page_set.wpr_archive_info:
      logging.warning('The archive info file is missing. '
                      'To fix this, either add svn-internal to your '
                      '.gclient using http://goto/read-src-internal, '
                      'or create a new archive using record_wpr.')

  # Potential problems with individual pages.
  pages_missing_archive_path = []
  pages_missing_archive_data = []

  for page in pages:
    if page.is_local:
      continue

    if not page.archive_path:
      pages_missing_archive_path.append(page)
    elif not os.path.isfile(page.archive_path):
      pages_missing_archive_data.append(page)

  if pages_missing_archive_path:
    logging.warning('The page set archives for some pages do not exist. '
                    'Skipping those pages. To fix this, record those pages '
                    'using record_wpr. To ignore this warning and run '
                    'against live sites, pass the flag --use-live-sites.')
  if pages_missing_archive_data:
    logging.warning('The page set archives for some pages are missing. '
                    'Someone forgot to check them in, or they were deleted. '
                    'Skipping those pages. To fix this, record those pages '
                    'using record_wpr. To ignore this warning and run '
                    'against live sites, pass the flag --use-live-sites.')

  for page in pages_missing_archive_path + pages_missing_archive_data:
    results.StartTest(page)
    results.AddFailureMessage(page, 'Page set archive doesn\'t exist.')
    results.StopTest(page)

  return [page for page in pages if page not in
          pages_missing_archive_path + pages_missing_archive_data]


def _RunPage(test, page, state, expectation, results, finder_options):
  if expectation == 'skip':
    logging.debug('Skipping test: Skip expectation for %s', page.url)
    results.AddSkip(page, 'Skipped by test expectations')
    return

  logging.info('Running %s', page.url)

  page_state = PageState(page, test.TabForPage(page, state.browser))

  def ProcessError():
    if expectation == 'fail':
      msg = 'Expected exception while running %s' % page.url
      results.AddSuccess(page)
    else:
      msg = 'Exception while running %s' % page.url
      results.AddFailure(page, sys.exc_info())
    exception_formatter.PrintFormattedException(msg=msg)

  try:
    page_state.PreparePage(test)
    if state.repeat_state.ShouldNavigate(
        finder_options.skip_navigate_on_repeat):
      page_state.ImplicitPageNavigation(test)
    test.RunPage(page, page_state.tab, results)
    util.CloseConnections(page_state.tab)
  except page_test.TestNotSupportedOnPlatformFailure:
    raise
  except page_test.Failure:
    if expectation == 'fail':
      exception_formatter.PrintFormattedException(
          msg='Expected failure while running %s' % page.url)
      results.AddSuccess(page)
    else:
      exception_formatter.PrintFormattedException(
          msg='Failure while running %s' % page.url)
      results.AddFailure(page, sys.exc_info())
  except (util.TimeoutException, exceptions.LoginException,
          exceptions.ProfilingException):
    ProcessError()
  except (exceptions.TabCrashException, exceptions.BrowserGoneException):
    ProcessError()
    # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
    raise
  except page_action.PageActionNotSupported as e:
    results.AddSkip(page, 'Unsupported page action: %s' % e)
  except Exception:
    exception_formatter.PrintFormattedException(
        msg='Unhandled exception while running %s' % page.url)
    results.AddFailure(page, sys.exc_info())
  else:
    if expectation == 'fail':
      logging.warning('%s was expected to fail, but passed.\n', page.url)
    results.AddSuccess(page)
  finally:
    page_state.CleanUpPage(test)


def _WaitForThermalThrottlingIfNeeded(platform):
  if not platform.CanMonitorThermalThrottling():
    return
  thermal_throttling_retry = 0
  while (platform.IsThermallyThrottled() and
         thermal_throttling_retry < 3):
    logging.warning('Thermally throttled, waiting (%d)...',
                    thermal_throttling_retry)
    thermal_throttling_retry += 1
    time.sleep(thermal_throttling_retry * 2)

  if thermal_throttling_retry and platform.IsThermallyThrottled():
    logging.warning('Device is thermally throttled before running '
                    'performance tests, results will vary.')


def _CheckThermalThrottling(platform):
  if not platform.CanMonitorThermalThrottling():
    return
  if platform.HasBeenThermallyThrottled():
    logging.warning('Device has been thermally throttled during '
                    'performance tests, results will vary.')