15f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved.
22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be
32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file.
45d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)
52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import logging
6a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochimport optparse
72a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import os
85d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import random
990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)import sys
10ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochimport tempfile
112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import time
122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from telemetry import decorators
147d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)from telemetry.core import browser_finder
15cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)from telemetry.core import browser_info
167d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)from telemetry.core import exceptions
172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.core import util
182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.core import wpr_modes
19a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)from telemetry.core.platform.profiler import profiler_finder
20a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from telemetry.page import page_filter
212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.page import page_test
223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)from telemetry.page.actions import navigate
235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from telemetry.page.actions import page_action
24f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)from telemetry.results import results_options
25116680a4aac90f2aa7413d9095a592090648e557Ben Murdochfrom telemetry.util import cloud_storage
26effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochfrom telemetry.util import exception_formatter
275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from telemetry.value import failure
285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from telemetry.value import skip
292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
306e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)
312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class _RunState(object):
322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def __init__(self):
332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self.browser = None
342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
357d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    self._append_to_existing_wpr = False
367d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    self._last_archive_path = None
377d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    self._first_browser = True
38ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch    self.profiler_dir = None
397d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)  def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
41cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                           credentials_path, archive_path, finder_options):
428bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)    started_browser = not self.browser
437d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    # Create a browser.
447d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    if not self.browser:
45cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      test.CustomizeBrowserOptionsForSinglePage(page, finder_options)
461320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      possible_browser.SetReplayArchivePath(archive_path,
47f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)                                        self._append_to_existing_wpr,
48f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)                                        page_set.make_javascript_deterministic)
491320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      possible_browser.SetCredentialsPath(credentials_path)
50f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)      self._last_archive_path = page.archive_path
51f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
521320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      test.WillStartBrowser(possible_browser.platform)
531320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      self.browser = possible_browser.Create()
54a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)      test.DidStartBrowser(self.browser)
557d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
567d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      if self._first_browser:
577d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)        self._first_browser = False
587d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)        self.browser.credentials.WarnIfMissingCredentials(page_set)
59a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)        logging.info('OS: %s %s',
60a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)                     self.browser.platform.GetOSName(),
61a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)                     self.browser.platform.GetOSVersionName())
6268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)        if self.browser.supports_system_info:
6368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)          system_info = self.browser.GetSystemInfo()
6468043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)          if system_info.model_name:
65cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)            logging.info('Model: %s', system_info.model_name)
6668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)          if system_info.gpu:
6768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)            for i, device in enumerate(system_info.gpu.devices):
684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)              logging.info('GPU device %d: %s', i, device)
694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)            if system_info.gpu.aux_attributes:
704e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)              logging.info('GPU Attributes:')
714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)              for k, v in sorted(system_info.gpu.aux_attributes.iteritems()):
724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)                logging.info('  %-20s: %s', k, v)
734e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)            if system_info.gpu.feature_status:
744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)              logging.info('Feature Status:')
754e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)              for k, v in sorted(system_info.gpu.feature_status.iteritems()):
764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)                logging.info('  %-20s: %s', k, v)
775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)            if system_info.gpu.driver_bug_workarounds:
785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)              logging.info('Driver Bug Workarounds:')
795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)              for workaround in system_info.gpu.driver_bug_workarounds:
805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)                logging.info('  %s', workaround)
8168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)          else:
8268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)            logging.info('No GPU devices')
8303b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)        else:
8403b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)          logging.warning('System info not supported')
857d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    else:
867d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      # Set up WPR path if it changed.
877dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch      if page.archive_path and self._last_archive_path != page.archive_path:
887dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch        self.browser.SetReplayArchivePath(
897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch            page.archive_path,
907dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch            self._append_to_existing_wpr,
917dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch            page_set.make_javascript_deterministic)
927d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)        self._last_archive_path = page.archive_path
937d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
941e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    if self.browser.supports_tab_control and test.close_tabs_before_run:
957d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      # Create a tab if there's none.
967d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      if len(self.browser.tabs) == 0:
977d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)        self.browser.tabs.New()
987d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
998bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      # Ensure only one tab is open, unless the test is a multi-tab test.
1008bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      if not test.is_multi_tab_test:
1018bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)        while len(self.browser.tabs) > 1:
1028bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)          self.browser.tabs[-1].Close()
1037d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
1044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      # Must wait for tab to commit otherwise it can commit after the next
105f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)      # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame()
1064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      # will cancel the next navigation because it's pending. This manifests as
1074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      # the first navigation in a PageSet freezing indefinitly because the
1084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)      # navigation was silently cancelled when |self.browser.tabs[0]| was
1098bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      # committed. Only do this when we just started the browser, otherwise
1108bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      # there are cases where previous pages in a PageSet never complete
1118bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      # loading so we'll wait forever.
1128bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      if started_browser:
1138bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)        self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete()
1144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)
1157d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  def StopBrowser(self):
1162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    if self.browser:
1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      self.browser.Close()
1182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      self.browser = None
1192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1207d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      # Restarting the state will also restart the wpr server. If we're
1217d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      # recording, we need to continue adding into the same wpr archive,
1227d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      # not overwrite it.
1237d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      self._append_to_existing_wpr = True
1242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
125424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  def StartProfiling(self, page, finder_options):
126ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch    if not self.profiler_dir:
127ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch      self.profiler_dir = tempfile.mkdtemp()
12858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)    output_file = os.path.join(self.profiler_dir, page.file_safe_name)
129a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch    is_repeating = (finder_options.page_repeat != 1 or
130a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch                    finder_options.pageset_repeat != 1)
131a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch    if is_repeating:
132cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      output_file = util.GetSequentialFileName(output_file)
1331320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    self.browser.platform.profiling_controller.Start(
1341320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci        finder_options.profiler, output_file)
1352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1367d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  def StopProfiling(self):
1374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    if self.browser:
1381320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      self.browser.platform.profiling_controller.Stop()
1392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
14090dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)
1417d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)class PageState(object):
1421e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)  def __init__(self, page, tab):
1431e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    self.page = page
1441e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    self.tab = tab
1451e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)
1467d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    self._did_login = False
1472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1481e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)  def PreparePage(self, test=None):
1491e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    if self.page.is_file:
1501320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      self.tab.browser.SetHTTPServerDirectories(
1511320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci          self.page.page_set.serving_dirs | set([self.page.serving_dir]))
1522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1531e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    if self.page.credentials:
1541e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)      if not self.tab.browser.credentials.LoginNeeded(
1551e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)          self.tab, self.page.credentials):
1561e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)        raise page_test.Failure('Login as ' + self.page.credentials + ' failed')
1577d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      self._did_login = True
1582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1597d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    if test:
160eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch      if test.clear_cache_before_each_run:
1615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)        self.tab.ClearCache(force=True)
1623551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)
1631e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)  def ImplicitPageNavigation(self, test=None):
1643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    """Executes the implicit navigation that occurs for every page iteration.
1653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)
1663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    This function will be called once per page before any actions are executed.
1673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    """
168eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch    if test:
1691e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)      test.WillNavigateToPage(self.page, self.tab)
1701e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)      test.RunNavigateSteps(self.page, self.tab)
1711e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)      test.DidNavigateToPage(self.page, self.tab)
1723551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    else:
1733551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)      i = navigate.NavigateAction()
1741e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)      i.RunAction(self.page, self.tab, None)
1751e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)
176a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  def CleanUpPage(self, test):
177c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch    test.CleanUpAfterPage(self.page, self.tab)
1781e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    if self.page.credentials and self._did_login:
1791e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)      self.tab.browser.credentials.LoginNoLongerNeeded(
1801e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)          self.tab, self.page.credentials)
1812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
183a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)def AddCommandLineArgs(parser):
184a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  page_filter.PageFilter.AddCommandLineArgs(parser)
1853551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)  results_options.AddResultsOptions(parser)
1867d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
187a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  # Page set options
188a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
189a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  group.add_option('--pageset-shuffle', action='store_true',
190a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch      dest='pageset_shuffle',
191a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch      help='Shuffle the order of pages within a pageset.')
192a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  group.add_option('--pageset-shuffle-order-file',
193a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch      dest='pageset_shuffle_order_file', default=None,
194a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch      help='Filename of an output of a previously run test on the current '
195a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch      'pageset. The tests will run in the same order again, overriding '
196a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch      'what is specified by --page-repeat and --pageset-repeat.')
197a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  group.add_option('--page-repeat', default=1, type='int',
198a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch                   help='Number of times to repeat each individual page '
199a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch                   'before proceeding with the next page in the pageset.')
200a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  group.add_option('--pageset-repeat', default=1, type='int',
201a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch                   help='Number of times to repeat the entire pageset.')
2021320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  group.add_option('--max-failures', default=None, type='int',
2031320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci                   help='Maximum number of test failures before aborting '
2041320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci                   'the run. Defaults to the number specified by the '
2051320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci                   'PageTest.')
206a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  parser.add_option_group(group)
207a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch
208a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  # WPR options
209a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  group = optparse.OptionGroup(parser, 'Web Page Replay options')
2100529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  group.add_option('--use-live-sites',
2110529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch      dest='use_live_sites', action='store_true',
2120529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch      help='Run against live sites and ignore the Web Page Replay archives.')
213a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  parser.add_option_group(group)
214a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch
215116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch  parser.add_option('-d', '--also-run-disabled-tests',
216116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch                    dest='run_disabled_tests',
217116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch                    action='store_true', default=False,
218116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch                    help='Ignore @Disabled and @Enabled restrictions.')
2197d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
220a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)def ProcessCommandLineArgs(parser, args):
221a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  page_filter.PageFilter.ProcessCommandLineArgs(parser, args)
222a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
223a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  # Page set options
224a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  if args.pageset_shuffle_order_file and not args.pageset_shuffle:
225a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch    parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')
226a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch
227a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  if args.page_repeat < 1:
228a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch    parser.error('--page-repeat must be a positive integer.')
229a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  if args.pageset_repeat < 1:
230a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch    parser.error('--pageset-repeat must be a positive integer.')
231a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch
232a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)
23358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)def _PrepareAndRunPage(test, page_set, expectations, finder_options,
23458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)                       browser_options, page, credentials_path,
23558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)                       possible_browser, results, state):
236f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  if finder_options.use_live_sites:
237f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    browser_options.wpr_mode = wpr_modes.WPR_OFF
238f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)  elif browser_options.wpr_mode != wpr_modes.WPR_RECORD:
239cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    browser_options.wpr_mode = (
24058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)        wpr_modes.WPR_REPLAY
24158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)        if page.archive_path and os.path.isfile(page.archive_path)
24258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)        else wpr_modes.WPR_OFF)
2435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)
2445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)  max_attempts = test.attempts
2455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)  attempt_num = 0
2465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)  while attempt_num < max_attempts:
2471320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    attempt_num += 1
248a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch    try:
2495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      results.WillAttemptPageRun(attempt_num, max_attempts)
2505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)
251a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      if test.RestartBrowserBeforeEachPage() or page.startup_url:
2525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)        state.StopBrowser()
2535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)        # If we are restarting the browser for each page customize the per page
2545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)        # options for just the current page before starting the browser.
2555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)      state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
256cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                                 credentials_path, page.archive_path,
257cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                                 finder_options)
258cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)):
259cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        logging.info('Skip test for page %s because browser is not supported.'
260cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                     % page.url)
2615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)        return
262a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
2633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)      expectation = expectations.GetExpectationForPage(state.browser, page)
2643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)
265a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      _WaitForThermalThrottlingIfNeeded(state.browser.platform)
266a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
267424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      if finder_options.profiler:
268424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)        state.StartProfiling(page, finder_options)
269a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
270a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      try:
2716e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)        _RunPage(test, page, state, expectation, results)
272a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch        _CheckThermalThrottling(state.browser.platform)
273cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      except exceptions.TabCrashException as e:
2748bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)        if test.is_multi_tab_test:
275cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          logging.error('Aborting multi-tab test after tab %s crashed',
276cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                        page.url)
2778bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)          raise
278116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch        logging.warning(str(e))
279cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        state.StopBrowser()
280a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
281424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)      if finder_options.profiler:
282a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch        state.StopProfiling()
283a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
2845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)      if (test.StopBrowserAfterPage(state.browser, page)):
285a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch        state.StopBrowser()
286a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
2875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      return
288cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    except exceptions.BrowserGoneException as e:
289a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      state.StopBrowser()
2905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      if attempt_num == max_attempts:
291cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        logging.error('Aborting after too many retries')
292a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch        raise
2938bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)      if test.is_multi_tab_test:
294cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        logging.error('Aborting multi-tab test after browser crashed')
2958bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)        raise
296116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch      logging.warning(str(e))
297a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
298a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
29903b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)@decorators.Cache
3001320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef _UpdateCredentials(page_set):
3010529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  # Attempt to download the credentials file.
3020529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  if page_set.credentials_path:
3030529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch    try:
3040529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch      cloud_storage.GetIfChanged(
3050529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch          os.path.join(page_set.base_dir, page_set.credentials_path))
306f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)    except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
307f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)            cloud_storage.CloudStorageError) as e:
308f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)      logging.warning('Cannot retrieve credential file %s due to cloud storage '
309f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)                      'error %s', page_set.credentials_path, str(e))
310f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)
3111320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci
3121320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci@decorators.Cache
3131320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef _UpdatePageSetArchivesIfChanged(page_set):
3140529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  # Scan every serving directory for .sha1 files
3150529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  # and download them from Cloud Storage. Assume all data is public.
3160529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  all_serving_dirs = page_set.serving_dirs.copy()
3170529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  # Add individual page dirs to all serving dirs.
3180529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  for page in page_set:
3190529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch    if page.is_file:
3200529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch      all_serving_dirs.add(page.serving_dir)
3210529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  # Scan all serving dirs.
3220529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch  for serving_dir in all_serving_dirs:
3230529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch    if os.path.splitdrive(serving_dir)[1] == '/':
3240529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch      raise ValueError('Trying to serve root directory from HTTP server.')
3250529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch    for dirpath, _, filenames in os.walk(serving_dir):
3260529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch      for filename in filenames:
3270529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch        path, extension = os.path.splitext(
3280529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch            os.path.join(dirpath, filename))
3290529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch        if extension != '.sha1':
3300529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch          continue
331116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch        cloud_storage.GetIfChanged(path, page_set.bucket)
3320529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch
3330529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch
3345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def Run(test, page_set, expectations, finder_options, results):
3357d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  """Runs a given test against a given page_set with the given options."""
3361e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)  test.ValidatePageSet(page_set)
3371e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)
3387d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  # Create a possible_browser with the given options.
339bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch  try:
340424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    possible_browser = browser_finder.FindBrowser(finder_options)
341bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch  except browser_finder.BrowserTypeRequiredException, e:
342bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch    sys.stderr.write(str(e) + '\n')
343c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch    sys.exit(-1)
3447d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if not possible_browser:
345bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch    sys.stderr.write(
3461320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci        'No browser found. Available browsers:\n%s\n' %
3471320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci        '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
348c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch    sys.exit(-1)
3497d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
350cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  browser_options = possible_browser.finder_options.browser_options
351f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  browser_options.browser_type = possible_browser.browser_type
352cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  test.CustomizeBrowserOptions(browser_options)
353f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
3541320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  if (not decorators.IsEnabled(test, possible_browser) and
3551320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      not finder_options.run_disabled_tests):
356116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch    logging.warning('You are trying to run a disabled test.')
357116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch    logging.warning('Pass --also-run-disabled-tests to squelch this message.')
3585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    return
3595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)
36003b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)  if possible_browser.IsRemote():
36103b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)    possible_browser.RunRemote()
36203b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)    sys.exit(0)
36303b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles)
3647d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  # Reorder page set based on options.
365424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  pages = _ShuffleAndFilterPageSet(page_set, finder_options)
3667d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
3671320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  if not finder_options.use_live_sites:
3681320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    _UpdateCredentials(page_set)
3691320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    if browser_options.wpr_mode != wpr_modes.WPR_RECORD:
3701320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      _UpdatePageSetArchivesIfChanged(page_set)
3711320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      pages = _CheckArchives(page_set, pages, results)
3727d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
3737d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  # Verify credentials path.
3747d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  credentials_path = None
3757d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if page_set.credentials_path:
3767d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    credentials_path = os.path.join(os.path.dirname(page_set.file_path),
3777d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                                    page_set.credentials_path)
3787d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    if not os.path.exists(credentials_path):
3797d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      credentials_path = None
3807d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
3817d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  # Set up user agent.
382cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  browser_options.browser_user_agent_type = page_set.user_agent_type or None
3837d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
3844e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)  if finder_options.profiler:
3854e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)    profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
386cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
387cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                                           finder_options)
3887d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
389a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch  for page in list(pages):
390a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch    if not test.CanRunForPage(page):
3915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      results.WillRunPage(page)
3925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)      logging.debug('Skipping test: it cannot run for %s', page.url)
3935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      results.AddValue(skip.SkipValue(page, 'Test cannot run'))
3945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      results.DidRunPage(page)
395a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      pages.remove(page)
396a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
397a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch  if not pages:
3985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    return
399a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch
4007d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  state = _RunState()
4011320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  pages_with_discarded_first_result = set()
4021320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  max_failures = finder_options.max_failures  # command-line gets priority
4031320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  if max_failures is None:
4041320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    max_failures = test.max_failures  # may be None
4057d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4067d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  try:
407a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    test.WillRunTest(finder_options)
4081320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    for _ in xrange(finder_options.pageset_repeat):
409a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch      for page in pages:
4106e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)        if test.IsExiting():
4116e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)          break
4121320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci        for _ in xrange(finder_options.page_repeat):
4135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)          results.WillRunPage(page)
414116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch          try:
4155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)            _PrepareAndRunPage(
416116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch                test, page_set, expectations, finder_options, browser_options,
417116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch                page, credentials_path, possible_browser, results, state)
418116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch          finally:
4191320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci            discard_run = (test.discard_first_result and
4201320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci                           page not in pages_with_discarded_first_result)
4211320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci            if discard_run:
4221320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci              pages_with_discarded_first_result.add(page)
4235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)            results.DidRunPage(page, discard_run=discard_run)
4241320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci        if max_failures is not None and len(results.failures) > max_failures:
425a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)          logging.error('Too many failures. Aborting.')
426a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)          test.RequestExit()
4277d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  finally:
4286e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)    test.DidRunTest(state.browser, results)
4297d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    state.StopBrowser()
4307d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4317d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
432424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)def _ShuffleAndFilterPageSet(page_set, finder_options):
433424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  if finder_options.pageset_shuffle_order_file:
434424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)
4357d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  pages = [page for page in page_set.pages[:]
436a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)           if not page.disabled and page_filter.PageFilter.IsSelected(page)]
437424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  if finder_options.pageset_shuffle:
4381320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    random.shuffle(pages)
439a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch  return pages
4407d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4417d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4427d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)def _CheckArchives(page_set, pages, results):
4437d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  """Returns a subset of pages that are local or have WPR archives.
4447d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4451320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  Logs warnings if any are missing.
4461320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  """
4471320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  # Warn of any problems with the entire page set.
4481320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  if any(not p.is_local for p in pages):
4497d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    if not page_set.archive_data_file:
4507d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      logging.warning('The page set is missing an "archive_data_file" '
4517d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                      'property. Skipping any live sites. To include them, '
4520529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch                      'pass the flag --use-live-sites.')
4537d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    if not page_set.wpr_archive_info:
4547d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      logging.warning('The archive info file is missing. '
4557d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                      'To fix this, either add svn-internal to your '
4567d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                      '.gclient using http://goto/read-src-internal, '
4577d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                      'or create a new archive using record_wpr.')
4587d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4591320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  # Warn of any problems with individual pages and return valid pages.
4607d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  pages_missing_archive_path = []
4617d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  pages_missing_archive_data = []
4621320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  valid_pages = []
4637d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  for page in pages:
4641320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    if not page.is_local and not page.archive_path:
4657d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      pages_missing_archive_path.append(page)
4661320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    elif not page.is_local and not os.path.isfile(page.archive_path):
4677d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)      pages_missing_archive_data.append(page)
4681320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci    else:
4691320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci      valid_pages.append(page)
4707d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if pages_missing_archive_path:
4717d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    logging.warning('The page set archives for some pages do not exist. '
4727d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                    'Skipping those pages. To fix this, record those pages '
4737d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                    'using record_wpr. To ignore this warning and run '
4740529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch                    'against live sites, pass the flag --use-live-sites.')
4757d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if pages_missing_archive_data:
4767d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    logging.warning('The page set archives for some pages are missing. '
4777d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                    'Someone forgot to check them in, or they were deleted. '
4787d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                    'Skipping those pages. To fix this, record those pages '
4797d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                    'using record_wpr. To ignore this warning and run '
4800529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch                    'against live sites, pass the flag --use-live-sites.')
4817d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  for page in pages_missing_archive_path + pages_missing_archive_data:
4825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    results.WillRunPage(page)
4835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    results.AddValue(failure.FailureValue.FromMessage(
4845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)        page, 'Page set archive doesn\'t exist.'))
4855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    results.DidRunPage(page)
4861320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci  return valid_pages
4877d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4887d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
4896e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)def _RunPage(test, page, state, expectation, results):
490424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)  if expectation == 'skip':
4915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)    logging.debug('Skipping test: Skip expectation for %s', page.url)
4925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
493424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)    return
494424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)
4951e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)  page_state = PageState(page, test.TabForPage(page, state.browser))
4963551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)
4973551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)  def ProcessError():
4983551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    if expectation == 'fail':
499cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      msg = 'Expected exception while running %s' % page.url
5003551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    else:
501cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      msg = 'Exception while running %s' % page.url
5025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      results.AddValue(failure.FailureValue(page, sys.exc_info()))
503cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    exception_formatter.PrintFormattedException(msg=msg)
5047d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
5057d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  try:
5061e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    page_state.PreparePage(test)
5076e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)    page_state.ImplicitPageNavigation(test)
50823730a6e56a168d1879203e4b3819bb36e3d8f1fTorne (Richard Coles)    test.RunPage(page, page_state.tab, results)
5091e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)    util.CloseConnections(page_state.tab)
5104ad1aa43a48567659193a298fad74f55e00b3dd9Ben Murdoch  except page_test.TestNotSupportedOnPlatformFailure:
5114ad1aa43a48567659193a298fad74f55e00b3dd9Ben Murdoch    raise
5127d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  except page_test.Failure:
513ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch    if expectation == 'fail':
514cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      exception_formatter.PrintFormattedException(
515cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          msg='Expected failure while running %s' % page.url)
516ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch    else:
517cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      exception_formatter.PrintFormattedException(
518cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)          msg='Failure while running %s' % page.url)
5195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)      results.AddValue(failure.FailureValue(page, sys.exc_info()))
520558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch  except (util.TimeoutException, exceptions.LoginException,
521558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch          exceptions.ProfilingException):
5223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    ProcessError()
5237d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  except (exceptions.TabCrashException, exceptions.BrowserGoneException):
5243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)    ProcessError()
5257d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
5267d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    raise
527a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch  except page_action.PageActionNotSupported as e:
5285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e))
5297d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  except Exception:
530cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    exception_formatter.PrintFormattedException(
531cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)        msg='Unhandled exception while running %s' % page.url)
5325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)    results.AddValue(failure.FailureValue(page, sys.exc_info()))
5337d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  else:
534ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch    if expectation == 'fail':
535ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch      logging.warning('%s was expected to fail, but passed.\n', page.url)
5367d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  finally:
537a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    page_state.CleanUpPage(test)
5387d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
5397d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
5407d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)def _WaitForThermalThrottlingIfNeeded(platform):
5417d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if not platform.CanMonitorThermalThrottling():
5427d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    return
5437d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  thermal_throttling_retry = 0
5447d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  while (platform.IsThermallyThrottled() and
5457d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)         thermal_throttling_retry < 3):
5467d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    logging.warning('Thermally throttled, waiting (%d)...',
5477d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)                    thermal_throttling_retry)
5487d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    thermal_throttling_retry += 1
5497d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    time.sleep(thermal_throttling_retry * 2)
5507d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
5517dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch  if thermal_throttling_retry and platform.IsThermallyThrottled():
552cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    logging.warning('Device is thermally throttled before running '
553cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                    'performance tests, results will vary.')
5547d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
5557d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)
5567d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)def _CheckThermalThrottling(platform):
5577d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if not platform.CanMonitorThermalThrottling():
5587d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)    return
5597d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)  if platform.HasBeenThermallyThrottled():
560cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    logging.warning('Device has been thermally throttled during '
561cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)                    'performance tests, results will vary.')
562