1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import optparse
7import os
8import random
9import sys
10import tempfile
11import time
12
13from telemetry import decorators
14from telemetry.core import browser_finder
15from telemetry.core import browser_info
16from telemetry.core import exceptions
17from telemetry.core import util
18from telemetry.core import wpr_modes
19from telemetry.core.platform.profiler import profiler_finder
20from telemetry.page import page_filter
21from telemetry.page import page_test
22from telemetry.page.actions import navigate
23from telemetry.page.actions import page_action
24from telemetry.results import results_options
25from telemetry.util import cloud_storage
26from telemetry.util import exception_formatter
27from telemetry.value import failure
28from telemetry.value import skip
29
30
31class _RunState(object):
32  def __init__(self):
33    self.browser = None
34
35    self._append_to_existing_wpr = False
36    self._last_archive_path = None
37    self._first_browser = True
38    self.profiler_dir = None
39
40  def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
41                           credentials_path, archive_path, finder_options):
42    started_browser = not self.browser
43    # Create a browser.
44    if not self.browser:
45      test.CustomizeBrowserOptionsForSinglePage(page, finder_options)
46      possible_browser.SetReplayArchivePath(archive_path,
47                                        self._append_to_existing_wpr,
48                                        page_set.make_javascript_deterministic)
49      possible_browser.SetCredentialsPath(credentials_path)
50      self._last_archive_path = page.archive_path
51
52      test.WillStartBrowser(possible_browser.platform)
53      self.browser = possible_browser.Create()
54      test.DidStartBrowser(self.browser)
55
56      if self._first_browser:
57        self._first_browser = False
58        self.browser.credentials.WarnIfMissingCredentials(page_set)
59        logging.info('OS: %s %s',
60                     self.browser.platform.GetOSName(),
61                     self.browser.platform.GetOSVersionName())
62        if self.browser.supports_system_info:
63          system_info = self.browser.GetSystemInfo()
64          if system_info.model_name:
65            logging.info('Model: %s', system_info.model_name)
66          if system_info.gpu:
67            for i, device in enumerate(system_info.gpu.devices):
68              logging.info('GPU device %d: %s', i, device)
69            if system_info.gpu.aux_attributes:
70              logging.info('GPU Attributes:')
71              for k, v in sorted(system_info.gpu.aux_attributes.iteritems()):
72                logging.info('  %-20s: %s', k, v)
73            if system_info.gpu.feature_status:
74              logging.info('Feature Status:')
75              for k, v in sorted(system_info.gpu.feature_status.iteritems()):
76                logging.info('  %-20s: %s', k, v)
77            if system_info.gpu.driver_bug_workarounds:
78              logging.info('Driver Bug Workarounds:')
79              for workaround in system_info.gpu.driver_bug_workarounds:
80                logging.info('  %s', workaround)
81          else:
82            logging.info('No GPU devices')
83        else:
84          logging.warning('System info not supported')
85    else:
86      # Set up WPR path if it changed.
87      if page.archive_path and self._last_archive_path != page.archive_path:
88        self.browser.SetReplayArchivePath(
89            page.archive_path,
90            self._append_to_existing_wpr,
91            page_set.make_javascript_deterministic)
92        self._last_archive_path = page.archive_path
93
94    if self.browser.supports_tab_control and test.close_tabs_before_run:
95      # Create a tab if there's none.
96      if len(self.browser.tabs) == 0:
97        self.browser.tabs.New()
98
99      # Ensure only one tab is open, unless the test is a multi-tab test.
100      if not test.is_multi_tab_test:
101        while len(self.browser.tabs) > 1:
102          self.browser.tabs[-1].Close()
103
104      # Must wait for tab to commit otherwise it can commit after the next
105      # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame()
106      # will cancel the next navigation because it's pending. This manifests as
107      # the first navigation in a PageSet freezing indefinitly because the
108      # navigation was silently cancelled when |self.browser.tabs[0]| was
109      # committed. Only do this when we just started the browser, otherwise
110      # there are cases where previous pages in a PageSet never complete
111      # loading so we'll wait forever.
112      if started_browser:
113        self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete()
114
115  def StopBrowser(self):
116    if self.browser:
117      self.browser.Close()
118      self.browser = None
119
120      # Restarting the state will also restart the wpr server. If we're
121      # recording, we need to continue adding into the same wpr archive,
122      # not overwrite it.
123      self._append_to_existing_wpr = True
124
125  def StartProfiling(self, page, finder_options):
126    if not self.profiler_dir:
127      self.profiler_dir = tempfile.mkdtemp()
128    output_file = os.path.join(self.profiler_dir, page.file_safe_name)
129    is_repeating = (finder_options.page_repeat != 1 or
130                    finder_options.pageset_repeat != 1)
131    if is_repeating:
132      output_file = util.GetSequentialFileName(output_file)
133    self.browser.platform.profiling_controller.Start(
134        finder_options.profiler, output_file)
135
136  def StopProfiling(self):
137    if self.browser:
138      self.browser.platform.profiling_controller.Stop()
139
140
141class PageState(object):
142  def __init__(self, page, tab):
143    self.page = page
144    self.tab = tab
145
146    self._did_login = False
147
148  def PreparePage(self, test=None):
149    if self.page.is_file:
150      self.tab.browser.SetHTTPServerDirectories(
151          self.page.page_set.serving_dirs | set([self.page.serving_dir]))
152
153    if self.page.credentials:
154      if not self.tab.browser.credentials.LoginNeeded(
155          self.tab, self.page.credentials):
156        raise page_test.Failure('Login as ' + self.page.credentials + ' failed')
157      self._did_login = True
158
159    if test:
160      if test.clear_cache_before_each_run:
161        self.tab.ClearCache(force=True)
162
163  def ImplicitPageNavigation(self, test=None):
164    """Executes the implicit navigation that occurs for every page iteration.
165
166    This function will be called once per page before any actions are executed.
167    """
168    if test:
169      test.WillNavigateToPage(self.page, self.tab)
170      test.RunNavigateSteps(self.page, self.tab)
171      test.DidNavigateToPage(self.page, self.tab)
172    else:
173      i = navigate.NavigateAction()
174      i.RunAction(self.page, self.tab, None)
175
176  def CleanUpPage(self, test):
177    test.CleanUpAfterPage(self.page, self.tab)
178    if self.page.credentials and self._did_login:
179      self.tab.browser.credentials.LoginNoLongerNeeded(
180          self.tab, self.page.credentials)
181
182
183def AddCommandLineArgs(parser):
184  page_filter.PageFilter.AddCommandLineArgs(parser)
185  results_options.AddResultsOptions(parser)
186
187  # Page set options
188  group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
189  group.add_option('--pageset-shuffle', action='store_true',
190      dest='pageset_shuffle',
191      help='Shuffle the order of pages within a pageset.')
192  group.add_option('--pageset-shuffle-order-file',
193      dest='pageset_shuffle_order_file', default=None,
194      help='Filename of an output of a previously run test on the current '
195      'pageset. The tests will run in the same order again, overriding '
196      'what is specified by --page-repeat and --pageset-repeat.')
197  group.add_option('--page-repeat', default=1, type='int',
198                   help='Number of times to repeat each individual page '
199                   'before proceeding with the next page in the pageset.')
200  group.add_option('--pageset-repeat', default=1, type='int',
201                   help='Number of times to repeat the entire pageset.')
202  group.add_option('--max-failures', default=None, type='int',
203                   help='Maximum number of test failures before aborting '
204                   'the run. Defaults to the number specified by the '
205                   'PageTest.')
206  parser.add_option_group(group)
207
208  # WPR options
209  group = optparse.OptionGroup(parser, 'Web Page Replay options')
210  group.add_option('--use-live-sites',
211      dest='use_live_sites', action='store_true',
212      help='Run against live sites and ignore the Web Page Replay archives.')
213  parser.add_option_group(group)
214
215  parser.add_option('-d', '--also-run-disabled-tests',
216                    dest='run_disabled_tests',
217                    action='store_true', default=False,
218                    help='Ignore @Disabled and @Enabled restrictions.')
219
220def ProcessCommandLineArgs(parser, args):
221  page_filter.PageFilter.ProcessCommandLineArgs(parser, args)
222
223  # Page set options
224  if args.pageset_shuffle_order_file and not args.pageset_shuffle:
225    parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')
226
227  if args.page_repeat < 1:
228    parser.error('--page-repeat must be a positive integer.')
229  if args.pageset_repeat < 1:
230    parser.error('--pageset-repeat must be a positive integer.')
231
232
233def _PrepareAndRunPage(test, page_set, expectations, finder_options,
234                       browser_options, page, credentials_path,
235                       possible_browser, results, state):
236  if finder_options.use_live_sites:
237    browser_options.wpr_mode = wpr_modes.WPR_OFF
238  elif browser_options.wpr_mode != wpr_modes.WPR_RECORD:
239    browser_options.wpr_mode = (
240        wpr_modes.WPR_REPLAY
241        if page.archive_path and os.path.isfile(page.archive_path)
242        else wpr_modes.WPR_OFF)
243
244  max_attempts = test.attempts
245  attempt_num = 0
246  while attempt_num < max_attempts:
247    attempt_num += 1
248    try:
249      results.WillAttemptPageRun(attempt_num, max_attempts)
250
251      if test.RestartBrowserBeforeEachPage() or page.startup_url:
252        state.StopBrowser()
253        # If we are restarting the browser for each page customize the per page
254        # options for just the current page before starting the browser.
255      state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
256                                 credentials_path, page.archive_path,
257                                 finder_options)
258      if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)):
259        logging.info('Skip test for page %s because browser is not supported.'
260                     % page.url)
261        return
262
263      expectation = expectations.GetExpectationForPage(state.browser, page)
264
265      _WaitForThermalThrottlingIfNeeded(state.browser.platform)
266
267      if finder_options.profiler:
268        state.StartProfiling(page, finder_options)
269
270      try:
271        _RunPage(test, page, state, expectation, results)
272        _CheckThermalThrottling(state.browser.platform)
273      except exceptions.TabCrashException as e:
274        if test.is_multi_tab_test:
275          logging.error('Aborting multi-tab test after tab %s crashed',
276                        page.url)
277          raise
278        logging.warning(str(e))
279        state.StopBrowser()
280
281      if finder_options.profiler:
282        state.StopProfiling()
283
284      if (test.StopBrowserAfterPage(state.browser, page)):
285        state.StopBrowser()
286
287      return
288    except exceptions.BrowserGoneException as e:
289      state.StopBrowser()
290      if attempt_num == max_attempts:
291        logging.error('Aborting after too many retries')
292        raise
293      if test.is_multi_tab_test:
294        logging.error('Aborting multi-tab test after browser crashed')
295        raise
296      logging.warning(str(e))
297
298
299@decorators.Cache
300def _UpdateCredentials(page_set):
301  # Attempt to download the credentials file.
302  if page_set.credentials_path:
303    try:
304      cloud_storage.GetIfChanged(
305          os.path.join(page_set.base_dir, page_set.credentials_path))
306    except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
307            cloud_storage.CloudStorageError) as e:
308      logging.warning('Cannot retrieve credential file %s due to cloud storage '
309                      'error %s', page_set.credentials_path, str(e))
310
311
312@decorators.Cache
313def _UpdatePageSetArchivesIfChanged(page_set):
314  # Scan every serving directory for .sha1 files
315  # and download them from Cloud Storage. Assume all data is public.
316  all_serving_dirs = page_set.serving_dirs.copy()
317  # Add individual page dirs to all serving dirs.
318  for page in page_set:
319    if page.is_file:
320      all_serving_dirs.add(page.serving_dir)
321  # Scan all serving dirs.
322  for serving_dir in all_serving_dirs:
323    if os.path.splitdrive(serving_dir)[1] == '/':
324      raise ValueError('Trying to serve root directory from HTTP server.')
325    for dirpath, _, filenames in os.walk(serving_dir):
326      for filename in filenames:
327        path, extension = os.path.splitext(
328            os.path.join(dirpath, filename))
329        if extension != '.sha1':
330          continue
331        cloud_storage.GetIfChanged(path, page_set.bucket)
332
333
334def Run(test, page_set, expectations, finder_options, results):
335  """Runs a given test against a given page_set with the given options."""
336  test.ValidatePageSet(page_set)
337
338  # Create a possible_browser with the given options.
339  try:
340    possible_browser = browser_finder.FindBrowser(finder_options)
341  except browser_finder.BrowserTypeRequiredException, e:
342    sys.stderr.write(str(e) + '\n')
343    sys.exit(-1)
344  if not possible_browser:
345    sys.stderr.write(
346        'No browser found. Available browsers:\n%s\n' %
347        '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
348    sys.exit(-1)
349
350  browser_options = possible_browser.finder_options.browser_options
351  browser_options.browser_type = possible_browser.browser_type
352  test.CustomizeBrowserOptions(browser_options)
353
354  if (not decorators.IsEnabled(test, possible_browser) and
355      not finder_options.run_disabled_tests):
356    logging.warning('You are trying to run a disabled test.')
357    logging.warning('Pass --also-run-disabled-tests to squelch this message.')
358    return
359
360  if possible_browser.IsRemote():
361    possible_browser.RunRemote()
362    sys.exit(0)
363
364  # Reorder page set based on options.
365  pages = _ShuffleAndFilterPageSet(page_set, finder_options)
366
367  if not finder_options.use_live_sites:
368    _UpdateCredentials(page_set)
369    if browser_options.wpr_mode != wpr_modes.WPR_RECORD:
370      _UpdatePageSetArchivesIfChanged(page_set)
371      pages = _CheckArchives(page_set, pages, results)
372
373  # Verify credentials path.
374  credentials_path = None
375  if page_set.credentials_path:
376    credentials_path = os.path.join(os.path.dirname(page_set.file_path),
377                                    page_set.credentials_path)
378    if not os.path.exists(credentials_path):
379      credentials_path = None
380
381  # Set up user agent.
382  browser_options.browser_user_agent_type = page_set.user_agent_type or None
383
384  if finder_options.profiler:
385    profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
386    profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
387                                           finder_options)
388
389  for page in list(pages):
390    if not test.CanRunForPage(page):
391      results.WillRunPage(page)
392      logging.debug('Skipping test: it cannot run for %s', page.url)
393      results.AddValue(skip.SkipValue(page, 'Test cannot run'))
394      results.DidRunPage(page)
395      pages.remove(page)
396
397  if not pages:
398    return
399
400  state = _RunState()
401  pages_with_discarded_first_result = set()
402  max_failures = finder_options.max_failures  # command-line gets priority
403  if max_failures is None:
404    max_failures = test.max_failures  # may be None
405
406  try:
407    test.WillRunTest(finder_options)
408    for _ in xrange(finder_options.pageset_repeat):
409      for page in pages:
410        if test.IsExiting():
411          break
412        for _ in xrange(finder_options.page_repeat):
413          results.WillRunPage(page)
414          try:
415            _PrepareAndRunPage(
416                test, page_set, expectations, finder_options, browser_options,
417                page, credentials_path, possible_browser, results, state)
418          finally:
419            discard_run = (test.discard_first_result and
420                           page not in pages_with_discarded_first_result)
421            if discard_run:
422              pages_with_discarded_first_result.add(page)
423            results.DidRunPage(page, discard_run=discard_run)
424        if max_failures is not None and len(results.failures) > max_failures:
425          logging.error('Too many failures. Aborting.')
426          test.RequestExit()
427  finally:
428    test.DidRunTest(state.browser, results)
429    state.StopBrowser()
430
431
432def _ShuffleAndFilterPageSet(page_set, finder_options):
433  if finder_options.pageset_shuffle_order_file:
434    return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)
435  pages = [page for page in page_set.pages[:]
436           if not page.disabled and page_filter.PageFilter.IsSelected(page)]
437  if finder_options.pageset_shuffle:
438    random.shuffle(pages)
439  return pages
440
441
442def _CheckArchives(page_set, pages, results):
443  """Returns a subset of pages that are local or have WPR archives.
444
445  Logs warnings if any are missing.
446  """
447  # Warn of any problems with the entire page set.
448  if any(not p.is_local for p in pages):
449    if not page_set.archive_data_file:
450      logging.warning('The page set is missing an "archive_data_file" '
451                      'property. Skipping any live sites. To include them, '
452                      'pass the flag --use-live-sites.')
453    if not page_set.wpr_archive_info:
454      logging.warning('The archive info file is missing. '
455                      'To fix this, either add svn-internal to your '
456                      '.gclient using http://goto/read-src-internal, '
457                      'or create a new archive using record_wpr.')
458
459  # Warn of any problems with individual pages and return valid pages.
460  pages_missing_archive_path = []
461  pages_missing_archive_data = []
462  valid_pages = []
463  for page in pages:
464    if not page.is_local and not page.archive_path:
465      pages_missing_archive_path.append(page)
466    elif not page.is_local and not os.path.isfile(page.archive_path):
467      pages_missing_archive_data.append(page)
468    else:
469      valid_pages.append(page)
470  if pages_missing_archive_path:
471    logging.warning('The page set archives for some pages do not exist. '
472                    'Skipping those pages. To fix this, record those pages '
473                    'using record_wpr. To ignore this warning and run '
474                    'against live sites, pass the flag --use-live-sites.')
475  if pages_missing_archive_data:
476    logging.warning('The page set archives for some pages are missing. '
477                    'Someone forgot to check them in, or they were deleted. '
478                    'Skipping those pages. To fix this, record those pages '
479                    'using record_wpr. To ignore this warning and run '
480                    'against live sites, pass the flag --use-live-sites.')
481  for page in pages_missing_archive_path + pages_missing_archive_data:
482    results.WillRunPage(page)
483    results.AddValue(failure.FailureValue.FromMessage(
484        page, 'Page set archive doesn\'t exist.'))
485    results.DidRunPage(page)
486  return valid_pages
487
488
489def _RunPage(test, page, state, expectation, results):
490  if expectation == 'skip':
491    logging.debug('Skipping test: Skip expectation for %s', page.url)
492    results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
493    return
494
495  page_state = PageState(page, test.TabForPage(page, state.browser))
496
497  def ProcessError():
498    if expectation == 'fail':
499      msg = 'Expected exception while running %s' % page.url
500    else:
501      msg = 'Exception while running %s' % page.url
502      results.AddValue(failure.FailureValue(page, sys.exc_info()))
503    exception_formatter.PrintFormattedException(msg=msg)
504
505  try:
506    page_state.PreparePage(test)
507    page_state.ImplicitPageNavigation(test)
508    test.RunPage(page, page_state.tab, results)
509    util.CloseConnections(page_state.tab)
510  except page_test.TestNotSupportedOnPlatformFailure:
511    raise
512  except page_test.Failure:
513    if expectation == 'fail':
514      exception_formatter.PrintFormattedException(
515          msg='Expected failure while running %s' % page.url)
516    else:
517      exception_formatter.PrintFormattedException(
518          msg='Failure while running %s' % page.url)
519      results.AddValue(failure.FailureValue(page, sys.exc_info()))
520  except (util.TimeoutException, exceptions.LoginException,
521          exceptions.ProfilingException):
522    ProcessError()
523  except (exceptions.TabCrashException, exceptions.BrowserGoneException):
524    ProcessError()
525    # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
526    raise
527  except page_action.PageActionNotSupported as e:
528    results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e))
529  except Exception:
530    exception_formatter.PrintFormattedException(
531        msg='Unhandled exception while running %s' % page.url)
532    results.AddValue(failure.FailureValue(page, sys.exc_info()))
533  else:
534    if expectation == 'fail':
535      logging.warning('%s was expected to fail, but passed.\n', page.url)
536  finally:
537    page_state.CleanUpPage(test)
538
539
540def _WaitForThermalThrottlingIfNeeded(platform):
541  if not platform.CanMonitorThermalThrottling():
542    return
543  thermal_throttling_retry = 0
544  while (platform.IsThermallyThrottled() and
545         thermal_throttling_retry < 3):
546    logging.warning('Thermally throttled, waiting (%d)...',
547                    thermal_throttling_retry)
548    thermal_throttling_retry += 1
549    time.sleep(thermal_throttling_retry * 2)
550
551  if thermal_throttling_retry and platform.IsThermallyThrottled():
552    logging.warning('Device is thermally throttled before running '
553                    'performance tests, results will vary.')
554
555
556def _CheckThermalThrottling(platform):
557  if not platform.CanMonitorThermalThrottling():
558    return
559  if platform.HasBeenThermallyThrottled():
560    logging.warning('Device has been thermally throttled during '
561                    'performance tests, results will vary.')
562