1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import optparse 7import os 8import random 9import sys 10import tempfile 11import time 12 13from telemetry import decorators 14from telemetry.core import browser_finder 15from telemetry.core import browser_info 16from telemetry.core import exceptions 17from telemetry.core import util 18from telemetry.core import wpr_modes 19from telemetry.core.platform.profiler import profiler_finder 20from telemetry.page import page_filter 21from telemetry.page import page_test 22from telemetry.page.actions import navigate 23from telemetry.page.actions import page_action 24from telemetry.results import results_options 25from telemetry.util import cloud_storage 26from telemetry.util import exception_formatter 27from telemetry.value import failure 28from telemetry.value import skip 29 30 31class _RunState(object): 32 def __init__(self): 33 self.browser = None 34 35 self._append_to_existing_wpr = False 36 self._last_archive_path = None 37 self._first_browser = True 38 self.profiler_dir = None 39 40 def StartBrowserIfNeeded(self, test, page_set, page, possible_browser, 41 credentials_path, archive_path, finder_options): 42 started_browser = not self.browser 43 # Create a browser. 44 if not self.browser: 45 test.CustomizeBrowserOptionsForSinglePage(page, finder_options) 46 possible_browser.SetReplayArchivePath(archive_path, 47 self._append_to_existing_wpr, 48 page_set.make_javascript_deterministic) 49 possible_browser.SetCredentialsPath(credentials_path) 50 self._last_archive_path = page.archive_path 51 52 test.WillStartBrowser(possible_browser.platform) 53 self.browser = possible_browser.Create() 54 test.DidStartBrowser(self.browser) 55 56 if self._first_browser: 57 self._first_browser = False 58 self.browser.credentials.WarnIfMissingCredentials(page_set) 59 logging.info('OS: %s %s', 60 self.browser.platform.GetOSName(), 61 self.browser.platform.GetOSVersionName()) 62 if self.browser.supports_system_info: 63 system_info = self.browser.GetSystemInfo() 64 if system_info.model_name: 65 logging.info('Model: %s', system_info.model_name) 66 if system_info.gpu: 67 for i, device in enumerate(system_info.gpu.devices): 68 logging.info('GPU device %d: %s', i, device) 69 if system_info.gpu.aux_attributes: 70 logging.info('GPU Attributes:') 71 for k, v in sorted(system_info.gpu.aux_attributes.iteritems()): 72 logging.info(' %-20s: %s', k, v) 73 if system_info.gpu.feature_status: 74 logging.info('Feature Status:') 75 for k, v in sorted(system_info.gpu.feature_status.iteritems()): 76 logging.info(' %-20s: %s', k, v) 77 if system_info.gpu.driver_bug_workarounds: 78 logging.info('Driver Bug Workarounds:') 79 for workaround in system_info.gpu.driver_bug_workarounds: 80 logging.info(' %s', workaround) 81 else: 82 logging.info('No GPU devices') 83 else: 84 logging.warning('System info not supported') 85 else: 86 # Set up WPR path if it changed. 87 if page.archive_path and self._last_archive_path != page.archive_path: 88 self.browser.SetReplayArchivePath( 89 page.archive_path, 90 self._append_to_existing_wpr, 91 page_set.make_javascript_deterministic) 92 self._last_archive_path = page.archive_path 93 94 if self.browser.supports_tab_control and test.close_tabs_before_run: 95 # Create a tab if there's none. 96 if len(self.browser.tabs) == 0: 97 self.browser.tabs.New() 98 99 # Ensure only one tab is open, unless the test is a multi-tab test. 100 if not test.is_multi_tab_test: 101 while len(self.browser.tabs) > 1: 102 self.browser.tabs[-1].Close() 103 104 # Must wait for tab to commit otherwise it can commit after the next 105 # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame() 106 # will cancel the next navigation because it's pending. This manifests as 107 # the first navigation in a PageSet freezing indefinitly because the 108 # navigation was silently cancelled when |self.browser.tabs[0]| was 109 # committed. Only do this when we just started the browser, otherwise 110 # there are cases where previous pages in a PageSet never complete 111 # loading so we'll wait forever. 112 if started_browser: 113 self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete() 114 115 def StopBrowser(self): 116 if self.browser: 117 self.browser.Close() 118 self.browser = None 119 120 # Restarting the state will also restart the wpr server. If we're 121 # recording, we need to continue adding into the same wpr archive, 122 # not overwrite it. 123 self._append_to_existing_wpr = True 124 125 def StartProfiling(self, page, finder_options): 126 if not self.profiler_dir: 127 self.profiler_dir = tempfile.mkdtemp() 128 output_file = os.path.join(self.profiler_dir, page.file_safe_name) 129 is_repeating = (finder_options.page_repeat != 1 or 130 finder_options.pageset_repeat != 1) 131 if is_repeating: 132 output_file = util.GetSequentialFileName(output_file) 133 self.browser.platform.profiling_controller.Start( 134 finder_options.profiler, output_file) 135 136 def StopProfiling(self): 137 if self.browser: 138 self.browser.platform.profiling_controller.Stop() 139 140 141class PageState(object): 142 def __init__(self, page, tab): 143 self.page = page 144 self.tab = tab 145 146 self._did_login = False 147 148 def PreparePage(self, test=None): 149 if self.page.is_file: 150 self.tab.browser.SetHTTPServerDirectories( 151 self.page.page_set.serving_dirs | set([self.page.serving_dir])) 152 153 if self.page.credentials: 154 if not self.tab.browser.credentials.LoginNeeded( 155 self.tab, self.page.credentials): 156 raise page_test.Failure('Login as ' + self.page.credentials + ' failed') 157 self._did_login = True 158 159 if test: 160 if test.clear_cache_before_each_run: 161 self.tab.ClearCache(force=True) 162 163 def ImplicitPageNavigation(self, test=None): 164 """Executes the implicit navigation that occurs for every page iteration. 165 166 This function will be called once per page before any actions are executed. 167 """ 168 if test: 169 test.WillNavigateToPage(self.page, self.tab) 170 test.RunNavigateSteps(self.page, self.tab) 171 test.DidNavigateToPage(self.page, self.tab) 172 else: 173 i = navigate.NavigateAction() 174 i.RunAction(self.page, self.tab, None) 175 176 def CleanUpPage(self, test): 177 test.CleanUpAfterPage(self.page, self.tab) 178 if self.page.credentials and self._did_login: 179 self.tab.browser.credentials.LoginNoLongerNeeded( 180 self.tab, self.page.credentials) 181 182 183def AddCommandLineArgs(parser): 184 page_filter.PageFilter.AddCommandLineArgs(parser) 185 results_options.AddResultsOptions(parser) 186 187 # Page set options 188 group = optparse.OptionGroup(parser, 'Page set ordering and repeat options') 189 group.add_option('--pageset-shuffle', action='store_true', 190 dest='pageset_shuffle', 191 help='Shuffle the order of pages within a pageset.') 192 group.add_option('--pageset-shuffle-order-file', 193 dest='pageset_shuffle_order_file', default=None, 194 help='Filename of an output of a previously run test on the current ' 195 'pageset. The tests will run in the same order again, overriding ' 196 'what is specified by --page-repeat and --pageset-repeat.') 197 group.add_option('--page-repeat', default=1, type='int', 198 help='Number of times to repeat each individual page ' 199 'before proceeding with the next page in the pageset.') 200 group.add_option('--pageset-repeat', default=1, type='int', 201 help='Number of times to repeat the entire pageset.') 202 group.add_option('--max-failures', default=None, type='int', 203 help='Maximum number of test failures before aborting ' 204 'the run. Defaults to the number specified by the ' 205 'PageTest.') 206 parser.add_option_group(group) 207 208 # WPR options 209 group = optparse.OptionGroup(parser, 'Web Page Replay options') 210 group.add_option('--use-live-sites', 211 dest='use_live_sites', action='store_true', 212 help='Run against live sites and ignore the Web Page Replay archives.') 213 parser.add_option_group(group) 214 215 parser.add_option('-d', '--also-run-disabled-tests', 216 dest='run_disabled_tests', 217 action='store_true', default=False, 218 help='Ignore @Disabled and @Enabled restrictions.') 219 220def ProcessCommandLineArgs(parser, args): 221 page_filter.PageFilter.ProcessCommandLineArgs(parser, args) 222 223 # Page set options 224 if args.pageset_shuffle_order_file and not args.pageset_shuffle: 225 parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.') 226 227 if args.page_repeat < 1: 228 parser.error('--page-repeat must be a positive integer.') 229 if args.pageset_repeat < 1: 230 parser.error('--pageset-repeat must be a positive integer.') 231 232 233def _PrepareAndRunPage(test, page_set, expectations, finder_options, 234 browser_options, page, credentials_path, 235 possible_browser, results, state): 236 if finder_options.use_live_sites: 237 browser_options.wpr_mode = wpr_modes.WPR_OFF 238 elif browser_options.wpr_mode != wpr_modes.WPR_RECORD: 239 browser_options.wpr_mode = ( 240 wpr_modes.WPR_REPLAY 241 if page.archive_path and os.path.isfile(page.archive_path) 242 else wpr_modes.WPR_OFF) 243 244 max_attempts = test.attempts 245 attempt_num = 0 246 while attempt_num < max_attempts: 247 attempt_num += 1 248 try: 249 results.WillAttemptPageRun(attempt_num, max_attempts) 250 251 if test.RestartBrowserBeforeEachPage() or page.startup_url: 252 state.StopBrowser() 253 # If we are restarting the browser for each page customize the per page 254 # options for just the current page before starting the browser. 255 state.StartBrowserIfNeeded(test, page_set, page, possible_browser, 256 credentials_path, page.archive_path, 257 finder_options) 258 if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)): 259 logging.info('Skip test for page %s because browser is not supported.' 260 % page.url) 261 return 262 263 expectation = expectations.GetExpectationForPage(state.browser, page) 264 265 _WaitForThermalThrottlingIfNeeded(state.browser.platform) 266 267 if finder_options.profiler: 268 state.StartProfiling(page, finder_options) 269 270 try: 271 _RunPage(test, page, state, expectation, results) 272 _CheckThermalThrottling(state.browser.platform) 273 except exceptions.TabCrashException as e: 274 if test.is_multi_tab_test: 275 logging.error('Aborting multi-tab test after tab %s crashed', 276 page.url) 277 raise 278 logging.warning(str(e)) 279 state.StopBrowser() 280 281 if finder_options.profiler: 282 state.StopProfiling() 283 284 if (test.StopBrowserAfterPage(state.browser, page)): 285 state.StopBrowser() 286 287 return 288 except exceptions.BrowserGoneException as e: 289 state.StopBrowser() 290 if attempt_num == max_attempts: 291 logging.error('Aborting after too many retries') 292 raise 293 if test.is_multi_tab_test: 294 logging.error('Aborting multi-tab test after browser crashed') 295 raise 296 logging.warning(str(e)) 297 298 299@decorators.Cache 300def _UpdateCredentials(page_set): 301 # Attempt to download the credentials file. 302 if page_set.credentials_path: 303 try: 304 cloud_storage.GetIfChanged( 305 os.path.join(page_set.base_dir, page_set.credentials_path)) 306 except (cloud_storage.CredentialsError, cloud_storage.PermissionError, 307 cloud_storage.CloudStorageError) as e: 308 logging.warning('Cannot retrieve credential file %s due to cloud storage ' 309 'error %s', page_set.credentials_path, str(e)) 310 311 312@decorators.Cache 313def _UpdatePageSetArchivesIfChanged(page_set): 314 # Scan every serving directory for .sha1 files 315 # and download them from Cloud Storage. Assume all data is public. 316 all_serving_dirs = page_set.serving_dirs.copy() 317 # Add individual page dirs to all serving dirs. 318 for page in page_set: 319 if page.is_file: 320 all_serving_dirs.add(page.serving_dir) 321 # Scan all serving dirs. 322 for serving_dir in all_serving_dirs: 323 if os.path.splitdrive(serving_dir)[1] == '/': 324 raise ValueError('Trying to serve root directory from HTTP server.') 325 for dirpath, _, filenames in os.walk(serving_dir): 326 for filename in filenames: 327 path, extension = os.path.splitext( 328 os.path.join(dirpath, filename)) 329 if extension != '.sha1': 330 continue 331 cloud_storage.GetIfChanged(path, page_set.bucket) 332 333 334def Run(test, page_set, expectations, finder_options, results): 335 """Runs a given test against a given page_set with the given options.""" 336 test.ValidatePageSet(page_set) 337 338 # Create a possible_browser with the given options. 339 try: 340 possible_browser = browser_finder.FindBrowser(finder_options) 341 except browser_finder.BrowserTypeRequiredException, e: 342 sys.stderr.write(str(e) + '\n') 343 sys.exit(-1) 344 if not possible_browser: 345 sys.stderr.write( 346 'No browser found. Available browsers:\n%s\n' % 347 '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options))) 348 sys.exit(-1) 349 350 browser_options = possible_browser.finder_options.browser_options 351 browser_options.browser_type = possible_browser.browser_type 352 test.CustomizeBrowserOptions(browser_options) 353 354 if (not decorators.IsEnabled(test, possible_browser) and 355 not finder_options.run_disabled_tests): 356 logging.warning('You are trying to run a disabled test.') 357 logging.warning('Pass --also-run-disabled-tests to squelch this message.') 358 return 359 360 if possible_browser.IsRemote(): 361 possible_browser.RunRemote() 362 sys.exit(0) 363 364 # Reorder page set based on options. 365 pages = _ShuffleAndFilterPageSet(page_set, finder_options) 366 367 if not finder_options.use_live_sites: 368 _UpdateCredentials(page_set) 369 if browser_options.wpr_mode != wpr_modes.WPR_RECORD: 370 _UpdatePageSetArchivesIfChanged(page_set) 371 pages = _CheckArchives(page_set, pages, results) 372 373 # Verify credentials path. 374 credentials_path = None 375 if page_set.credentials_path: 376 credentials_path = os.path.join(os.path.dirname(page_set.file_path), 377 page_set.credentials_path) 378 if not os.path.exists(credentials_path): 379 credentials_path = None 380 381 # Set up user agent. 382 browser_options.browser_user_agent_type = page_set.user_agent_type or None 383 384 if finder_options.profiler: 385 profiler_class = profiler_finder.FindProfiler(finder_options.profiler) 386 profiler_class.CustomizeBrowserOptions(browser_options.browser_type, 387 finder_options) 388 389 for page in list(pages): 390 if not test.CanRunForPage(page): 391 results.WillRunPage(page) 392 logging.debug('Skipping test: it cannot run for %s', page.url) 393 results.AddValue(skip.SkipValue(page, 'Test cannot run')) 394 results.DidRunPage(page) 395 pages.remove(page) 396 397 if not pages: 398 return 399 400 state = _RunState() 401 pages_with_discarded_first_result = set() 402 max_failures = finder_options.max_failures # command-line gets priority 403 if max_failures is None: 404 max_failures = test.max_failures # may be None 405 406 try: 407 test.WillRunTest(finder_options) 408 for _ in xrange(finder_options.pageset_repeat): 409 for page in pages: 410 if test.IsExiting(): 411 break 412 for _ in xrange(finder_options.page_repeat): 413 results.WillRunPage(page) 414 try: 415 _PrepareAndRunPage( 416 test, page_set, expectations, finder_options, browser_options, 417 page, credentials_path, possible_browser, results, state) 418 finally: 419 discard_run = (test.discard_first_result and 420 page not in pages_with_discarded_first_result) 421 if discard_run: 422 pages_with_discarded_first_result.add(page) 423 results.DidRunPage(page, discard_run=discard_run) 424 if max_failures is not None and len(results.failures) > max_failures: 425 logging.error('Too many failures. Aborting.') 426 test.RequestExit() 427 finally: 428 test.DidRunTest(state.browser, results) 429 state.StopBrowser() 430 431 432def _ShuffleAndFilterPageSet(page_set, finder_options): 433 if finder_options.pageset_shuffle_order_file: 434 return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file) 435 pages = [page for page in page_set.pages[:] 436 if not page.disabled and page_filter.PageFilter.IsSelected(page)] 437 if finder_options.pageset_shuffle: 438 random.shuffle(pages) 439 return pages 440 441 442def _CheckArchives(page_set, pages, results): 443 """Returns a subset of pages that are local or have WPR archives. 444 445 Logs warnings if any are missing. 446 """ 447 # Warn of any problems with the entire page set. 448 if any(not p.is_local for p in pages): 449 if not page_set.archive_data_file: 450 logging.warning('The page set is missing an "archive_data_file" ' 451 'property. Skipping any live sites. To include them, ' 452 'pass the flag --use-live-sites.') 453 if not page_set.wpr_archive_info: 454 logging.warning('The archive info file is missing. ' 455 'To fix this, either add svn-internal to your ' 456 '.gclient using http://goto/read-src-internal, ' 457 'or create a new archive using record_wpr.') 458 459 # Warn of any problems with individual pages and return valid pages. 460 pages_missing_archive_path = [] 461 pages_missing_archive_data = [] 462 valid_pages = [] 463 for page in pages: 464 if not page.is_local and not page.archive_path: 465 pages_missing_archive_path.append(page) 466 elif not page.is_local and not os.path.isfile(page.archive_path): 467 pages_missing_archive_data.append(page) 468 else: 469 valid_pages.append(page) 470 if pages_missing_archive_path: 471 logging.warning('The page set archives for some pages do not exist. ' 472 'Skipping those pages. To fix this, record those pages ' 473 'using record_wpr. To ignore this warning and run ' 474 'against live sites, pass the flag --use-live-sites.') 475 if pages_missing_archive_data: 476 logging.warning('The page set archives for some pages are missing. ' 477 'Someone forgot to check them in, or they were deleted. ' 478 'Skipping those pages. To fix this, record those pages ' 479 'using record_wpr. To ignore this warning and run ' 480 'against live sites, pass the flag --use-live-sites.') 481 for page in pages_missing_archive_path + pages_missing_archive_data: 482 results.WillRunPage(page) 483 results.AddValue(failure.FailureValue.FromMessage( 484 page, 'Page set archive doesn\'t exist.')) 485 results.DidRunPage(page) 486 return valid_pages 487 488 489def _RunPage(test, page, state, expectation, results): 490 if expectation == 'skip': 491 logging.debug('Skipping test: Skip expectation for %s', page.url) 492 results.AddValue(skip.SkipValue(page, 'Skipped by test expectations')) 493 return 494 495 page_state = PageState(page, test.TabForPage(page, state.browser)) 496 497 def ProcessError(): 498 if expectation == 'fail': 499 msg = 'Expected exception while running %s' % page.url 500 else: 501 msg = 'Exception while running %s' % page.url 502 results.AddValue(failure.FailureValue(page, sys.exc_info())) 503 exception_formatter.PrintFormattedException(msg=msg) 504 505 try: 506 page_state.PreparePage(test) 507 page_state.ImplicitPageNavigation(test) 508 test.RunPage(page, page_state.tab, results) 509 util.CloseConnections(page_state.tab) 510 except page_test.TestNotSupportedOnPlatformFailure: 511 raise 512 except page_test.Failure: 513 if expectation == 'fail': 514 exception_formatter.PrintFormattedException( 515 msg='Expected failure while running %s' % page.url) 516 else: 517 exception_formatter.PrintFormattedException( 518 msg='Failure while running %s' % page.url) 519 results.AddValue(failure.FailureValue(page, sys.exc_info())) 520 except (util.TimeoutException, exceptions.LoginException, 521 exceptions.ProfilingException): 522 ProcessError() 523 except (exceptions.TabCrashException, exceptions.BrowserGoneException): 524 ProcessError() 525 # Run() catches these exceptions to relaunch the tab/browser, so re-raise. 526 raise 527 except page_action.PageActionNotSupported as e: 528 results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e)) 529 except Exception: 530 exception_formatter.PrintFormattedException( 531 msg='Unhandled exception while running %s' % page.url) 532 results.AddValue(failure.FailureValue(page, sys.exc_info())) 533 else: 534 if expectation == 'fail': 535 logging.warning('%s was expected to fail, but passed.\n', page.url) 536 finally: 537 page_state.CleanUpPage(test) 538 539 540def _WaitForThermalThrottlingIfNeeded(platform): 541 if not platform.CanMonitorThermalThrottling(): 542 return 543 thermal_throttling_retry = 0 544 while (platform.IsThermallyThrottled() and 545 thermal_throttling_retry < 3): 546 logging.warning('Thermally throttled, waiting (%d)...', 547 thermal_throttling_retry) 548 thermal_throttling_retry += 1 549 time.sleep(thermal_throttling_retry * 2) 550 551 if thermal_throttling_retry and platform.IsThermallyThrottled(): 552 logging.warning('Device is thermally throttled before running ' 553 'performance tests, results will vary.') 554 555 556def _CheckThermalThrottling(platform): 557 if not platform.CanMonitorThermalThrottling(): 558 return 559 if platform.HasBeenThermallyThrottled(): 560 logging.warning('Device has been thermally throttled during ' 561 'performance tests, results will vary.') 562