page_runner.py revision c5cede9ae108bb15f6b7a8aea21c7e1fefa2834c
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import collections 6import copy 7import glob 8import logging 9import os 10import random 11import sys 12import tempfile 13import time 14 15from telemetry import decorators 16from telemetry.core import browser_finder 17from telemetry.core import exceptions 18from telemetry.core import util 19from telemetry.core import wpr_modes 20from telemetry.core.platform.profiler import profiler_finder 21from telemetry.page import page_filter 22from telemetry.page import page_runner_repeat 23from telemetry.page import page_test 24from telemetry.page import results_options 25from telemetry.page.actions import navigate 26from telemetry.page.actions import page_action 27from telemetry.util import exception_formatter 28 29 30class _RunState(object): 31 def __init__(self): 32 self.browser = None 33 34 self._append_to_existing_wpr = False 35 self._last_archive_path = None 36 self._first_browser = True 37 self.first_page = collections.defaultdict(lambda: True) 38 self.profiler_dir = None 39 self.repeat_state = None 40 41 def StartBrowserIfNeeded(self, test, page_set, page, possible_browser, 42 credentials_path, archive_path): 43 started_browser = not self.browser 44 # Create a browser. 45 if not self.browser: 46 test.CustomizeBrowserOptionsForSinglePage(page, 47 possible_browser.finder_options) 48 self.browser = possible_browser.Create() 49 self.browser.credentials.credentials_path = credentials_path 50 51 # Set up WPR path on the new browser. 52 self.browser.SetReplayArchivePath(archive_path, 53 self._append_to_existing_wpr, 54 page_set.make_javascript_deterministic) 55 self._last_archive_path = page.archive_path 56 57 test.WillStartBrowser(self.browser) 58 self.browser.Start() 59 test.DidStartBrowser(self.browser) 60 61 if self._first_browser: 62 self._first_browser = False 63 self.browser.credentials.WarnIfMissingCredentials(page_set) 64 logging.info('OS: %s %s', 65 self.browser.platform.GetOSName(), 66 self.browser.platform.GetOSVersionName()) 67 if self.browser.supports_system_info: 68 system_info = self.browser.GetSystemInfo() 69 if system_info.model_name: 70 logging.info('Model: %s' % system_info.model_name) 71 if system_info.gpu: 72 for i, device in enumerate(system_info.gpu.devices): 73 logging.info('GPU device %d: %s', i, device) 74 if system_info.gpu.aux_attributes: 75 logging.info('GPU Attributes:') 76 for k, v in sorted(system_info.gpu.aux_attributes.iteritems()): 77 logging.info(' %-20s: %s', k, v) 78 if system_info.gpu.feature_status: 79 logging.info('Feature Status:') 80 for k, v in sorted(system_info.gpu.feature_status.iteritems()): 81 logging.info(' %-20s: %s', k, v) 82 if system_info.gpu.driver_bug_workarounds: 83 logging.info('Driver Bug Workarounds:') 84 for workaround in system_info.gpu.driver_bug_workarounds: 85 logging.info(' %s', workaround) 86 else: 87 logging.info('No GPU devices') 88 else: 89 # Set up WPR path if it changed. 90 if page.archive_path and self._last_archive_path != page.archive_path: 91 self.browser.SetReplayArchivePath( 92 page.archive_path, 93 self._append_to_existing_wpr, 94 page_set.make_javascript_deterministic) 95 self._last_archive_path = page.archive_path 96 97 if self.browser.supports_tab_control and test.close_tabs_before_run: 98 # Create a tab if there's none. 99 if len(self.browser.tabs) == 0: 100 # TODO(nduca/tonyg): Remove this line. Added as part of crbug.com/348337 101 # chasing. 102 logging.warning('Making a new tab\n') 103 self.browser.tabs.New() 104 105 # Ensure only one tab is open, unless the test is a multi-tab test. 106 if not test.is_multi_tab_test: 107 while len(self.browser.tabs) > 1: 108 self.browser.tabs[-1].Close() 109 110 # Must wait for tab to commit otherwise it can commit after the next 111 # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame() 112 # will cancel the next navigation because it's pending. This manifests as 113 # the first navigation in a PageSet freezing indefinitly because the 114 # navigation was silently cancelled when |self.browser.tabs[0]| was 115 # committed. Only do this when we just started the browser, otherwise 116 # there are cases where previous pages in a PageSet never complete 117 # loading so we'll wait forever. 118 if started_browser: 119 self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete() 120 121 def StopBrowser(self): 122 if self.browser: 123 self.browser.Close() 124 self.browser = None 125 126 # Restarting the state will also restart the wpr server. If we're 127 # recording, we need to continue adding into the same wpr archive, 128 # not overwrite it. 129 self._append_to_existing_wpr = True 130 131 def StartProfiling(self, page, finder_options): 132 if not self.profiler_dir: 133 self.profiler_dir = tempfile.mkdtemp() 134 output_file = os.path.join(self.profiler_dir, page.file_safe_name) 135 if finder_options.repeat_options.IsRepeating(): 136 output_file = _GetSequentialFileName(output_file) 137 self.browser.StartProfiling(finder_options.profiler, output_file) 138 139 def StopProfiling(self): 140 if self.browser: 141 self.browser.StopProfiling() 142 143 144class PageState(object): 145 def __init__(self, page, tab): 146 self.page = page 147 self.tab = tab 148 149 self._did_login = False 150 151 def PreparePage(self, test=None): 152 if self.page.is_file: 153 server_started = self.tab.browser.SetHTTPServerDirectories( 154 self.page.page_set.serving_dirs | set([self.page.serving_dir])) 155 if server_started and test: 156 test.DidStartHTTPServer(self.tab) 157 158 if self.page.credentials: 159 if not self.tab.browser.credentials.LoginNeeded( 160 self.tab, self.page.credentials): 161 raise page_test.Failure('Login as ' + self.page.credentials + ' failed') 162 self._did_login = True 163 164 if test: 165 if test.clear_cache_before_each_run: 166 self.tab.ClearCache(force=True) 167 168 def ImplicitPageNavigation(self, test=None): 169 """Executes the implicit navigation that occurs for every page iteration. 170 171 This function will be called once per page before any actions are executed. 172 """ 173 if test: 174 test.WillNavigateToPage(self.page, self.tab) 175 test.RunNavigateSteps(self.page, self.tab) 176 test.DidNavigateToPage(self.page, self.tab) 177 else: 178 i = navigate.NavigateAction() 179 i.RunAction(self.page, self.tab, None) 180 181 def CleanUpPage(self, test): 182 test.CleanUpAfterPage(self.page, self.tab) 183 if self.page.credentials and self._did_login: 184 self.tab.browser.credentials.LoginNoLongerNeeded( 185 self.tab, self.page.credentials) 186 187 188def AddCommandLineArgs(parser): 189 page_filter.PageFilter.AddCommandLineArgs(parser) 190 results_options.AddResultsOptions(parser) 191 192 193def ProcessCommandLineArgs(parser, args): 194 page_filter.PageFilter.ProcessCommandLineArgs(parser, args) 195 196 197def _LogStackTrace(title, browser): 198 if browser: 199 stack_trace = browser.GetStackTrace() 200 else: 201 stack_trace = 'Browser object is empty, no stack trace.' 202 stack_trace = (('\nStack Trace:\n') + 203 ('*' * 80) + 204 '\n\t' + stack_trace.replace('\n', '\n\t') + '\n' + 205 ('*' * 80)) 206 logging.warning('%s%s', title, stack_trace) 207 208 209def _PrepareAndRunPage(test, page_set, expectations, finder_options, 210 browser_options, page, credentials_path, 211 possible_browser, results, state): 212 if browser_options.wpr_mode != wpr_modes.WPR_RECORD: 213 possible_browser.finder_options.browser_options.wpr_mode = ( 214 wpr_modes.WPR_REPLAY 215 if page.archive_path and os.path.isfile(page.archive_path) 216 else wpr_modes.WPR_OFF) 217 218 tries = test.attempts 219 while tries: 220 tries -= 1 221 try: 222 results_for_current_run = copy.copy(results) 223 results_for_current_run.StartTest(page) 224 if test.RestartBrowserBeforeEachPage() or page.startup_url: 225 state.StopBrowser() 226 # If we are restarting the browser for each page customize the per page 227 # options for just the current page before starting the browser. 228 state.StartBrowserIfNeeded(test, page_set, page, possible_browser, 229 credentials_path, page.archive_path) 230 231 expectation = expectations.GetExpectationForPage(state.browser, page) 232 233 _WaitForThermalThrottlingIfNeeded(state.browser.platform) 234 235 if finder_options.profiler: 236 state.StartProfiling(page, finder_options) 237 238 try: 239 _RunPage(test, page, state, expectation, 240 results_for_current_run, finder_options) 241 _CheckThermalThrottling(state.browser.platform) 242 except exceptions.TabCrashException: 243 _LogStackTrace('Tab crashed: %s' % page.url, state.browser) 244 if test.is_multi_tab_test: 245 logging.error('Stopping multi-tab test after tab %s crashed' 246 % page.url) 247 raise 248 else: 249 state.StopBrowser() 250 251 if finder_options.profiler: 252 state.StopProfiling() 253 254 if (test.StopBrowserAfterPage(state.browser, page)): 255 state.StopBrowser() 256 257 results_for_current_run.StopTest(page) 258 259 if state.first_page[page]: 260 state.first_page[page] = False 261 if test.discard_first_result: 262 return results 263 return results_for_current_run 264 except exceptions.BrowserGoneException: 265 _LogStackTrace('Browser crashed', state.browser) 266 logging.warning('Lost connection to browser. Retrying.') 267 state.StopBrowser() 268 if not tries: 269 logging.error('Lost connection to browser 3 times. Failing.') 270 raise 271 if test.is_multi_tab_test: 272 logging.error( 273 'Lost connection to browser during multi-tab test. Failing.') 274 raise 275 276 277def Run(test, page_set, expectations, finder_options): 278 """Runs a given test against a given page_set with the given options.""" 279 results = results_options.PrepareResults(test, finder_options) 280 browser_options = finder_options.browser_options 281 282 test.ValidatePageSet(page_set) 283 284 # Create a possible_browser with the given options. 285 test.CustomizeBrowserOptions(finder_options) 286 try: 287 possible_browser = browser_finder.FindBrowser(finder_options) 288 except browser_finder.BrowserTypeRequiredException, e: 289 sys.stderr.write(str(e) + '\n') 290 sys.exit(-1) 291 if not possible_browser: 292 sys.stderr.write( 293 'No browser found. Available browsers:\n' + 294 '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)) + 295 '\n') 296 sys.exit(-1) 297 298 browser_options.browser_type = possible_browser.browser_type 299 300 if not decorators.IsEnabled( 301 test, browser_options.browser_type, possible_browser.platform): 302 return results 303 304 # Reorder page set based on options. 305 pages = _ShuffleAndFilterPageSet(page_set, finder_options) 306 307 if (not finder_options.allow_live_sites and 308 browser_options.wpr_mode != wpr_modes.WPR_RECORD): 309 pages = _CheckArchives(page_set, pages, results) 310 311 # Verify credentials path. 312 credentials_path = None 313 if page_set.credentials_path: 314 credentials_path = os.path.join(os.path.dirname(page_set.file_path), 315 page_set.credentials_path) 316 if not os.path.exists(credentials_path): 317 credentials_path = None 318 319 # Set up user agent. 320 if page_set.user_agent_type: 321 browser_options.browser_user_agent_type = page_set.user_agent_type 322 323 if finder_options.profiler: 324 profiler_class = profiler_finder.FindProfiler(finder_options.profiler) 325 profiler_class.CustomizeBrowserOptions(possible_browser.browser_type, 326 possible_browser.finder_options) 327 328 for page in list(pages): 329 if not test.CanRunForPage(page): 330 logging.debug('Skipping test: it cannot run for %s', page.url) 331 results.AddSkip(page, 'Test cannot run') 332 pages.remove(page) 333 334 if not pages: 335 return results 336 337 state = _RunState() 338 # TODO(dtu): Move results creation and results_for_current_run into RunState. 339 340 try: 341 test.WillRunTest(finder_options) 342 state.repeat_state = page_runner_repeat.PageRunnerRepeatState( 343 finder_options.repeat_options) 344 345 state.repeat_state.WillRunPageSet() 346 while state.repeat_state.ShouldRepeatPageSet() and not test.IsExiting(): 347 for page in pages: 348 state.repeat_state.WillRunPage() 349 test.WillRunPageRepeats(page) 350 while state.repeat_state.ShouldRepeatPage(): 351 results = _PrepareAndRunPage( 352 test, page_set, expectations, finder_options, browser_options, 353 page, credentials_path, possible_browser, results, state) 354 state.repeat_state.DidRunPage() 355 test.DidRunPageRepeats(page) 356 if (not test.max_failures is None and 357 len(results.failures) > test.max_failures): 358 logging.error('Too many failures. Aborting.') 359 test.RequestExit() 360 if (not test.max_errors is None and 361 len(results.errors) > test.max_errors): 362 logging.error('Too many errors. Aborting.') 363 test.RequestExit() 364 if test.IsExiting(): 365 break 366 state.repeat_state.DidRunPageSet() 367 368 test.DidRunTest(state.browser, results) 369 finally: 370 state.StopBrowser() 371 372 return results 373 374 375def _ShuffleAndFilterPageSet(page_set, finder_options): 376 if (finder_options.pageset_shuffle_order_file and 377 not finder_options.pageset_shuffle): 378 raise Exception('--pageset-shuffle-order-file requires --pageset-shuffle.') 379 380 if finder_options.pageset_shuffle_order_file: 381 return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file) 382 383 pages = [page for page in page_set.pages[:] 384 if not page.disabled and page_filter.PageFilter.IsSelected(page)] 385 386 if finder_options.pageset_shuffle: 387 random.Random().shuffle(pages) 388 389 return pages 390 391 392def _CheckArchives(page_set, pages, results): 393 """Returns a subset of pages that are local or have WPR archives. 394 395 Logs warnings if any are missing.""" 396 page_set_has_live_sites = False 397 for page in pages: 398 if not page.is_local: 399 page_set_has_live_sites = True 400 break 401 402 # Potential problems with the entire page set. 403 if page_set_has_live_sites: 404 if not page_set.archive_data_file: 405 logging.warning('The page set is missing an "archive_data_file" ' 406 'property. Skipping any live sites. To include them, ' 407 'pass the flag --allow-live-sites.') 408 if not page_set.wpr_archive_info: 409 logging.warning('The archive info file is missing. ' 410 'To fix this, either add svn-internal to your ' 411 '.gclient using http://goto/read-src-internal, ' 412 'or create a new archive using record_wpr.') 413 414 # Potential problems with individual pages. 415 pages_missing_archive_path = [] 416 pages_missing_archive_data = [] 417 418 for page in pages: 419 if page.is_local: 420 continue 421 422 if not page.archive_path: 423 pages_missing_archive_path.append(page) 424 elif not os.path.isfile(page.archive_path): 425 pages_missing_archive_data.append(page) 426 427 if pages_missing_archive_path: 428 logging.warning('The page set archives for some pages do not exist. ' 429 'Skipping those pages. To fix this, record those pages ' 430 'using record_wpr. To ignore this warning and run ' 431 'against live sites, pass the flag --allow-live-sites.') 432 if pages_missing_archive_data: 433 logging.warning('The page set archives for some pages are missing. ' 434 'Someone forgot to check them in, or they were deleted. ' 435 'Skipping those pages. To fix this, record those pages ' 436 'using record_wpr. To ignore this warning and run ' 437 'against live sites, pass the flag --allow-live-sites.') 438 439 for page in pages_missing_archive_path + pages_missing_archive_data: 440 results.StartTest(page) 441 results.AddErrorMessage(page, 'Page set archive doesn\'t exist.') 442 results.StopTest(page) 443 444 return [page for page in pages if page not in 445 pages_missing_archive_path + pages_missing_archive_data] 446 447 448def _RunPage(test, page, state, expectation, results, finder_options): 449 if expectation == 'skip': 450 logging.debug('Skipping test: Skip expectation for %s', page.url) 451 results.AddSkip(page, 'Skipped by test expectations') 452 return 453 454 logging.info('Running %s' % page.url) 455 456 page_state = PageState(page, test.TabForPage(page, state.browser)) 457 458 page_action.PageAction.ResetNextTimelineMarkerId() 459 460 def ProcessError(): 461 logging.error('%s:', page.url) 462 exception_formatter.PrintFormattedException() 463 if expectation == 'fail': 464 logging.info('Error was expected\n') 465 results.AddSuccess(page) 466 else: 467 results.AddError(page, sys.exc_info()) 468 469 try: 470 page_state.PreparePage(test) 471 if state.repeat_state.ShouldNavigate( 472 finder_options.skip_navigate_on_repeat): 473 page_state.ImplicitPageNavigation(test) 474 test.RunPage(page, page_state.tab, results) 475 util.CloseConnections(page_state.tab) 476 except page_test.TestNotSupportedOnPlatformFailure: 477 raise 478 except page_test.Failure: 479 if expectation == 'fail': 480 logging.info('%s:', page.url) 481 exception_formatter.PrintFormattedException() 482 logging.info('Failure was expected\n') 483 results.AddSuccess(page) 484 else: 485 logging.warning('%s:', page.url) 486 exception_formatter.PrintFormattedException() 487 results.AddFailure(page, sys.exc_info()) 488 except (util.TimeoutException, exceptions.LoginException, 489 exceptions.ProfilingException): 490 ProcessError() 491 except (exceptions.TabCrashException, exceptions.BrowserGoneException): 492 ProcessError() 493 # Run() catches these exceptions to relaunch the tab/browser, so re-raise. 494 raise 495 except Exception: 496 logging.warning('While running %s', page.url) 497 exception_formatter.PrintFormattedException() 498 results.AddFailure(page, sys.exc_info()) 499 else: 500 if expectation == 'fail': 501 logging.warning('%s was expected to fail, but passed.\n', page.url) 502 results.AddSuccess(page) 503 finally: 504 page_state.CleanUpPage(test) 505 506 507def _GetSequentialFileName(base_name): 508 """Returns the next sequential file name based on |base_name| and the 509 existing files.""" 510 index = 0 511 while True: 512 output_name = '%s_%03d' % (base_name, index) 513 if not glob.glob(output_name + '.*'): 514 break 515 index = index + 1 516 return output_name 517 518 519def _WaitForThermalThrottlingIfNeeded(platform): 520 if not platform.CanMonitorThermalThrottling(): 521 return 522 thermal_throttling_retry = 0 523 while (platform.IsThermallyThrottled() and 524 thermal_throttling_retry < 3): 525 logging.warning('Thermally throttled, waiting (%d)...', 526 thermal_throttling_retry) 527 thermal_throttling_retry += 1 528 time.sleep(thermal_throttling_retry * 2) 529 530 if thermal_throttling_retry and platform.IsThermallyThrottled(): 531 logging.error('Device is thermally throttled before running ' 532 'performance tests, results will vary.') 533 534 535def _CheckThermalThrottling(platform): 536 if not platform.CanMonitorThermalThrottling(): 537 return 538 if platform.HasBeenThermallyThrottled(): 539 logging.error('Device has been thermally throttled during ' 540 'performance tests, results will vary.') 541