page_runner_unittest.py revision 116680a4aac90f2aa7413d9095a592090648e557
1# Copyright 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import tempfile 8import unittest 9 10from telemetry import decorators 11from telemetry.core import browser_finder 12from telemetry.core import exceptions 13from telemetry.core import user_agent 14from telemetry.core import util 15from telemetry.page import page as page_module 16from telemetry.page import page_measurement 17from telemetry.page import page_set 18from telemetry.page import page_test 19from telemetry.page import page_runner 20from telemetry.page import test_expectations 21from telemetry.unittest import options_for_unittests 22from telemetry.value import scalar 23 24 25SIMPLE_CREDENTIALS_STRING = """ 26{ 27 "test": { 28 "username": "example", 29 "password": "asdf" 30 } 31} 32""" 33 34 35def SetUpPageRunnerArguments(options): 36 parser = options.CreateParser() 37 page_runner.AddCommandLineArgs(parser) 38 options.MergeDefaultValues(parser.get_default_values()) 39 page_runner.ProcessCommandLineArgs(parser, options) 40 41 42class StubCredentialsBackend(object): 43 def __init__(self, login_return_value): 44 self.did_get_login = False 45 self.did_get_login_no_longer_needed = False 46 self.login_return_value = login_return_value 47 48 @property 49 def credentials_type(self): 50 return 'test' 51 52 def LoginNeeded(self, *_): 53 self.did_get_login = True 54 return self.login_return_value 55 56 def LoginNoLongerNeeded(self, _): 57 self.did_get_login_no_longer_needed = True 58 59 60class PageRunnerTests(unittest.TestCase): 61 # TODO(nduca): Move the basic "test failed, test succeeded" tests from 62 # page_measurement_unittest to here. 63 64 def testHandlingOfCrashedTab(self): 65 ps = page_set.PageSet() 66 expectations = test_expectations.TestExpectations() 67 page1 = page_module.Page('chrome://crash', ps) 68 ps.pages.append(page1) 69 70 class Test(page_test.PageTest): 71 def ValidatePage(self, *args): 72 pass 73 74 options = options_for_unittests.GetCopy() 75 options.output_format = 'none' 76 SetUpPageRunnerArguments(options) 77 results = page_runner.Run(Test(), ps, expectations, options) 78 self.assertEquals(0, len(results.successes)) 79 self.assertEquals(1, len(results.failures)) 80 81 def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self): 82 ps = page_set.PageSet() 83 expectations = test_expectations.TestExpectations() 84 ps.pages.append(page_module.Page( 85 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 86 ps.pages.append(page_module.Page( 87 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 88 89 class ExpectedException(Exception): 90 pass 91 92 class Test(page_test.PageTest): 93 def __init__(self, *args): 94 super(Test, self).__init__(*args) 95 self.run_count = 0 96 def ValidatePage(self, *_): 97 old_run_count = self.run_count 98 self.run_count += 1 99 if old_run_count == 0: 100 raise ExpectedException() 101 102 options = options_for_unittests.GetCopy() 103 options.output_format = 'none' 104 test = Test() 105 SetUpPageRunnerArguments(options) 106 results = page_runner.Run(test, ps, expectations, options) 107 self.assertEquals(2, test.run_count) 108 self.assertEquals(1, len(results.successes)) 109 self.assertEquals(1, len(results.failures)) 110 111 def testHandlingOfCrashedTabWithExpectedFailure(self): 112 ps = page_set.PageSet() 113 expectations = test_expectations.TestExpectations() 114 expectations.Fail('chrome://crash') 115 page1 = page_module.Page('chrome://crash', ps) 116 ps.pages.append(page1) 117 118 class Test(page_test.PageTest): 119 def ValidatePage(self, *_): 120 pass 121 122 options = options_for_unittests.GetCopy() 123 options.output_format = 'none' 124 SetUpPageRunnerArguments(options) 125 results = page_runner.Run( 126 Test(), ps, expectations, options) 127 self.assertEquals(1, len(results.successes)) 128 self.assertEquals(0, len(results.failures)) 129 130 def testRetryOnBrowserCrash(self): 131 ps = page_set.PageSet() 132 expectations = test_expectations.TestExpectations() 133 ps.pages.append(page_module.Page( 134 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 135 136 class CrashyMeasurement(page_measurement.PageMeasurement): 137 has_crashed = False 138 def MeasurePage(self, _, tab, __): 139 if not self.has_crashed: 140 self.has_crashed = True 141 raise exceptions.BrowserGoneException(tab.browser) 142 143 options = options_for_unittests.GetCopy() 144 options.output_format = 'csv' 145 146 SetUpPageRunnerArguments(options) 147 results = page_runner.Run(CrashyMeasurement(), ps, expectations, options) 148 149 self.assertEquals(1, len(results.successes)) 150 self.assertEquals(0, len(results.failures)) 151 152 @decorators.Disabled('xp') # Flaky, http://crbug.com/390079. 153 def testDiscardFirstResult(self): 154 ps = page_set.PageSet() 155 expectations = test_expectations.TestExpectations() 156 ps.pages.append(page_module.Page( 157 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 158 ps.pages.append(page_module.Page( 159 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 160 161 class Measurement(page_measurement.PageMeasurement): 162 @property 163 def discard_first_result(self): 164 return True 165 def MeasurePage(self, *args): 166 pass 167 168 options = options_for_unittests.GetCopy() 169 options.output_format = 'none' 170 options.reset_results = None 171 options.upload_results = None 172 options.results_label = None 173 174 options.page_repeat = 1 175 options.pageset_repeat = 1 176 SetUpPageRunnerArguments(options) 177 results = page_runner.Run(Measurement(), ps, expectations, options) 178 self.assertEquals(0, len(results.successes)) 179 self.assertEquals(0, len(results.failures)) 180 181 options.page_repeat = 1 182 options.pageset_repeat = 2 183 SetUpPageRunnerArguments(options) 184 results = page_runner.Run(Measurement(), ps, expectations, options) 185 self.assertEquals(2, len(results.successes)) 186 self.assertEquals(0, len(results.failures)) 187 188 options.page_repeat = 2 189 options.pageset_repeat = 1 190 SetUpPageRunnerArguments(options) 191 results = page_runner.Run(Measurement(), ps, expectations, options) 192 self.assertEquals(2, len(results.successes)) 193 self.assertEquals(0, len(results.failures)) 194 195 options.output_format = 'html' 196 options.page_repeat = 1 197 options.pageset_repeat = 1 198 SetUpPageRunnerArguments(options) 199 results = page_runner.Run(Measurement(), ps, expectations, options) 200 self.assertEquals(0, len(results.successes)) 201 self.assertEquals(0, len(results.failures)) 202 203 @decorators.Disabled('win') 204 def testPagesetRepeat(self): 205 ps = page_set.PageSet() 206 expectations = test_expectations.TestExpectations() 207 ps.pages.append(page_module.Page( 208 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 209 ps.pages.append(page_module.Page( 210 'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir())) 211 212 class Measurement(page_measurement.PageMeasurement): 213 i = 0 214 def MeasurePage(self, page, _, results): 215 self.i += 1 216 results.AddValue(scalar.ScalarValue( 217 page, 'metric', 'unit', self.i)) 218 219 output_file = tempfile.NamedTemporaryFile(delete=False).name 220 try: 221 options = options_for_unittests.GetCopy() 222 options.output_format = 'buildbot' 223 options.output_file = output_file 224 options.reset_results = None 225 options.upload_results = None 226 options.results_label = None 227 228 options.page_repeat = 1 229 options.pageset_repeat = 2 230 SetUpPageRunnerArguments(options) 231 results = page_runner.Run(Measurement(), ps, expectations, options) 232 results.PrintSummary() 233 self.assertEquals(4, len(results.successes)) 234 self.assertEquals(0, len(results.failures)) 235 with open(output_file) as f: 236 stdout = f.read() 237 self.assertIn('RESULT metric: blank.html= [1,3] unit', stdout) 238 self.assertIn('RESULT metric: green_rect.html= [2,4] unit', stdout) 239 self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout) 240 finally: 241 results._output_stream.close() # pylint: disable=W0212 242 os.remove(output_file) 243 244 def testCredentialsWhenLoginFails(self): 245 credentials_backend = StubCredentialsBackend(login_return_value=False) 246 did_run = self.runCredentialsTest(credentials_backend) 247 assert credentials_backend.did_get_login == True 248 assert credentials_backend.did_get_login_no_longer_needed == False 249 assert did_run == False 250 251 def testCredentialsWhenLoginSucceeds(self): 252 credentials_backend = StubCredentialsBackend(login_return_value=True) 253 did_run = self.runCredentialsTest(credentials_backend) 254 assert credentials_backend.did_get_login == True 255 assert credentials_backend.did_get_login_no_longer_needed == True 256 assert did_run 257 258 def runCredentialsTest(self, credentials_backend): 259 ps = page_set.PageSet() 260 expectations = test_expectations.TestExpectations() 261 page = page_module.Page( 262 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 263 page.credentials = "test" 264 ps.pages.append(page) 265 266 did_run = [False] 267 268 try: 269 with tempfile.NamedTemporaryFile(delete=False) as f: 270 f.write(SIMPLE_CREDENTIALS_STRING) 271 ps.credentials_path = f.name 272 273 class TestThatInstallsCredentialsBackend(page_test.PageTest): 274 def __init__(self, credentials_backend): 275 super(TestThatInstallsCredentialsBackend, self).__init__() 276 self._credentials_backend = credentials_backend 277 278 def DidStartBrowser(self, browser): 279 browser.credentials.AddBackend(self._credentials_backend) 280 281 def ValidatePage(self, *_): 282 did_run[0] = True 283 284 test = TestThatInstallsCredentialsBackend(credentials_backend) 285 options = options_for_unittests.GetCopy() 286 options.output_format = 'none' 287 SetUpPageRunnerArguments(options) 288 page_runner.Run(test, ps, expectations, options) 289 finally: 290 os.remove(f.name) 291 292 return did_run[0] 293 294 def testUserAgent(self): 295 ps = page_set.PageSet() 296 expectations = test_expectations.TestExpectations() 297 page = page_module.Page( 298 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 299 ps.pages.append(page) 300 ps.user_agent_type = 'tablet' 301 302 class TestUserAgent(page_test.PageTest): 303 def ValidatePage(self, _1, tab, _2): 304 actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent') 305 expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] 306 assert actual_user_agent.strip() == expected_user_agent 307 308 # This is so we can check later that the test actually made it into this 309 # function. Previously it was timing out before even getting here, which 310 # should fail, but since it skipped all the asserts, it slipped by. 311 self.hasRun = True # pylint: disable=W0201 312 313 test = TestUserAgent() 314 options = options_for_unittests.GetCopy() 315 options.output_format = 'none' 316 SetUpPageRunnerArguments(options) 317 page_runner.Run(test, ps, expectations, options) 318 319 self.assertTrue(hasattr(test, 'hasRun') and test.hasRun) 320 321 # Ensure that page_runner forces exactly 1 tab before running a page. 322 @decorators.Enabled('has tabs') 323 def testOneTab(self): 324 ps = page_set.PageSet() 325 expectations = test_expectations.TestExpectations() 326 page = page_module.Page( 327 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 328 ps.pages.append(page) 329 330 class TestOneTab(page_test.PageTest): 331 def __init__(self): 332 super(TestOneTab, self).__init__() 333 self._browser = None 334 335 def DidStartBrowser(self, browser): 336 self._browser = browser 337 self._browser.tabs.New() 338 339 def ValidatePage(self, *_): 340 assert len(self._browser.tabs) == 1 341 342 test = TestOneTab() 343 options = options_for_unittests.GetCopy() 344 options.output_format = 'none' 345 SetUpPageRunnerArguments(options) 346 page_runner.Run(test, ps, expectations, options) 347 348 # Ensure that page_runner allows the test to customize the browser before it 349 # launches. 350 def testBrowserBeforeLaunch(self): 351 ps = page_set.PageSet() 352 expectations = test_expectations.TestExpectations() 353 page = page_module.Page( 354 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 355 ps.pages.append(page) 356 357 class TestBeforeLaunch(page_test.PageTest): 358 def __init__(self): 359 super(TestBeforeLaunch, self).__init__() 360 self._did_call_will_start = False 361 self._did_call_did_start = False 362 363 def WillStartBrowser(self, browser): 364 self._did_call_will_start = True 365 # TODO(simonjam): Test that the profile is available. 366 367 def DidStartBrowser(self, browser): 368 assert self._did_call_will_start 369 self._did_call_did_start = True 370 371 def ValidatePage(self, *_): 372 assert self._did_call_did_start 373 374 test = TestBeforeLaunch() 375 options = options_for_unittests.GetCopy() 376 options.output_format = 'none' 377 SetUpPageRunnerArguments(options) 378 page_runner.Run(test, ps, expectations, options) 379 380 def testRunPageWithStartupUrl(self): 381 ps = page_set.PageSet() 382 expectations = test_expectations.TestExpectations() 383 expectations = test_expectations.TestExpectations() 384 page = page_module.Page( 385 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 386 page.startup_url = 'about:blank' 387 ps.pages.append(page) 388 389 class Measurement(page_measurement.PageMeasurement): 390 def __init__(self): 391 super(Measurement, self).__init__() 392 self.browser_restarted = False 393 394 def CustomizeBrowserOptionsForSinglePage(self, ps, options): 395 self.browser_restarted = True 396 super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps, 397 options) 398 def MeasurePage(self, page, tab, results): 399 pass 400 401 options = options_for_unittests.GetCopy() 402 options.page_repeat = 2 403 options.output_format = 'none' 404 if not browser_finder.FindBrowser(options): 405 return 406 test = Measurement() 407 SetUpPageRunnerArguments(options) 408 page_runner.Run(test, ps, expectations, options) 409 self.assertEquals('about:blank', options.browser_options.startup_url) 410 self.assertTrue(test.browser_restarted) 411 412 # Ensure that page_runner calls cleanUp when a page run fails. 413 def testCleanUpPage(self): 414 ps = page_set.PageSet() 415 expectations = test_expectations.TestExpectations() 416 page = page_module.Page( 417 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 418 ps.pages.append(page) 419 420 class Test(page_test.PageTest): 421 def __init__(self): 422 super(Test, self).__init__() 423 self.did_call_clean_up = False 424 425 def ValidatePage(self, *_): 426 raise exceptions.IntentionalException 427 428 def CleanUpAfterPage(self, page, tab): 429 self.did_call_clean_up = True 430 431 432 test = Test() 433 options = options_for_unittests.GetCopy() 434 options.output_format = 'none' 435 SetUpPageRunnerArguments(options) 436 page_runner.Run(test, ps, expectations, options) 437 assert test.did_call_clean_up 438 439 # Ensure skipping the test if page cannot be run on the browser 440 def testPageCannotRunOnBrowser(self): 441 ps = page_set.PageSet() 442 expectations = test_expectations.TestExpectations() 443 444 class PageThatCannotRunOnBrowser(page_module.Page): 445 446 def __init__(self): 447 super(PageThatCannotRunOnBrowser, self).__init__( 448 url='file://blank.html', page_set=ps, 449 base_dir=util.GetUnittestDataDir()) 450 451 def CanRunOnBrowser(self, _): 452 return False 453 454 def ValidatePage(self, _): 455 pass 456 457 class Test(page_test.PageTest): 458 def __init__(self, *args, **kwargs): 459 super(Test, self).__init__(*args, **kwargs) 460 self.will_navigate_to_page_called = False 461 462 def ValidatePage(self, *args): 463 pass 464 465 def WillNavigateToPage(self, _1, _2): 466 self.will_navigate_to_page_called = True 467 468 test = Test() 469 options = options_for_unittests.GetCopy() 470 options.output_format = 'none' 471 SetUpPageRunnerArguments(options) 472 results = page_runner.Run(test, ps, expectations, options) 473 self.assertFalse(test.will_navigate_to_page_called) 474 self.assertEquals(0, len(results.successes)) 475 self.assertEquals(0, len(results.failures)) 476 477 def TestUseLiveSitesFlag(self, options, expect_from_archive): 478 ps = page_set.PageSet( 479 file_path=util.GetUnittestDataDir(), 480 archive_data_file='data/archive_blank.json') 481 ps.pages.append(page_module.Page( 482 'file://blank.html', ps, base_dir=ps.base_dir)) 483 expectations = test_expectations.TestExpectations() 484 485 class ArchiveTest(page_measurement.PageMeasurement): 486 def __init__(self): 487 super(ArchiveTest, self).__init__() 488 self.is_page_from_archive = False 489 self.archive_path_exist = True 490 491 def WillNavigateToPage(self, page, tab): 492 self.archive_path_exist = (page.archive_path 493 and os.path.isfile(page.archive_path)) 494 self.is_page_from_archive = ( 495 tab.browser._wpr_server is not None) # pylint: disable=W0212 496 497 def MeasurePage(self, _, __, results): 498 pass 499 500 test = ArchiveTest() 501 page_runner.Run(test, ps, expectations, options) 502 if expect_from_archive and not test.archive_path_exist: 503 logging.warning('archive path did not exist, asserting that page ' 504 'is from archive is skipped.') 505 return 506 self.assertEquals(expect_from_archive, test.is_page_from_archive) 507 508 def testUseLiveSitesFlagSet(self): 509 options = options_for_unittests.GetCopy() 510 options.output_format = 'none' 511 options.use_live_sites = True 512 SetUpPageRunnerArguments(options) 513 self.TestUseLiveSitesFlag(options, expect_from_archive=False) 514 515 def testUseLiveSitesFlagUnset(self): 516 options = options_for_unittests.GetCopy() 517 options.output_format = 'none' 518 SetUpPageRunnerArguments(options) 519 self.TestUseLiveSitesFlag(options, expect_from_archive=True) 520 521 def testMaxFailuresOptionIsRespected(self): 522 class TestPage(page_module.Page): 523 def __init__(self, *args, **kwargs): 524 super(TestPage, self).__init__(*args, **kwargs) 525 self.was_run = False 526 527 def RunNavigateSteps(self, action_runner): 528 self.was_run = True 529 raise Exception('Test exception') 530 531 class Test(page_test.PageTest): 532 def ValidatePage(self, *args): 533 pass 534 535 ps = page_set.PageSet() 536 expectations = test_expectations.TestExpectations() 537 page1 = TestPage( 538 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 539 ps.pages.append(page1) 540 page2 = TestPage( 541 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 542 ps.pages.append(page2) 543 page3 = TestPage( 544 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 545 ps.pages.append(page3) 546 page4 = TestPage( 547 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 548 ps.pages.append(page4) 549 page5 = TestPage( 550 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 551 ps.pages.append(page5) 552 553 options = options_for_unittests.GetCopy() 554 options.output_format = 'none' 555 SetUpPageRunnerArguments(options) 556 results = page_runner.Run(Test(max_failures=2), ps, expectations, options) 557 self.assertEquals(0, len(results.successes)) 558 # Runs up to max_failures+1 failing tests before stopping, since 559 # every tests after max_failures failures have been encountered 560 # may all be passing. 561 self.assertEquals(3, len(results.failures)) 562 self.assertTrue(page1.was_run) 563 self.assertTrue(page2.was_run) 564 self.assertTrue(page3.was_run) 565 self.assertFalse(page4.was_run) 566 self.assertFalse(page5.was_run) 567