page_runner_unittest.py revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import tempfile 8import unittest 9 10from telemetry.core import browser_finder 11from telemetry.core import exceptions 12from telemetry.core import user_agent 13from telemetry.core import util 14from telemetry.page import page as page_module 15from telemetry.page import page_measurement 16from telemetry.page import page_set 17from telemetry.page import page_test 18from telemetry.page import page_runner 19from telemetry.page import test_expectations 20from telemetry.unittest import options_for_unittests 21 22 23SIMPLE_CREDENTIALS_STRING = """ 24{ 25 "test": { 26 "username": "example", 27 "password": "asdf" 28 } 29} 30""" 31 32 33def SetUpPageRunnerArguments(options): 34 parser = options.CreateParser() 35 page_runner.AddCommandLineArgs(parser) 36 options.MergeDefaultValues(parser.get_default_values()) 37 page_runner.ProcessCommandLineArgs(parser, options) 38 39 40class StubCredentialsBackend(object): 41 def __init__(self, login_return_value): 42 self.did_get_login = False 43 self.did_get_login_no_longer_needed = False 44 self.login_return_value = login_return_value 45 46 @property 47 def credentials_type(self): # pylint: disable=R0201 48 return 'test' 49 50 def LoginNeeded(self, tab, config): # pylint: disable=W0613 51 self.did_get_login = True 52 return self.login_return_value 53 54 def LoginNoLongerNeeded(self, tab): # pylint: disable=W0613 55 self.did_get_login_no_longer_needed = True 56 57 58class PageRunnerTests(unittest.TestCase): 59 # TODO(nduca): Move the basic "test failed, test succeeded" tests from 60 # page_measurement_unittest to here. 61 62 def testHandlingOfCrashedTab(self): 63 ps = page_set.PageSet() 64 expectations = test_expectations.TestExpectations() 65 page1 = page_module.Page('chrome://crash', ps) 66 ps.pages.append(page1) 67 68 class Test(page_test.PageTest): 69 def RunTest(self, *args): 70 pass 71 72 options = options_for_unittests.GetCopy() 73 options.output_format = 'none' 74 SetUpPageRunnerArguments(options) 75 results = page_runner.Run(Test('RunTest'), ps, expectations, options) 76 self.assertEquals(0, len(results.successes)) 77 self.assertEquals(0, len(results.failures)) 78 self.assertEquals(1, len(results.errors)) 79 80 def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self): 81 ps = page_set.PageSet() 82 expectations = test_expectations.TestExpectations() 83 ps.pages.append(page_module.Page( 84 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 85 ps.pages.append(page_module.Page( 86 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 87 88 class ExpectedException(Exception): 89 pass 90 91 class Test(page_test.PageTest): 92 def __init__(self, *args): 93 super(Test, self).__init__(*args) 94 self.run_count = 0 95 def RunTest(self, *_): 96 old_run_count = self.run_count 97 self.run_count += 1 98 if old_run_count == 0: 99 raise ExpectedException() 100 101 options = options_for_unittests.GetCopy() 102 options.output_format = 'none' 103 test = Test('RunTest') 104 SetUpPageRunnerArguments(options) 105 results = page_runner.Run(test, ps, expectations, options) 106 self.assertEquals(2, test.run_count) 107 self.assertEquals(1, len(results.successes)) 108 self.assertEquals(1, len(results.failures)) 109 110 def testHandlingOfCrashedTabWithExpectedFailure(self): 111 ps = page_set.PageSet() 112 expectations = test_expectations.TestExpectations() 113 expectations.Fail('chrome://crash') 114 page1 = page_module.Page('chrome://crash', ps) 115 ps.pages.append(page1) 116 117 class Test(page_test.PageTest): 118 def RunTest(self, *_): 119 pass 120 121 options = options_for_unittests.GetCopy() 122 options.output_format = 'none' 123 SetUpPageRunnerArguments(options) 124 results = page_runner.Run( 125 Test('RunTest'), ps, expectations, options) 126 self.assertEquals(1, len(results.successes)) 127 self.assertEquals(0, len(results.failures)) 128 self.assertEquals(0, len(results.errors)) 129 130 def testRetryOnBrowserCrash(self): 131 ps = page_set.PageSet() 132 expectations = test_expectations.TestExpectations() 133 ps.pages.append(page_module.Page( 134 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 135 136 class CrashyMeasurement(page_measurement.PageMeasurement): 137 has_crashed = False 138 def MeasurePage(self, *_): 139 if not self.has_crashed: 140 self.has_crashed = True 141 raise exceptions.BrowserGoneException() 142 143 options = options_for_unittests.GetCopy() 144 options.output_format = 'csv' 145 146 SetUpPageRunnerArguments(options) 147 results = page_runner.Run(CrashyMeasurement(), ps, expectations, options) 148 149 self.assertEquals(1, len(results.successes)) 150 self.assertEquals(0, len(results.failures)) 151 self.assertEquals(0, len(results.errors)) 152 153 def testDiscardFirstResult(self): 154 ps = page_set.PageSet() 155 expectations = test_expectations.TestExpectations() 156 ps.pages.append(page_module.Page( 157 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 158 ps.pages.append(page_module.Page( 159 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 160 161 class Measurement(page_measurement.PageMeasurement): 162 @property 163 def discard_first_result(self): 164 return True 165 def MeasurePage(self, *args): 166 pass 167 168 options = options_for_unittests.GetCopy() 169 options.output_format = 'none' 170 options.reset_results = None 171 options.upload_results = None 172 options.results_label = None 173 174 options.repeat_options.page_repeat_iters = 1 175 options.repeat_options.pageset_repeat_iters = 1 176 SetUpPageRunnerArguments(options) 177 results = page_runner.Run(Measurement(), ps, expectations, options) 178 self.assertEquals(0, len(results.successes)) 179 self.assertEquals(0, len(results.failures)) 180 181 options.repeat_options.page_repeat_iters = 1 182 options.repeat_options.pageset_repeat_iters = 2 183 SetUpPageRunnerArguments(options) 184 results = page_runner.Run(Measurement(), ps, expectations, options) 185 self.assertEquals(2, len(results.successes)) 186 self.assertEquals(0, len(results.failures)) 187 188 options.repeat_options.page_repeat_iters = 2 189 options.repeat_options.pageset_repeat_iters = 1 190 SetUpPageRunnerArguments(options) 191 results = page_runner.Run(Measurement(), ps, expectations, options) 192 self.assertEquals(2, len(results.successes)) 193 self.assertEquals(0, len(results.failures)) 194 195 options.output_format = 'html' 196 options.repeat_options.page_repeat_iters = 1 197 options.repeat_options.pageset_repeat_iters = 1 198 SetUpPageRunnerArguments(options) 199 results = page_runner.Run(Measurement(), ps, expectations, options) 200 self.assertEquals(0, len(results.successes)) 201 self.assertEquals(0, len(results.failures)) 202 203 def testPagesetRepeat(self): 204 ps = page_set.PageSet() 205 expectations = test_expectations.TestExpectations() 206 ps.pages.append(page_module.Page( 207 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 208 ps.pages.append(page_module.Page( 209 'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir())) 210 211 class Measurement(page_measurement.PageMeasurement): 212 i = 0 213 def MeasurePage(self, _, __, results): 214 self.i += 1 215 results.Add('metric', 'unit', self.i) 216 217 output_file = tempfile.NamedTemporaryFile(delete=False).name 218 try: 219 options = options_for_unittests.GetCopy() 220 options.output_format = 'buildbot' 221 options.output_file = output_file 222 options.reset_results = None 223 options.upload_results = None 224 options.results_label = None 225 226 options.repeat_options.page_repeat_iters = 1 227 options.repeat_options.pageset_repeat_iters = 2 228 SetUpPageRunnerArguments(options) 229 results = page_runner.Run(Measurement(), ps, expectations, options) 230 results.PrintSummary() 231 self.assertEquals(4, len(results.successes)) 232 self.assertEquals(0, len(results.failures)) 233 stdout = open(output_file).read() 234 self.assertIn('RESULT metric_by_url: blank.html= [1,3] unit', stdout) 235 self.assertIn('RESULT metric_by_url: green_rect.html= [2,4] unit', stdout) 236 self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout) 237 finally: 238 results._output_stream.close() # pylint: disable=W0212 239 os.remove(output_file) 240 241 def testCredentialsWhenLoginFails(self): 242 credentials_backend = StubCredentialsBackend(login_return_value=False) 243 did_run = self.runCredentialsTest(credentials_backend) 244 assert credentials_backend.did_get_login == True 245 assert credentials_backend.did_get_login_no_longer_needed == False 246 assert did_run == False 247 248 def testCredentialsWhenLoginSucceeds(self): 249 credentials_backend = StubCredentialsBackend(login_return_value=True) 250 did_run = self.runCredentialsTest(credentials_backend) 251 assert credentials_backend.did_get_login == True 252 assert credentials_backend.did_get_login_no_longer_needed == True 253 assert did_run 254 255 def runCredentialsTest(self, # pylint: disable=R0201 256 credentials_backend): 257 ps = page_set.PageSet() 258 expectations = test_expectations.TestExpectations() 259 page = page_module.Page( 260 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 261 page.credentials = "test" 262 ps.pages.append(page) 263 264 did_run = [False] 265 266 try: 267 with tempfile.NamedTemporaryFile(delete=False) as f: 268 f.write(SIMPLE_CREDENTIALS_STRING) 269 ps.credentials_path = f.name 270 271 class TestThatInstallsCredentialsBackend(page_test.PageTest): 272 def __init__(self, credentials_backend): 273 super(TestThatInstallsCredentialsBackend, self).__init__('RunTest') 274 self._credentials_backend = credentials_backend 275 276 def DidStartBrowser(self, browser): 277 browser.credentials.AddBackend(self._credentials_backend) 278 279 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 280 did_run[0] = True 281 282 test = TestThatInstallsCredentialsBackend(credentials_backend) 283 options = options_for_unittests.GetCopy() 284 options.output_format = 'none' 285 SetUpPageRunnerArguments(options) 286 page_runner.Run(test, ps, expectations, options) 287 finally: 288 os.remove(f.name) 289 290 return did_run[0] 291 292 def testUserAgent(self): 293 ps = page_set.PageSet() 294 expectations = test_expectations.TestExpectations() 295 page = page_module.Page( 296 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 297 ps.pages.append(page) 298 ps.user_agent_type = 'tablet' 299 300 class TestUserAgent(page_test.PageTest): 301 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 302 actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent') 303 expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] 304 assert actual_user_agent.strip() == expected_user_agent 305 306 # This is so we can check later that the test actually made it into this 307 # function. Previously it was timing out before even getting here, which 308 # should fail, but since it skipped all the asserts, it slipped by. 309 self.hasRun = True # pylint: disable=W0201 310 311 test = TestUserAgent('RunTest') 312 options = options_for_unittests.GetCopy() 313 options.output_format = 'none' 314 SetUpPageRunnerArguments(options) 315 page_runner.Run(test, ps, expectations, options) 316 317 self.assertTrue(hasattr(test, 'hasRun') and test.hasRun) 318 319 # Ensure that page_runner forces exactly 1 tab before running a page. 320 def testOneTab(self): 321 ps = page_set.PageSet() 322 expectations = test_expectations.TestExpectations() 323 page = page_module.Page( 324 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 325 ps.pages.append(page) 326 327 class TestOneTab(page_test.PageTest): 328 def __init__(self, 329 test_method_name, 330 action_name_to_run='', 331 needs_browser_restart_after_each_page=False): 332 super(TestOneTab, self).__init__(test_method_name, action_name_to_run, 333 needs_browser_restart_after_each_page) 334 self._browser = None 335 336 def DidStartBrowser(self, browser): 337 self._browser = browser 338 if self._browser.supports_tab_control: 339 self._browser.tabs.New() 340 341 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 342 if not self._browser.supports_tab_control: 343 logging.warning('Browser does not support tab control, skipping test') 344 return 345 assert len(self._browser.tabs) == 1 346 347 test = TestOneTab('RunTest') 348 options = options_for_unittests.GetCopy() 349 options.output_format = 'none' 350 SetUpPageRunnerArguments(options) 351 page_runner.Run(test, ps, expectations, options) 352 353 # Ensure that page_runner allows the test to customize the browser before it 354 # launches. 355 def testBrowserBeforeLaunch(self): 356 ps = page_set.PageSet() 357 expectations = test_expectations.TestExpectations() 358 page = page_module.Page( 359 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 360 ps.pages.append(page) 361 362 class TestBeforeLaunch(page_test.PageTest): 363 def __init__(self, 364 test_method_name, 365 action_name_to_run=''): 366 super(TestBeforeLaunch, self).__init__( 367 test_method_name, action_name_to_run, False) 368 self._did_call_will_start = False 369 self._did_call_did_start = False 370 371 def WillStartBrowser(self, browser): 372 self._did_call_will_start = True 373 # TODO(simonjam): Test that the profile is available. 374 375 def DidStartBrowser(self, browser): 376 assert self._did_call_will_start 377 self._did_call_did_start = True 378 379 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 380 assert self._did_call_did_start 381 382 test = TestBeforeLaunch('RunTest') 383 options = options_for_unittests.GetCopy() 384 options.output_format = 'none' 385 SetUpPageRunnerArguments(options) 386 page_runner.Run(test, ps, expectations, options) 387 388 def testRunPageWithStartupUrl(self): 389 ps = page_set.PageSet() 390 expectations = test_expectations.TestExpectations() 391 expectations = test_expectations.TestExpectations() 392 page = page_module.Page( 393 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 394 page.startup_url = 'about:blank' 395 ps.pages.append(page) 396 397 class Measurement(page_measurement.PageMeasurement): 398 def __init__(self): 399 super(Measurement, self).__init__() 400 self.browser_restarted = False 401 402 def CustomizeBrowserOptionsForPageSet(self, ps, options): 403 self.browser_restarted = True 404 super(Measurement, self).CustomizeBrowserOptionsForPageSet(ps, 405 options) 406 def MeasurePage(self, page, tab, results): 407 pass 408 409 options = options_for_unittests.GetCopy() 410 options.repeat_options.page_repeat_iters = 2 411 options.output_format = 'none' 412 if not browser_finder.FindBrowser(options): 413 return 414 test = Measurement() 415 page_runner.Run(test, ps, expectations, options) 416 self.assertEquals('about:blank', options.browser_options.startup_url) 417 self.assertTrue(test.browser_restarted) 418 419 # Ensure that page_runner calls cleanUp when a page run fails. 420 def testCleanUpPage(self): 421 ps = page_set.PageSet() 422 expectations = test_expectations.TestExpectations() 423 page = page_module.Page( 424 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 425 ps.pages.append(page) 426 427 class Test(page_test.PageTest): 428 def __init__(self, 429 test_method_name, 430 action_name_to_run=''): 431 super(Test, self).__init__( 432 test_method_name, action_name_to_run, False) 433 self.did_call_clean_up = False 434 435 def RunTest(self, _, _2, _3): 436 raise Exception('Intentional failure') 437 438 def CleanUpAfterPage(self, page, tab): 439 self.did_call_clean_up = True 440 441 442 test = Test('RunTest') 443 options = options_for_unittests.GetCopy() 444 options.output_format = 'none' 445 SetUpPageRunnerArguments(options) 446 page_runner.Run(test, ps, expectations, options) 447 assert test.did_call_clean_up 448