page_runner_unittest.py revision 8bcbed890bc3ce4d7a057a8f32cab53fa534672e
19f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson# Copyright (c) 2012 The Chromium Authors. All rights reserved. 29f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson# Use of this source code is governed by a BSD-style license that can be 39f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson# found in the LICENSE file. 49f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonimport logging 59f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonimport os 69f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonimport tempfile 79f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonimport unittest 89f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 99f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.core import user_agent 109f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.core import util 119f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.page import page as page_module 129f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.page import page_measurement 139f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.page import page_set 149f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.page import page_test 159f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.page import page_runner 169f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.unittest import options_for_unittests 179f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonfrom telemetry.page import test_expectations 189f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 199f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse WilsonSIMPLE_CREDENTIALS_STRING = """ 209f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson{ 219f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson "test": { 229f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson "username": "example", 239f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson "password": "asdf" 249f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson } 259f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson} 269f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson""" 279f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonclass StubCredentialsBackend(object): 289f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def __init__(self, login_return_value): 299f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.did_get_login = False 309f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.did_get_login_no_longer_needed = False 319f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.login_return_value = login_return_value 329f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 339f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson @property 349f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def credentials_type(self): # pylint: disable=R0201 359f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson return 'test' 369f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 379f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def LoginNeeded(self, tab, config): # pylint: disable=W0613 389f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.did_get_login = True 399f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson return self.login_return_value 409f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 419f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def LoginNoLongerNeeded(self, tab): # pylint: disable=W0613 429f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.did_get_login_no_longer_needed = True 439f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 449f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilsonclass PageRunnerTests(unittest.TestCase): 459f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson # TODO(nduca): Move the basic "test failed, test succeeded" tests from 469f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson # page_measurement_unittest to here. 479f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 489f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def testHandlingOfCrashedTab(self): 499f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson ps = page_set.PageSet() 509f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson expectations = test_expectations.TestExpectations() 519f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson page1 = page_module.Page('chrome://crash', ps) 529f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson ps.pages.append(page1) 539f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 549f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson class Test(page_test.PageTest): 559f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def RunTest(self, *args): 569f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson pass 579f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 589f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson options = options_for_unittests.GetCopy() 599f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson options.output_format = 'none' 609f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson results = page_runner.Run(Test('RunTest'), ps, expectations, options) 619f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.assertEquals(0, len(results.successes)) 629f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.assertEquals(0, len(results.failures)) 639f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson self.assertEquals(1, len(results.errors)) 649f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 659f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson def testHandlingOfCrashedTabWithExpectedFailure(self): 669f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson ps = page_set.PageSet() 679f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson expectations = test_expectations.TestExpectations() 689f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson expectations.Fail('chrome://crash') 699f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson page1 = page_module.Page('chrome://crash', ps) 709f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson ps.pages.append(page1) 719f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson 729f8118474e9513f7a5b7d2a05e4a0fb15d1a6569Jesse Wilson class Test(page_test.PageTest): 73 def RunTest(self, *args): 74 pass 75 76 options = options_for_unittests.GetCopy() 77 options.output_format = 'none' 78 results = page_runner.Run(Test('RunTest'), ps, expectations, options) 79 self.assertEquals(1, len(results.successes)) 80 self.assertEquals(0, len(results.failures)) 81 self.assertEquals(0, len(results.errors)) 82 83 def testDiscardFirstResult(self): 84 ps = page_set.PageSet() 85 expectations = test_expectations.TestExpectations() 86 ps.pages.append(page_module.Page( 87 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 88 ps.pages.append(page_module.Page( 89 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 90 91 class Measurement(page_measurement.PageMeasurement): 92 @property 93 def discard_first_result(self): 94 return True 95 def MeasurePage(self, *args): 96 pass 97 98 options = options_for_unittests.GetCopy() 99 options.output_format = 'none' 100 options.reset_results = None 101 options.upload_results = None 102 options.results_label = None 103 104 options.repeat_options.page_repeat_iters = 1 105 options.repeat_options.pageset_repeat_iters = 1 106 results = page_runner.Run(Measurement(), ps, expectations, options) 107 self.assertEquals(0, len(results.successes)) 108 self.assertEquals(0, len(results.failures)) 109 110 options.repeat_options.page_repeat_iters = 1 111 options.repeat_options.pageset_repeat_iters = 2 112 results = page_runner.Run(Measurement(), ps, expectations, options) 113 self.assertEquals(2, len(results.successes)) 114 self.assertEquals(0, len(results.failures)) 115 116 options.repeat_options.page_repeat_iters = 2 117 options.repeat_options.pageset_repeat_iters = 1 118 results = page_runner.Run(Measurement(), ps, expectations, options) 119 self.assertEquals(2, len(results.successes)) 120 self.assertEquals(0, len(results.failures)) 121 122 options.output_format = 'html' 123 options.repeat_options.page_repeat_iters = 1 124 options.repeat_options.pageset_repeat_iters = 1 125 results = page_runner.Run(Measurement(), ps, expectations, options) 126 self.assertEquals(0, len(results.successes)) 127 self.assertEquals(0, len(results.failures)) 128 129 def testCredentialsWhenLoginFails(self): 130 credentials_backend = StubCredentialsBackend(login_return_value=False) 131 did_run = self.runCredentialsTest(credentials_backend) 132 assert credentials_backend.did_get_login == True 133 assert credentials_backend.did_get_login_no_longer_needed == False 134 assert did_run == False 135 136 def testCredentialsWhenLoginSucceeds(self): 137 credentials_backend = StubCredentialsBackend(login_return_value=True) 138 did_run = self.runCredentialsTest(credentials_backend) 139 assert credentials_backend.did_get_login == True 140 assert credentials_backend.did_get_login_no_longer_needed == True 141 assert did_run 142 143 def runCredentialsTest(self, # pylint: disable=R0201 144 credentials_backend): 145 ps = page_set.PageSet() 146 expectations = test_expectations.TestExpectations() 147 page = page_module.Page( 148 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 149 page.credentials = "test" 150 ps.pages.append(page) 151 152 did_run = [False] 153 154 try: 155 with tempfile.NamedTemporaryFile(delete=False) as f: 156 f.write(SIMPLE_CREDENTIALS_STRING) 157 ps.credentials_path = f.name 158 159 class TestThatInstallsCredentialsBackend(page_test.PageTest): 160 def __init__(self, credentials_backend): 161 super(TestThatInstallsCredentialsBackend, self).__init__('RunTest') 162 self._credentials_backend = credentials_backend 163 164 def DidStartBrowser(self, browser): 165 browser.credentials.AddBackend(self._credentials_backend) 166 167 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 168 did_run[0] = True 169 170 test = TestThatInstallsCredentialsBackend(credentials_backend) 171 options = options_for_unittests.GetCopy() 172 options.output_format = 'none' 173 page_runner.Run(test, ps, expectations, options) 174 finally: 175 os.remove(f.name) 176 177 return did_run[0] 178 179 def testUserAgent(self): 180 ps = page_set.PageSet() 181 expectations = test_expectations.TestExpectations() 182 page = page_module.Page( 183 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 184 ps.pages.append(page) 185 ps.user_agent_type = 'tablet' 186 187 class TestUserAgent(page_test.PageTest): 188 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 189 actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent') 190 expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] 191 assert actual_user_agent.strip() == expected_user_agent 192 193 # This is so we can check later that the test actually made it into this 194 # function. Previously it was timing out before even getting here, which 195 # should fail, but since it skipped all the asserts, it slipped by. 196 self.hasRun = True # pylint: disable=W0201 197 198 test = TestUserAgent('RunTest') 199 options = options_for_unittests.GetCopy() 200 options.output_format = 'none' 201 page_runner.Run(test, ps, expectations, options) 202 203 self.assertTrue(hasattr(test, 'hasRun') and test.hasRun) 204 205 # Ensure that page_runner forces exactly 1 tab before running a page. 206 def testOneTab(self): 207 ps = page_set.PageSet() 208 expectations = test_expectations.TestExpectations() 209 page = page_module.Page( 210 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 211 ps.pages.append(page) 212 213 class TestOneTab(page_test.PageTest): 214 def __init__(self, 215 test_method_name, 216 action_name_to_run='', 217 needs_browser_restart_after_each_run=False): 218 super(TestOneTab, self).__init__(test_method_name, action_name_to_run, 219 needs_browser_restart_after_each_run) 220 self._browser = None 221 222 def DidStartBrowser(self, browser): 223 self._browser = browser 224 if self._browser.supports_tab_control: 225 self._browser.tabs.New() 226 227 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 228 if not self._browser.supports_tab_control: 229 logging.warning('Browser does not support tab control, skipping test') 230 return 231 assert len(self._browser.tabs) == 1 232 233 test = TestOneTab('RunTest') 234 options = options_for_unittests.GetCopy() 235 options.output_format = 'none' 236 page_runner.Run(test, ps, expectations, options) 237 238 # Ensure that page_runner allows the test to customize the browser before it 239 # launches. 240 def testBrowserBeforeLaunch(self): 241 ps = page_set.PageSet() 242 expectations = test_expectations.TestExpectations() 243 page = page_module.Page( 244 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 245 ps.pages.append(page) 246 247 class TestBeforeLaunch(page_test.PageTest): 248 def __init__(self, 249 test_method_name, 250 action_name_to_run=''): 251 super(TestBeforeLaunch, self).__init__( 252 test_method_name, action_name_to_run, False) 253 self._did_call_will_start = False 254 self._did_call_did_start = False 255 256 def WillStartBrowser(self, browser): 257 self._did_call_will_start = True 258 # TODO(simonjam): Test that the profile is available. 259 260 def DidStartBrowser(self, browser): 261 assert self._did_call_will_start 262 self._did_call_did_start = True 263 264 def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201 265 assert self._did_call_did_start 266 267 test = TestBeforeLaunch('RunTest') 268 options = options_for_unittests.GetCopy() 269 options.output_format = 'none' 270 page_runner.Run(test, ps, expectations, options) 271