page_runner_unittest.py revision 116680a4aac90f2aa7413d9095a592090648e557
146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles)# Copyright 2012 The Chromium Authors. All rights reserved. 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 4a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)import logging 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import os 72a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import tempfile 82a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import unittest 92a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 105c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liufrom telemetry import decorators 11a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from telemetry.core import browser_finder 125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from telemetry.core import exceptions 132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.core import user_agent 14a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles)from telemetry.core import util 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.page import page as page_module 16ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdochfrom telemetry.page import page_measurement 172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.page import page_set 182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.page import page_test 192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.page import page_runner 20ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdochfrom telemetry.page import test_expectations 215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from telemetry.unittest import options_for_unittests 22116680a4aac90f2aa7413d9095a592090648e557Ben Murdochfrom telemetry.value import scalar 232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 24a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)SIMPLE_CREDENTIALS_STRING = """ 262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles){ 272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) "test": { 282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) "username": "example", 292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) "password": "asdf" 302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) } 312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)} 322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)""" 33a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 34a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 35a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)def SetUpPageRunnerArguments(options): 36a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) parser = options.CreateParser() 37a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page_runner.AddCommandLineArgs(parser) 38a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) options.MergeDefaultValues(parser.get_default_values()) 39a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page_runner.ProcessCommandLineArgs(parser, options) 40a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 41a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 422a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class StubCredentialsBackend(object): 432a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def __init__(self, login_return_value): 442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.did_get_login = False 452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.did_get_login_no_longer_needed = False 462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.login_return_value = login_return_value 472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) @property 49010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def credentials_type(self): 502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return 'test' 512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 52010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def LoginNeeded(self, *_): 532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.did_get_login = True 542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return self.login_return_value 552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 56010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def LoginNoLongerNeeded(self, _): 572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.did_get_login_no_longer_needed = True 582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 59a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class PageRunnerTests(unittest.TestCase): 612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # TODO(nduca): Move the basic "test failed, test succeeded" tests from 62c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # page_measurement_unittest to here. 632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def testHandlingOfCrashedTab(self): 652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps = page_set.PageSet() 66ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch expectations = test_expectations.TestExpectations() 67010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) page1 = page_module.Page('chrome://crash', ps) 682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps.pages.append(page1) 692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) class Test(page_test.PageTest): 71010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *args): 722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) pass 732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 747d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options = options_for_unittests.GetCopy() 757d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options.output_format = 'none' 76a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 77010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) results = page_runner.Run(Test(), ps, expectations, options) 7890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) self.assertEquals(0, len(results.successes)) 79116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertEquals(1, len(results.failures)) 802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 81a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self): 82a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) ps = page_set.PageSet() 83a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) expectations = test_expectations.TestExpectations() 84a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) ps.pages.append(page_module.Page( 85a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 86a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) ps.pages.append(page_module.Page( 87a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 88a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 89a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) class ExpectedException(Exception): 90a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) pass 91a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 92a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) class Test(page_test.PageTest): 93a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) def __init__(self, *args): 94a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) super(Test, self).__init__(*args) 95a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) self.run_count = 0 96010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *_): 97a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) old_run_count = self.run_count 98a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) self.run_count += 1 99a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) if old_run_count == 0: 100a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) raise ExpectedException() 101a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 102a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) options = options_for_unittests.GetCopy() 103a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) options.output_format = 'none' 104010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) test = Test() 105a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 106a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) results = page_runner.Run(test, ps, expectations, options) 107a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) self.assertEquals(2, test.run_count) 108a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) self.assertEquals(1, len(results.successes)) 109a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) self.assertEquals(1, len(results.failures)) 110a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) 1113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) def testHandlingOfCrashedTabWithExpectedFailure(self): 1123551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) ps = page_set.PageSet() 1133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) expectations = test_expectations.TestExpectations() 1143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) expectations.Fail('chrome://crash') 1153551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) page1 = page_module.Page('chrome://crash', ps) 1163551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) ps.pages.append(page1) 1173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) class Test(page_test.PageTest): 119010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *_): 1203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) pass 1213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) options = options_for_unittests.GetCopy() 1233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) options.output_format = 'none' 124a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 125a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) results = page_runner.Run( 126010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) Test(), ps, expectations, options) 1273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) self.assertEquals(1, len(results.successes)) 1283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) self.assertEquals(0, len(results.failures)) 1293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def testRetryOnBrowserCrash(self): 1315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ps = page_set.PageSet() 1325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) expectations = test_expectations.TestExpectations() 1335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ps.pages.append(page_module.Page( 1345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 1355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) class CrashyMeasurement(page_measurement.PageMeasurement): 1375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) has_crashed = False 138cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def MeasurePage(self, _, tab, __): 1395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not self.has_crashed: 1405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.has_crashed = True 141cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) raise exceptions.BrowserGoneException(tab.browser) 1425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options = options_for_unittests.GetCopy() 1445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options.output_format = 'csv' 1455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 146a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 1475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) results = page_runner.Run(CrashyMeasurement(), ps, expectations, options) 1485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.assertEquals(1, len(results.successes)) 1505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.assertEquals(0, len(results.failures)) 1515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 152116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch @decorators.Disabled('xp') # Flaky, http://crbug.com/390079. 153c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def testDiscardFirstResult(self): 154c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) ps = page_set.PageSet() 155ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch expectations = test_expectations.TestExpectations() 156a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch ps.pages.append(page_module.Page( 1574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 158a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch ps.pages.append(page_module.Page( 1594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 160c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 161ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch class Measurement(page_measurement.PageMeasurement): 162c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) @property 163c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def discard_first_result(self): 164c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) return True 165ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch def MeasurePage(self, *args): 166c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) pass 167c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1687d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options = options_for_unittests.GetCopy() 1697d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options.output_format = 'none' 1708bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) options.reset_results = None 1718bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) options.upload_results = None 1728bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) options.results_label = None 173a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch 174a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.page_repeat = 1 175a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.pageset_repeat = 1 176a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 177ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch results = page_runner.Run(Measurement(), ps, expectations, options) 17890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) self.assertEquals(0, len(results.successes)) 17990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) self.assertEquals(0, len(results.failures)) 180c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 181a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.page_repeat = 1 182a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.pageset_repeat = 2 183a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 184ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch results = page_runner.Run(Measurement(), ps, expectations, options) 185a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch self.assertEquals(2, len(results.successes)) 186a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch self.assertEquals(0, len(results.failures)) 187a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch 188a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.page_repeat = 2 189a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.pageset_repeat = 1 190a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 191ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch results = page_runner.Run(Measurement(), ps, expectations, options) 192a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch self.assertEquals(2, len(results.successes)) 193a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch self.assertEquals(0, len(results.failures)) 194a3f7b4e666c476898878fa745f637129375cd889Ben Murdoch 195ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch options.output_format = 'html' 196a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.page_repeat = 1 197a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.pageset_repeat = 1 198a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 199ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch results = page_runner.Run(Measurement(), ps, expectations, options) 200ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch self.assertEquals(0, len(results.successes)) 201ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch self.assertEquals(0, len(results.failures)) 202ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch 2035c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu @decorators.Disabled('win') 2045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def testPagesetRepeat(self): 2055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ps = page_set.PageSet() 2065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) expectations = test_expectations.TestExpectations() 2075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ps.pages.append(page_module.Page( 2085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) 2095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ps.pages.append(page_module.Page( 2105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir())) 2115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) class Measurement(page_measurement.PageMeasurement): 2135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) i = 0 214116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch def MeasurePage(self, page, _, results): 2155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.i += 1 216116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch results.AddValue(scalar.ScalarValue( 217116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch page, 'metric', 'unit', self.i)) 2185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) output_file = tempfile.NamedTemporaryFile(delete=False).name 2205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 2215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options = options_for_unittests.GetCopy() 2225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options.output_format = 'buildbot' 2235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options.output_file = output_file 2245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options.reset_results = None 2255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options.upload_results = None 2265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) options.results_label = None 2275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 228a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.page_repeat = 1 229a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.pageset_repeat = 2 230a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 2315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) results = page_runner.Run(Measurement(), ps, expectations, options) 2325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) results.PrintSummary() 2335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.assertEquals(4, len(results.successes)) 2345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.assertEquals(0, len(results.failures)) 2355c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu with open(output_file) as f: 2365c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu stdout = f.read() 237cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.assertIn('RESULT metric: blank.html= [1,3] unit', stdout) 238cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.assertIn('RESULT metric: green_rect.html= [2,4] unit', stdout) 2395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout) 2405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) finally: 2415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) results._output_stream.close() # pylint: disable=W0212 2425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) os.remove(output_file) 2435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 24490dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) def testCredentialsWhenLoginFails(self): 2452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) credentials_backend = StubCredentialsBackend(login_return_value=False) 2467d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) did_run = self.runCredentialsTest(credentials_backend) 2472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert credentials_backend.did_get_login == True 2482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert credentials_backend.did_get_login_no_longer_needed == False 2492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert did_run == False 2502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 25190dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) def testCredentialsWhenLoginSucceeds(self): 2522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) credentials_backend = StubCredentialsBackend(login_return_value=True) 2537d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) did_run = self.runCredentialsTest(credentials_backend) 2542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert credentials_backend.did_get_login == True 2552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert credentials_backend.did_get_login_no_longer_needed == True 2562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert did_run 2572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 258010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def runCredentialsTest(self, credentials_backend): 2592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps = page_set.PageSet() 260ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch expectations = test_expectations.TestExpectations() 26190dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) page = page_module.Page( 2624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 2632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) page.credentials = "test" 2642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps.pages.append(page) 2652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) did_run = [False] 2672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 26890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) try: 26990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) with tempfile.NamedTemporaryFile(delete=False) as f: 27090dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) f.write(SIMPLE_CREDENTIALS_STRING) 27190dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) ps.credentials_path = f.name 2722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) class TestThatInstallsCredentialsBackend(page_test.PageTest): 2742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def __init__(self, credentials_backend): 275010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) super(TestThatInstallsCredentialsBackend, self).__init__() 2762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._credentials_backend = credentials_backend 2772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 278a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def DidStartBrowser(self, browser): 2792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) browser.credentials.AddBackend(self._credentials_backend) 2802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 281010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *_): 2822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) did_run[0] = True 2832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test = TestThatInstallsCredentialsBackend(credentials_backend) 2857d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options = options_for_unittests.GetCopy() 2867d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options.output_format = 'none' 287a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 288ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch page_runner.Run(test, ps, expectations, options) 28990dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) finally: 29090dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) os.remove(f.name) 2912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return did_run[0] 2932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def testUserAgent(self): 2952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps = page_set.PageSet() 296ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch expectations = test_expectations.TestExpectations() 2972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) page = page_module.Page( 2984e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 2992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps.pages.append(page) 3002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps.user_agent_type = 'tablet' 3012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) class TestUserAgent(page_test.PageTest): 303010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, _1, tab, _2): 3042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent') 3052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] 3062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert actual_user_agent.strip() == expected_user_agent 3072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # This is so we can check later that the test actually made it into this 3092a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # function. Previously it was timing out before even getting here, which 3102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # should fail, but since it skipped all the asserts, it slipped by. 3112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.hasRun = True # pylint: disable=W0201 3122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 313010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) test = TestUserAgent() 3147d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options = options_for_unittests.GetCopy() 3157d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options.output_format = 'none' 316a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 317ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch page_runner.Run(test, ps, expectations, options) 3182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.assertTrue(hasattr(test, 'hasRun') and test.hasRun) 3202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Ensure that page_runner forces exactly 1 tab before running a page. 322116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch @decorators.Enabled('has tabs') 3232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def testOneTab(self): 3242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps = page_set.PageSet() 325ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch expectations = test_expectations.TestExpectations() 3262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) page = page_module.Page( 3274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 3282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ps.pages.append(page) 3292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) class TestOneTab(page_test.PageTest): 331010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def __init__(self): 332010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) super(TestOneTab, self).__init__() 3332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._browser = None 3342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 335a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def DidStartBrowser(self, browser): 3362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._browser = browser 337116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self._browser.tabs.New() 3382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 339010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *_): 3402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) assert len(self._browser.tabs) == 1 3412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 342010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) test = TestOneTab() 3437d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options = options_for_unittests.GetCopy() 3447d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) options.output_format = 'none' 345a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 346ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch page_runner.Run(test, ps, expectations, options) 347a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 348a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # Ensure that page_runner allows the test to customize the browser before it 349a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # launches. 350a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def testBrowserBeforeLaunch(self): 351a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) ps = page_set.PageSet() 352a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) expectations = test_expectations.TestExpectations() 353a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) page = page_module.Page( 3544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 355a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) ps.pages.append(page) 356a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 357a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) class TestBeforeLaunch(page_test.PageTest): 358010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def __init__(self): 359010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) super(TestBeforeLaunch, self).__init__() 360a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) self._did_call_will_start = False 361a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) self._did_call_did_start = False 362a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 363a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def WillStartBrowser(self, browser): 364a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) self._did_call_will_start = True 365a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # TODO(simonjam): Test that the profile is available. 366a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 367a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) def DidStartBrowser(self, browser): 368a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) assert self._did_call_will_start 369a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) self._did_call_did_start = True 370a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 371010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *_): 372a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) assert self._did_call_did_start 373a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) 374010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) test = TestBeforeLaunch() 375a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options = options_for_unittests.GetCopy() 376a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) options.output_format = 'none' 377a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 378a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page_runner.Run(test, ps, expectations, options) 379a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 380a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def testRunPageWithStartupUrl(self): 381a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ps = page_set.PageSet() 382a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) expectations = test_expectations.TestExpectations() 383a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) expectations = test_expectations.TestExpectations() 384a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page = page_module.Page( 385a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 386a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page.startup_url = 'about:blank' 387a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ps.pages.append(page) 388a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 389a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) class Measurement(page_measurement.PageMeasurement): 390a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def __init__(self): 391a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) super(Measurement, self).__init__() 392a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self.browser_restarted = False 393a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 394effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch def CustomizeBrowserOptionsForSinglePage(self, ps, options): 395a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self.browser_restarted = True 396effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps, 397effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch options) 398a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def MeasurePage(self, page, tab, results): 399a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) pass 400a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 401a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) options = options_for_unittests.GetCopy() 402a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch options.page_repeat = 2 403a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) options.output_format = 'none' 404a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if not browser_finder.FindBrowser(options): 405a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return 406a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) test = Measurement() 407effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch SetUpPageRunnerArguments(options) 408a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page_runner.Run(test, ps, expectations, options) 409a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self.assertEquals('about:blank', options.browser_options.startup_url) 410a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self.assertTrue(test.browser_restarted) 411a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 412a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) # Ensure that page_runner calls cleanUp when a page run fails. 413a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def testCleanUpPage(self): 414a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ps = page_set.PageSet() 415a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) expectations = test_expectations.TestExpectations() 416a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) page = page_module.Page( 417a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 418a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ps.pages.append(page) 419a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 420a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) class Test(page_test.PageTest): 421010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def __init__(self): 422010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) super(Test, self).__init__() 423a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self.did_call_clean_up = False 424a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 425010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) def ValidatePage(self, *_): 426cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) raise exceptions.IntentionalException 427a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 428a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def CleanUpAfterPage(self, page, tab): 429a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self.did_call_clean_up = True 430a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 431a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 432010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) test = Test() 433a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) options = options_for_unittests.GetCopy() 434a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) options.output_format = 'none' 435a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) SetUpPageRunnerArguments(options) 436a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) page_runner.Run(test, ps, expectations, options) 437a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert test.did_call_clean_up 438cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 439cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Ensure skipping the test if page cannot be run on the browser 440cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def testPageCannotRunOnBrowser(self): 441cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) ps = page_set.PageSet() 442cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) expectations = test_expectations.TestExpectations() 443cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 444cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) class PageThatCannotRunOnBrowser(page_module.Page): 445cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 446cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def __init__(self): 447cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) super(PageThatCannotRunOnBrowser, self).__init__( 448cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) url='file://blank.html', page_set=ps, 449cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) base_dir=util.GetUnittestDataDir()) 450cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 451cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def CanRunOnBrowser(self, _): 452cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return False 453cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 454cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def ValidatePage(self, _): 455cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) pass 456cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 457cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) class Test(page_test.PageTest): 458cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def __init__(self, *args, **kwargs): 459cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) super(Test, self).__init__(*args, **kwargs) 460cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.will_navigate_to_page_called = False 461cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 462cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def ValidatePage(self, *args): 463cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) pass 464cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 465cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def WillNavigateToPage(self, _1, _2): 466cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.will_navigate_to_page_called = True 467cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 468cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) test = Test() 469cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) options = options_for_unittests.GetCopy() 470cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) options.output_format = 'none' 471cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) SetUpPageRunnerArguments(options) 472cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) results = page_runner.Run(test, ps, expectations, options) 473cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.assertFalse(test.will_navigate_to_page_called) 474cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.assertEquals(0, len(results.successes)) 475cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.assertEquals(0, len(results.failures)) 476f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 477f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) def TestUseLiveSitesFlag(self, options, expect_from_archive): 478f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) ps = page_set.PageSet( 479f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) file_path=util.GetUnittestDataDir(), 480f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) archive_data_file='data/archive_blank.json') 481f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) ps.pages.append(page_module.Page( 482f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 'file://blank.html', ps, base_dir=ps.base_dir)) 483f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) expectations = test_expectations.TestExpectations() 484f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 485f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) class ArchiveTest(page_measurement.PageMeasurement): 486f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) def __init__(self): 487f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) super(ArchiveTest, self).__init__() 488f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.is_page_from_archive = False 489f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.archive_path_exist = True 490f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 491f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) def WillNavigateToPage(self, page, tab): 492f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.archive_path_exist = (page.archive_path 493f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) and os.path.isfile(page.archive_path)) 494f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.is_page_from_archive = ( 495f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) tab.browser._wpr_server is not None) # pylint: disable=W0212 496f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 497f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) def MeasurePage(self, _, __, results): 498f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) pass 499f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 500f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) test = ArchiveTest() 501f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) page_runner.Run(test, ps, expectations, options) 502f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if expect_from_archive and not test.archive_path_exist: 503f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) logging.warning('archive path did not exist, asserting that page ' 504f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 'is from archive is skipped.') 505f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return 506f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.assertEquals(expect_from_archive, test.is_page_from_archive) 507f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 508f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) def testUseLiveSitesFlagSet(self): 509f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) options = options_for_unittests.GetCopy() 510f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) options.output_format = 'none' 511f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) options.use_live_sites = True 512f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) SetUpPageRunnerArguments(options) 513f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.TestUseLiveSitesFlag(options, expect_from_archive=False) 514f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 515f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) def testUseLiveSitesFlagUnset(self): 516f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) options = options_for_unittests.GetCopy() 517f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) options.output_format = 'none' 518f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) SetUpPageRunnerArguments(options) 519f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) self.TestUseLiveSitesFlag(options, expect_from_archive=True) 520116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 521116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch def testMaxFailuresOptionIsRespected(self): 522116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch class TestPage(page_module.Page): 523116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch def __init__(self, *args, **kwargs): 524116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch super(TestPage, self).__init__(*args, **kwargs) 525116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.was_run = False 526116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 527116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch def RunNavigateSteps(self, action_runner): 528116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.was_run = True 529116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch raise Exception('Test exception') 530116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 531116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch class Test(page_test.PageTest): 532116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch def ValidatePage(self, *args): 533116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch pass 534116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 535116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ps = page_set.PageSet() 536116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch expectations = test_expectations.TestExpectations() 537116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch page1 = TestPage( 538116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 539116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ps.pages.append(page1) 540116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch page2 = TestPage( 541116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 542116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ps.pages.append(page2) 543116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch page3 = TestPage( 544116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 545116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ps.pages.append(page3) 546116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch page4 = TestPage( 547116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 548116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ps.pages.append(page4) 549116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch page5 = TestPage( 550116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) 551116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ps.pages.append(page5) 552116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 553116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch options = options_for_unittests.GetCopy() 554116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch options.output_format = 'none' 555116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch SetUpPageRunnerArguments(options) 556116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch results = page_runner.Run(Test(max_failures=2), ps, expectations, options) 557116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertEquals(0, len(results.successes)) 558116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # Runs up to max_failures+1 failing tests before stopping, since 559116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # every tests after max_failures failures have been encountered 560116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # may all be passing. 561116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertEquals(3, len(results.failures)) 562116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertTrue(page1.was_run) 563116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertTrue(page2.was_run) 564116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertTrue(page3.was_run) 565116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertFalse(page4.was_run) 566116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.assertFalse(page5.was_run) 567