page_runner_unittest.py revision 010d83a9304c5a91596085d917d248abff47903a 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import tempfile
8import unittest
9
10from telemetry import decorators
11from telemetry.core import browser_finder
12from telemetry.core import exceptions
13from telemetry.core import user_agent
14from telemetry.core import util
15from telemetry.page import page as page_module
16from telemetry.page import page_measurement
17from telemetry.page import page_set
18from telemetry.page import page_test
19from telemetry.page import page_runner
20from telemetry.page import test_expectations
21from telemetry.unittest import options_for_unittests
22
23
24SIMPLE_CREDENTIALS_STRING = """
25{
26  "test": {
27    "username": "example",
28    "password": "asdf"
29  }
30}
31"""
32
33
34def SetUpPageRunnerArguments(options):
35  parser = options.CreateParser()
36  page_runner.AddCommandLineArgs(parser)
37  options.MergeDefaultValues(parser.get_default_values())
38  page_runner.ProcessCommandLineArgs(parser, options)
39
40
41class StubCredentialsBackend(object):
42  def __init__(self, login_return_value):
43    self.did_get_login = False
44    self.did_get_login_no_longer_needed = False
45    self.login_return_value = login_return_value
46
47  @property
48  def credentials_type(self):
49    return 'test'
50
51  def LoginNeeded(self, *_):
52    self.did_get_login = True
53    return self.login_return_value
54
55  def LoginNoLongerNeeded(self, _):
56    self.did_get_login_no_longer_needed = True
57
58
59class PageRunnerTests(unittest.TestCase):
60  # TODO(nduca): Move the basic "test failed, test succeeded" tests from
61  # page_measurement_unittest to here.
62
63  def testHandlingOfCrashedTab(self):
64    ps = page_set.PageSet()
65    expectations = test_expectations.TestExpectations()
66    page1 = page_module.Page('chrome://crash', ps)
67    ps.pages.append(page1)
68
69    class Test(page_test.PageTest):
70      def ValidatePage(self, *args):
71        pass
72
73    options = options_for_unittests.GetCopy()
74    options.output_format = 'none'
75    SetUpPageRunnerArguments(options)
76    results = page_runner.Run(Test(), ps, expectations, options)
77    self.assertEquals(0, len(results.successes))
78    self.assertEquals(0, len(results.failures))
79    self.assertEquals(1, len(results.errors))
80
81  def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
82    ps = page_set.PageSet()
83    expectations = test_expectations.TestExpectations()
84    ps.pages.append(page_module.Page(
85        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
86    ps.pages.append(page_module.Page(
87        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
88
89    class ExpectedException(Exception):
90      pass
91
92    class Test(page_test.PageTest):
93      def __init__(self, *args):
94        super(Test, self).__init__(*args)
95        self.run_count = 0
96      def ValidatePage(self, *_):
97        old_run_count = self.run_count
98        self.run_count += 1
99        if old_run_count == 0:
100          raise ExpectedException()
101
102    options = options_for_unittests.GetCopy()
103    options.output_format = 'none'
104    test = Test()
105    SetUpPageRunnerArguments(options)
106    results = page_runner.Run(test, ps, expectations, options)
107    self.assertEquals(2, test.run_count)
108    self.assertEquals(1, len(results.successes))
109    self.assertEquals(1, len(results.failures))
110
111  def testHandlingOfCrashedTabWithExpectedFailure(self):
112    ps = page_set.PageSet()
113    expectations = test_expectations.TestExpectations()
114    expectations.Fail('chrome://crash')
115    page1 = page_module.Page('chrome://crash', ps)
116    ps.pages.append(page1)
117
118    class Test(page_test.PageTest):
119      def ValidatePage(self, *_):
120        pass
121
122    options = options_for_unittests.GetCopy()
123    options.output_format = 'none'
124    SetUpPageRunnerArguments(options)
125    results = page_runner.Run(
126        Test(), ps, expectations, options)
127    self.assertEquals(1, len(results.successes))
128    self.assertEquals(0, len(results.failures))
129    self.assertEquals(0, len(results.errors))
130
131  def testRetryOnBrowserCrash(self):
132    ps = page_set.PageSet()
133    expectations = test_expectations.TestExpectations()
134    ps.pages.append(page_module.Page(
135        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
136
137    class CrashyMeasurement(page_measurement.PageMeasurement):
138      has_crashed = False
139      def MeasurePage(self, *_):
140        if not self.has_crashed:
141          self.has_crashed = True
142          raise exceptions.BrowserGoneException()
143
144    options = options_for_unittests.GetCopy()
145    options.output_format = 'csv'
146
147    SetUpPageRunnerArguments(options)
148    results = page_runner.Run(CrashyMeasurement(), ps, expectations, options)
149
150    self.assertEquals(1, len(results.successes))
151    self.assertEquals(0, len(results.failures))
152    self.assertEquals(0, len(results.errors))
153
154  def testDiscardFirstResult(self):
155    ps = page_set.PageSet()
156    expectations = test_expectations.TestExpectations()
157    ps.pages.append(page_module.Page(
158        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
159    ps.pages.append(page_module.Page(
160        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
161
162    class Measurement(page_measurement.PageMeasurement):
163      @property
164      def discard_first_result(self):
165        return True
166      def MeasurePage(self, *args):
167        pass
168
169    options = options_for_unittests.GetCopy()
170    options.output_format = 'none'
171    options.reset_results = None
172    options.upload_results = None
173    options.results_label = None
174
175    options.page_repeat = 1
176    options.pageset_repeat = 1
177    SetUpPageRunnerArguments(options)
178    results = page_runner.Run(Measurement(), ps, expectations, options)
179    self.assertEquals(0, len(results.successes))
180    self.assertEquals(0, len(results.failures))
181
182    options.page_repeat = 1
183    options.pageset_repeat = 2
184    SetUpPageRunnerArguments(options)
185    results = page_runner.Run(Measurement(), ps, expectations, options)
186    self.assertEquals(2, len(results.successes))
187    self.assertEquals(0, len(results.failures))
188
189    options.page_repeat = 2
190    options.pageset_repeat = 1
191    SetUpPageRunnerArguments(options)
192    results = page_runner.Run(Measurement(), ps, expectations, options)
193    self.assertEquals(2, len(results.successes))
194    self.assertEquals(0, len(results.failures))
195
196    options.output_format = 'html'
197    options.page_repeat = 1
198    options.pageset_repeat = 1
199    SetUpPageRunnerArguments(options)
200    results = page_runner.Run(Measurement(), ps, expectations, options)
201    self.assertEquals(0, len(results.successes))
202    self.assertEquals(0, len(results.failures))
203
204  @decorators.Disabled('win')
205  def testPagesetRepeat(self):
206    ps = page_set.PageSet()
207    expectations = test_expectations.TestExpectations()
208    ps.pages.append(page_module.Page(
209        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
210    ps.pages.append(page_module.Page(
211        'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir()))
212
213    class Measurement(page_measurement.PageMeasurement):
214      i = 0
215      def MeasurePage(self, _1, _2, results):
216        self.i += 1
217        results.Add('metric', 'unit', self.i)
218
219    output_file = tempfile.NamedTemporaryFile(delete=False).name
220    try:
221      options = options_for_unittests.GetCopy()
222      options.output_format = 'buildbot'
223      options.output_file = output_file
224      options.reset_results = None
225      options.upload_results = None
226      options.results_label = None
227
228      options.page_repeat = 1
229      options.pageset_repeat = 2
230      SetUpPageRunnerArguments(options)
231      results = page_runner.Run(Measurement(), ps, expectations, options)
232      results.PrintSummary()
233      self.assertEquals(4, len(results.successes))
234      self.assertEquals(0, len(results.failures))
235      with open(output_file) as f:
236        stdout = f.read()
237      self.assertIn('RESULT metric_by_url: blank.html= [1,3] unit', stdout)
238      self.assertIn('RESULT metric_by_url: green_rect.html= [2,4] unit', stdout)
239      self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout)
240    finally:
241      results._output_stream.close()  # pylint: disable=W0212
242      os.remove(output_file)
243
244  def testCredentialsWhenLoginFails(self):
245    credentials_backend = StubCredentialsBackend(login_return_value=False)
246    did_run = self.runCredentialsTest(credentials_backend)
247    assert credentials_backend.did_get_login == True
248    assert credentials_backend.did_get_login_no_longer_needed == False
249    assert did_run == False
250
251  def testCredentialsWhenLoginSucceeds(self):
252    credentials_backend = StubCredentialsBackend(login_return_value=True)
253    did_run = self.runCredentialsTest(credentials_backend)
254    assert credentials_backend.did_get_login == True
255    assert credentials_backend.did_get_login_no_longer_needed == True
256    assert did_run
257
258  def runCredentialsTest(self, credentials_backend):
259    ps = page_set.PageSet()
260    expectations = test_expectations.TestExpectations()
261    page = page_module.Page(
262        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
263    page.credentials = "test"
264    ps.pages.append(page)
265
266    did_run = [False]
267
268    try:
269      with tempfile.NamedTemporaryFile(delete=False) as f:
270        f.write(SIMPLE_CREDENTIALS_STRING)
271        ps.credentials_path = f.name
272
273      class TestThatInstallsCredentialsBackend(page_test.PageTest):
274        def __init__(self, credentials_backend):
275          super(TestThatInstallsCredentialsBackend, self).__init__()
276          self._credentials_backend = credentials_backend
277
278        def DidStartBrowser(self, browser):
279          browser.credentials.AddBackend(self._credentials_backend)
280
281        def ValidatePage(self, *_):
282          did_run[0] = True
283
284      test = TestThatInstallsCredentialsBackend(credentials_backend)
285      options = options_for_unittests.GetCopy()
286      options.output_format = 'none'
287      SetUpPageRunnerArguments(options)
288      page_runner.Run(test, ps, expectations, options)
289    finally:
290      os.remove(f.name)
291
292    return did_run[0]
293
294  def testUserAgent(self):
295    ps = page_set.PageSet()
296    expectations = test_expectations.TestExpectations()
297    page = page_module.Page(
298        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
299    ps.pages.append(page)
300    ps.user_agent_type = 'tablet'
301
302    class TestUserAgent(page_test.PageTest):
303      def ValidatePage(self, _1, tab, _2):
304        actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
305        expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
306        assert actual_user_agent.strip() == expected_user_agent
307
308        # This is so we can check later that the test actually made it into this
309        # function. Previously it was timing out before even getting here, which
310        # should fail, but since it skipped all the asserts, it slipped by.
311        self.hasRun = True # pylint: disable=W0201
312
313    test = TestUserAgent()
314    options = options_for_unittests.GetCopy()
315    options.output_format = 'none'
316    SetUpPageRunnerArguments(options)
317    page_runner.Run(test, ps, expectations, options)
318
319    self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
320
321  # Ensure that page_runner forces exactly 1 tab before running a page.
322  def testOneTab(self):
323    ps = page_set.PageSet()
324    expectations = test_expectations.TestExpectations()
325    page = page_module.Page(
326        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
327    ps.pages.append(page)
328
329    class TestOneTab(page_test.PageTest):
330      def __init__(self):
331        super(TestOneTab, self).__init__()
332        self._browser = None
333
334      def DidStartBrowser(self, browser):
335        self._browser = browser
336        if self._browser.supports_tab_control:
337          self._browser.tabs.New()
338
339      def ValidatePage(self, *_):
340        if not self._browser.supports_tab_control:
341          logging.warning('Browser does not support tab control, skipping test')
342          return
343        assert len(self._browser.tabs) == 1
344
345    test = TestOneTab()
346    options = options_for_unittests.GetCopy()
347    options.output_format = 'none'
348    SetUpPageRunnerArguments(options)
349    page_runner.Run(test, ps, expectations, options)
350
351  # Ensure that page_runner allows the test to customize the browser before it
352  # launches.
353  def testBrowserBeforeLaunch(self):
354    ps = page_set.PageSet()
355    expectations = test_expectations.TestExpectations()
356    page = page_module.Page(
357        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
358    ps.pages.append(page)
359
360    class TestBeforeLaunch(page_test.PageTest):
361      def __init__(self):
362        super(TestBeforeLaunch, self).__init__()
363        self._did_call_will_start = False
364        self._did_call_did_start = False
365
366      def WillStartBrowser(self, browser):
367        self._did_call_will_start = True
368        # TODO(simonjam): Test that the profile is available.
369
370      def DidStartBrowser(self, browser):
371        assert self._did_call_will_start
372        self._did_call_did_start = True
373
374      def ValidatePage(self, *_):
375        assert self._did_call_did_start
376
377    test = TestBeforeLaunch()
378    options = options_for_unittests.GetCopy()
379    options.output_format = 'none'
380    SetUpPageRunnerArguments(options)
381    page_runner.Run(test, ps, expectations, options)
382
383  def testRunPageWithStartupUrl(self):
384    ps = page_set.PageSet()
385    expectations = test_expectations.TestExpectations()
386    expectations = test_expectations.TestExpectations()
387    page = page_module.Page(
388        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
389    page.startup_url = 'about:blank'
390    ps.pages.append(page)
391
392    class Measurement(page_measurement.PageMeasurement):
393      def __init__(self):
394        super(Measurement, self).__init__()
395        self.browser_restarted = False
396
397      def CustomizeBrowserOptionsForSinglePage(self, ps, options):
398        self.browser_restarted = True
399        super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps,
400                                                                      options)
401      def MeasurePage(self, page, tab, results):
402        pass
403
404    options = options_for_unittests.GetCopy()
405    options.page_repeat = 2
406    options.output_format = 'none'
407    if not browser_finder.FindBrowser(options):
408      return
409    test = Measurement()
410    SetUpPageRunnerArguments(options)
411    page_runner.Run(test, ps, expectations, options)
412    self.assertEquals('about:blank', options.browser_options.startup_url)
413    self.assertTrue(test.browser_restarted)
414
415  # Ensure that page_runner calls cleanUp when a page run fails.
416  def testCleanUpPage(self):
417    ps = page_set.PageSet()
418    expectations = test_expectations.TestExpectations()
419    page = page_module.Page(
420        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
421    ps.pages.append(page)
422
423    class Test(page_test.PageTest):
424      def __init__(self):
425        super(Test, self).__init__()
426        self.did_call_clean_up = False
427
428      def ValidatePage(self, *_):
429        raise Exception('Intentional failure')
430
431      def CleanUpAfterPage(self, page, tab):
432        self.did_call_clean_up = True
433
434
435    test = Test()
436    options = options_for_unittests.GetCopy()
437    options.output_format = 'none'
438    SetUpPageRunnerArguments(options)
439    page_runner.Run(test, ps, expectations, options)
440    assert test.did_call_clean_up
441