page_runner_unittest.py revision a02191e04bc25c4935f804f2c080ae28663d096d 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import tempfile
8import unittest
9
10from telemetry.core import browser_finder
11from telemetry.core import exceptions
12from telemetry.core import user_agent
13from telemetry.core import util
14from telemetry.page import page as page_module
15from telemetry.page import page_measurement
16from telemetry.page import page_set
17from telemetry.page import page_test
18from telemetry.page import page_runner
19from telemetry.page import test_expectations
20from telemetry.unittest import options_for_unittests
21
22
23SIMPLE_CREDENTIALS_STRING = """
24{
25  "test": {
26    "username": "example",
27    "password": "asdf"
28  }
29}
30"""
31
32
33def SetUpPageRunnerArguments(options):
34  parser = options.CreateParser()
35  page_runner.AddCommandLineArgs(parser)
36  options.MergeDefaultValues(parser.get_default_values())
37  page_runner.ProcessCommandLineArgs(parser, options)
38
39
40class StubCredentialsBackend(object):
41  def __init__(self, login_return_value):
42    self.did_get_login = False
43    self.did_get_login_no_longer_needed = False
44    self.login_return_value = login_return_value
45
46  @property
47  def credentials_type(self): # pylint: disable=R0201
48    return 'test'
49
50  def LoginNeeded(self, tab, config): # pylint: disable=W0613
51    self.did_get_login = True
52    return self.login_return_value
53
54  def LoginNoLongerNeeded(self, tab): # pylint: disable=W0613
55    self.did_get_login_no_longer_needed = True
56
57
58class PageRunnerTests(unittest.TestCase):
59  # TODO(nduca): Move the basic "test failed, test succeeded" tests from
60  # page_measurement_unittest to here.
61
62  def testHandlingOfCrashedTab(self):
63    ps = page_set.PageSet()
64    expectations = test_expectations.TestExpectations()
65    page1 = page_module.Page('chrome://crash', ps)
66    ps.pages.append(page1)
67
68    class Test(page_test.PageTest):
69      def RunTest(self, *args):
70        pass
71
72    options = options_for_unittests.GetCopy()
73    options.output_format = 'none'
74    SetUpPageRunnerArguments(options)
75    results = page_runner.Run(Test('RunTest'), ps, expectations, options)
76    self.assertEquals(0, len(results.successes))
77    self.assertEquals(0, len(results.failures))
78    self.assertEquals(1, len(results.errors))
79
80  def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
81    ps = page_set.PageSet()
82    expectations = test_expectations.TestExpectations()
83    ps.pages.append(page_module.Page(
84        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
85    ps.pages.append(page_module.Page(
86        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
87
88    class ExpectedException(Exception):
89      pass
90
91    class Test(page_test.PageTest):
92      def __init__(self, *args):
93        super(Test, self).__init__(*args)
94        self.run_count = 0
95      def RunTest(self, *_):
96        old_run_count = self.run_count
97        self.run_count += 1
98        if old_run_count == 0:
99          raise ExpectedException()
100
101    options = options_for_unittests.GetCopy()
102    options.output_format = 'none'
103    test = Test('RunTest')
104    SetUpPageRunnerArguments(options)
105    results = page_runner.Run(test, ps, expectations, options)
106    self.assertEquals(2, test.run_count)
107    self.assertEquals(1, len(results.successes))
108    self.assertEquals(1, len(results.failures))
109
110  def testHandlingOfCrashedTabWithExpectedFailure(self):
111    ps = page_set.PageSet()
112    expectations = test_expectations.TestExpectations()
113    expectations.Fail('chrome://crash')
114    page1 = page_module.Page('chrome://crash', ps)
115    ps.pages.append(page1)
116
117    class Test(page_test.PageTest):
118      def RunTest(self, *_):
119        pass
120
121    options = options_for_unittests.GetCopy()
122    options.output_format = 'none'
123    SetUpPageRunnerArguments(options)
124    results = page_runner.Run(
125        Test('RunTest'), ps, expectations, options)
126    self.assertEquals(1, len(results.successes))
127    self.assertEquals(0, len(results.failures))
128    self.assertEquals(0, len(results.errors))
129
130  def testRetryOnBrowserCrash(self):
131    ps = page_set.PageSet()
132    expectations = test_expectations.TestExpectations()
133    ps.pages.append(page_module.Page(
134        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
135
136    class CrashyMeasurement(page_measurement.PageMeasurement):
137      has_crashed = False
138      def MeasurePage(self, *_):
139        if not self.has_crashed:
140          self.has_crashed = True
141          raise exceptions.BrowserGoneException()
142
143    options = options_for_unittests.GetCopy()
144    options.output_format = 'csv'
145
146    SetUpPageRunnerArguments(options)
147    results = page_runner.Run(CrashyMeasurement(), ps, expectations, options)
148
149    self.assertEquals(1, len(results.successes))
150    self.assertEquals(0, len(results.failures))
151    self.assertEquals(0, len(results.errors))
152
153  def testDiscardFirstResult(self):
154    ps = page_set.PageSet()
155    expectations = test_expectations.TestExpectations()
156    ps.pages.append(page_module.Page(
157        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
158    ps.pages.append(page_module.Page(
159        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
160
161    class Measurement(page_measurement.PageMeasurement):
162      @property
163      def discard_first_result(self):
164        return True
165      def MeasurePage(self, *args):
166        pass
167
168    options = options_for_unittests.GetCopy()
169    options.output_format = 'none'
170    options.reset_results = None
171    options.upload_results = None
172    options.results_label = None
173
174    options.page_repeat = 1
175    options.pageset_repeat = 1
176    SetUpPageRunnerArguments(options)
177    results = page_runner.Run(Measurement(), ps, expectations, options)
178    self.assertEquals(0, len(results.successes))
179    self.assertEquals(0, len(results.failures))
180
181    options.page_repeat = 1
182    options.pageset_repeat = 2
183    SetUpPageRunnerArguments(options)
184    results = page_runner.Run(Measurement(), ps, expectations, options)
185    self.assertEquals(2, len(results.successes))
186    self.assertEquals(0, len(results.failures))
187
188    options.page_repeat = 2
189    options.pageset_repeat = 1
190    SetUpPageRunnerArguments(options)
191    results = page_runner.Run(Measurement(), ps, expectations, options)
192    self.assertEquals(2, len(results.successes))
193    self.assertEquals(0, len(results.failures))
194
195    options.output_format = 'html'
196    options.page_repeat = 1
197    options.pageset_repeat = 1
198    SetUpPageRunnerArguments(options)
199    results = page_runner.Run(Measurement(), ps, expectations, options)
200    self.assertEquals(0, len(results.successes))
201    self.assertEquals(0, len(results.failures))
202
203  def testPagesetRepeat(self):
204    ps = page_set.PageSet()
205    expectations = test_expectations.TestExpectations()
206    ps.pages.append(page_module.Page(
207        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
208    ps.pages.append(page_module.Page(
209        'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir()))
210
211    class Measurement(page_measurement.PageMeasurement):
212      i = 0
213      def MeasurePage(self, _, __, results):
214        self.i += 1
215        results.Add('metric', 'unit', self.i)
216
217    output_file = tempfile.NamedTemporaryFile(delete=False).name
218    try:
219      options = options_for_unittests.GetCopy()
220      options.output_format = 'buildbot'
221      options.output_file = output_file
222      options.reset_results = None
223      options.upload_results = None
224      options.results_label = None
225
226      options.page_repeat = 1
227      options.pageset_repeat = 2
228      SetUpPageRunnerArguments(options)
229      results = page_runner.Run(Measurement(), ps, expectations, options)
230      results.PrintSummary()
231      self.assertEquals(4, len(results.successes))
232      self.assertEquals(0, len(results.failures))
233      stdout = open(output_file).read()
234      self.assertIn('RESULT metric_by_url: blank.html= [1,3] unit', stdout)
235      self.assertIn('RESULT metric_by_url: green_rect.html= [2,4] unit', stdout)
236      self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout)
237    finally:
238      results._output_stream.close()  # pylint: disable=W0212
239      os.remove(output_file)
240
241  def testCredentialsWhenLoginFails(self):
242    credentials_backend = StubCredentialsBackend(login_return_value=False)
243    did_run = self.runCredentialsTest(credentials_backend)
244    assert credentials_backend.did_get_login == True
245    assert credentials_backend.did_get_login_no_longer_needed == False
246    assert did_run == False
247
248  def testCredentialsWhenLoginSucceeds(self):
249    credentials_backend = StubCredentialsBackend(login_return_value=True)
250    did_run = self.runCredentialsTest(credentials_backend)
251    assert credentials_backend.did_get_login == True
252    assert credentials_backend.did_get_login_no_longer_needed == True
253    assert did_run
254
255  def runCredentialsTest(self, # pylint: disable=R0201
256                         credentials_backend):
257    ps = page_set.PageSet()
258    expectations = test_expectations.TestExpectations()
259    page = page_module.Page(
260        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
261    page.credentials = "test"
262    ps.pages.append(page)
263
264    did_run = [False]
265
266    try:
267      with tempfile.NamedTemporaryFile(delete=False) as f:
268        f.write(SIMPLE_CREDENTIALS_STRING)
269        ps.credentials_path = f.name
270
271      class TestThatInstallsCredentialsBackend(page_test.PageTest):
272        def __init__(self, credentials_backend):
273          super(TestThatInstallsCredentialsBackend, self).__init__('RunTest')
274          self._credentials_backend = credentials_backend
275
276        def DidStartBrowser(self, browser):
277          browser.credentials.AddBackend(self._credentials_backend)
278
279        def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
280          did_run[0] = True
281
282      test = TestThatInstallsCredentialsBackend(credentials_backend)
283      options = options_for_unittests.GetCopy()
284      options.output_format = 'none'
285      SetUpPageRunnerArguments(options)
286      page_runner.Run(test, ps, expectations, options)
287    finally:
288      os.remove(f.name)
289
290    return did_run[0]
291
292  def testUserAgent(self):
293    ps = page_set.PageSet()
294    expectations = test_expectations.TestExpectations()
295    page = page_module.Page(
296        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
297    ps.pages.append(page)
298    ps.user_agent_type = 'tablet'
299
300    class TestUserAgent(page_test.PageTest):
301      def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
302        actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
303        expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
304        assert actual_user_agent.strip() == expected_user_agent
305
306        # This is so we can check later that the test actually made it into this
307        # function. Previously it was timing out before even getting here, which
308        # should fail, but since it skipped all the asserts, it slipped by.
309        self.hasRun = True # pylint: disable=W0201
310
311    test = TestUserAgent('RunTest')
312    options = options_for_unittests.GetCopy()
313    options.output_format = 'none'
314    SetUpPageRunnerArguments(options)
315    page_runner.Run(test, ps, expectations, options)
316
317    self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
318
319  # Ensure that page_runner forces exactly 1 tab before running a page.
320  def testOneTab(self):
321    ps = page_set.PageSet()
322    expectations = test_expectations.TestExpectations()
323    page = page_module.Page(
324        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
325    ps.pages.append(page)
326
327    class TestOneTab(page_test.PageTest):
328      def __init__(self,
329                   test_method_name,
330                   action_name_to_run='',
331                   needs_browser_restart_after_each_page=False):
332        super(TestOneTab, self).__init__(test_method_name, action_name_to_run,
333                                         needs_browser_restart_after_each_page)
334        self._browser = None
335
336      def DidStartBrowser(self, browser):
337        self._browser = browser
338        if self._browser.supports_tab_control:
339          self._browser.tabs.New()
340
341      def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
342        if not self._browser.supports_tab_control:
343          logging.warning('Browser does not support tab control, skipping test')
344          return
345        assert len(self._browser.tabs) == 1
346
347    test = TestOneTab('RunTest')
348    options = options_for_unittests.GetCopy()
349    options.output_format = 'none'
350    SetUpPageRunnerArguments(options)
351    page_runner.Run(test, ps, expectations, options)
352
353  # Ensure that page_runner allows the test to customize the browser before it
354  # launches.
355  def testBrowserBeforeLaunch(self):
356    ps = page_set.PageSet()
357    expectations = test_expectations.TestExpectations()
358    page = page_module.Page(
359        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
360    ps.pages.append(page)
361
362    class TestBeforeLaunch(page_test.PageTest):
363      def __init__(self,
364                   test_method_name,
365                   action_name_to_run=''):
366        super(TestBeforeLaunch, self).__init__(
367            test_method_name, action_name_to_run, False)
368        self._did_call_will_start = False
369        self._did_call_did_start = False
370
371      def WillStartBrowser(self, browser):
372        self._did_call_will_start = True
373        # TODO(simonjam): Test that the profile is available.
374
375      def DidStartBrowser(self, browser):
376        assert self._did_call_will_start
377        self._did_call_did_start = True
378
379      def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
380        assert self._did_call_did_start
381
382    test = TestBeforeLaunch('RunTest')
383    options = options_for_unittests.GetCopy()
384    options.output_format = 'none'
385    SetUpPageRunnerArguments(options)
386    page_runner.Run(test, ps, expectations, options)
387
388  def testRunPageWithStartupUrl(self):
389    ps = page_set.PageSet()
390    expectations = test_expectations.TestExpectations()
391    expectations = test_expectations.TestExpectations()
392    page = page_module.Page(
393        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
394    page.startup_url = 'about:blank'
395    ps.pages.append(page)
396
397    class Measurement(page_measurement.PageMeasurement):
398      def __init__(self):
399        super(Measurement, self).__init__()
400        self.browser_restarted = False
401
402      def CustomizeBrowserOptionsForSinglePage(self, ps, options):
403        self.browser_restarted = True
404        super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps,
405                                                                      options)
406      def MeasurePage(self, page, tab, results):
407        pass
408
409    options = options_for_unittests.GetCopy()
410    options.page_repeat = 2
411    options.output_format = 'none'
412    if not browser_finder.FindBrowser(options):
413      return
414    test = Measurement()
415    SetUpPageRunnerArguments(options)
416    page_runner.Run(test, ps, expectations, options)
417    self.assertEquals('about:blank', options.browser_options.startup_url)
418    self.assertTrue(test.browser_restarted)
419
420  # Ensure that page_runner calls cleanUp when a page run fails.
421  def testCleanUpPage(self):
422    ps = page_set.PageSet()
423    expectations = test_expectations.TestExpectations()
424    page = page_module.Page(
425        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
426    ps.pages.append(page)
427
428    class Test(page_test.PageTest):
429      def __init__(self,
430                   test_method_name,
431                   action_name_to_run=''):
432        super(Test, self).__init__(
433            test_method_name, action_name_to_run, False)
434        self.did_call_clean_up = False
435
436      def RunTest(self, _, _2, _3):
437        raise Exception('Intentional failure')
438
439      def CleanUpAfterPage(self, page, tab):
440        self.did_call_clean_up = True
441
442
443    test = Test('RunTest')
444    options = options_for_unittests.GetCopy()
445    options.output_format = 'none'
446    SetUpPageRunnerArguments(options)
447    page_runner.Run(test, ps, expectations, options)
448    assert test.did_call_clean_up
449