page_runner_unittest.py revision 5d1f7b1de12d16ceb2c938c56701a3e8bfa558f7 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import logging
5import os
6import tempfile
7import unittest
8
9from telemetry.core import exceptions
10from telemetry.core import user_agent
11from telemetry.core import util
12from telemetry.page import page as page_module
13from telemetry.page import page_measurement
14from telemetry.page import page_set
15from telemetry.page import page_test
16from telemetry.page import page_runner
17from telemetry.page import test_expectations
18from telemetry.unittest import options_for_unittests
19
20SIMPLE_CREDENTIALS_STRING = """
21{
22  "test": {
23    "username": "example",
24    "password": "asdf"
25  }
26}
27"""
28class StubCredentialsBackend(object):
29  def __init__(self, login_return_value):
30    self.did_get_login = False
31    self.did_get_login_no_longer_needed = False
32    self.login_return_value = login_return_value
33
34  @property
35  def credentials_type(self): # pylint: disable=R0201
36    return 'test'
37
38  def LoginNeeded(self, tab, config): # pylint: disable=W0613
39    self.did_get_login = True
40    return self.login_return_value
41
42  def LoginNoLongerNeeded(self, tab): # pylint: disable=W0613
43    self.did_get_login_no_longer_needed = True
44
45class PageRunnerTests(unittest.TestCase):
46  # TODO(nduca): Move the basic "test failed, test succeeded" tests from
47  # page_measurement_unittest to here.
48
49  def testHandlingOfCrashedTab(self):
50    ps = page_set.PageSet()
51    expectations = test_expectations.TestExpectations()
52    page1 = page_module.Page('chrome://crash', ps)
53    ps.pages.append(page1)
54
55    class Test(page_test.PageTest):
56      def RunTest(self, *args):
57        pass
58
59    options = options_for_unittests.GetCopy()
60    options.output_format = 'none'
61    results = page_runner.Run(Test('RunTest'), ps, expectations, options)
62    self.assertEquals(0, len(results.successes))
63    self.assertEquals(0, len(results.failures))
64    self.assertEquals(1, len(results.errors))
65
66  def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
67    ps = page_set.PageSet()
68    expectations = test_expectations.TestExpectations()
69    ps.pages.append(page_module.Page(
70        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
71    ps.pages.append(page_module.Page(
72        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
73
74    class ExpectedException(Exception):
75      pass
76
77    class Test(page_test.PageTest):
78      def __init__(self, *args):
79        super(Test, self).__init__(*args)
80        self.run_count = 0
81      def RunTest(self, *_):
82        old_run_count = self.run_count
83        self.run_count += 1
84        if old_run_count == 0:
85          raise ExpectedException()
86
87    options = options_for_unittests.GetCopy()
88    options.output_format = 'none'
89    test = Test('RunTest')
90    results = page_runner.Run(test, ps, expectations, options)
91    self.assertEquals(2, test.run_count)
92    self.assertEquals(1, len(results.successes))
93    self.assertEquals(1, len(results.failures))
94
95  def testHandlingOfCrashedTabWithExpectedFailure(self):
96    ps = page_set.PageSet()
97    expectations = test_expectations.TestExpectations()
98    expectations.Fail('chrome://crash')
99    page1 = page_module.Page('chrome://crash', ps)
100    ps.pages.append(page1)
101
102    class Test(page_test.PageTest):
103      def RunTest(self, *_):
104        pass
105
106    options = options_for_unittests.GetCopy()
107    options.output_format = 'none'
108    results = page_runner.Run(
109        Test('RunTest'), ps, expectations, options)
110    self.assertEquals(1, len(results.successes))
111    self.assertEquals(0, len(results.failures))
112    self.assertEquals(0, len(results.errors))
113
114  def testRetryOnBrowserCrash(self):
115    ps = page_set.PageSet()
116    expectations = test_expectations.TestExpectations()
117    ps.pages.append(page_module.Page(
118        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
119
120    class CrashyMeasurement(page_measurement.PageMeasurement):
121      has_crashed = False
122      def MeasurePage(self, *_):
123        if not self.has_crashed:
124          self.has_crashed = True
125          raise exceptions.BrowserGoneException()
126
127    options = options_for_unittests.GetCopy()
128    options.output_format = 'csv'
129
130    results = page_runner.Run(CrashyMeasurement(), ps, expectations, options)
131
132    self.assertEquals(1, len(results.successes))
133    self.assertEquals(0, len(results.failures))
134    self.assertEquals(0, len(results.errors))
135
136  def testDiscardFirstResult(self):
137    ps = page_set.PageSet()
138    expectations = test_expectations.TestExpectations()
139    ps.pages.append(page_module.Page(
140        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
141    ps.pages.append(page_module.Page(
142        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
143
144    class Measurement(page_measurement.PageMeasurement):
145      @property
146      def discard_first_result(self):
147        return True
148      def MeasurePage(self, *args):
149        pass
150
151    options = options_for_unittests.GetCopy()
152    options.output_format = 'none'
153    options.reset_results = None
154    options.upload_results = None
155    options.results_label = None
156
157    options.repeat_options.page_repeat_iters = 1
158    options.repeat_options.pageset_repeat_iters = 1
159    results = page_runner.Run(Measurement(), ps, expectations, options)
160    self.assertEquals(0, len(results.successes))
161    self.assertEquals(0, len(results.failures))
162
163    options.repeat_options.page_repeat_iters = 1
164    options.repeat_options.pageset_repeat_iters = 2
165    results = page_runner.Run(Measurement(), ps, expectations, options)
166    self.assertEquals(2, len(results.successes))
167    self.assertEquals(0, len(results.failures))
168
169    options.repeat_options.page_repeat_iters = 2
170    options.repeat_options.pageset_repeat_iters = 1
171    results = page_runner.Run(Measurement(), ps, expectations, options)
172    self.assertEquals(2, len(results.successes))
173    self.assertEquals(0, len(results.failures))
174
175    options.output_format = 'html'
176    options.repeat_options.page_repeat_iters = 1
177    options.repeat_options.pageset_repeat_iters = 1
178    results = page_runner.Run(Measurement(), ps, expectations, options)
179    self.assertEquals(0, len(results.successes))
180    self.assertEquals(0, len(results.failures))
181
182  def testPagesetRepeat(self):
183    ps = page_set.PageSet()
184    expectations = test_expectations.TestExpectations()
185    ps.pages.append(page_module.Page(
186        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
187    ps.pages.append(page_module.Page(
188        'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir()))
189
190    class Measurement(page_measurement.PageMeasurement):
191      i = 0
192      def MeasurePage(self, _, __, results):
193        self.i += 1
194        results.Add('metric', 'unit', self.i)
195
196    output_file = tempfile.NamedTemporaryFile(delete=False).name
197    try:
198      options = options_for_unittests.GetCopy()
199      options.output_format = 'buildbot'
200      options.output_file = output_file
201      options.reset_results = None
202      options.upload_results = None
203      options.results_label = None
204
205      options.repeat_options.page_repeat_iters = 1
206      options.repeat_options.pageset_repeat_iters = 2
207      results = page_runner.Run(Measurement(), ps, expectations, options)
208      results.PrintSummary()
209      self.assertEquals(4, len(results.successes))
210      self.assertEquals(0, len(results.failures))
211      stdout = open(output_file).read()
212      self.assertIn('RESULT metric_by_url: blank.html= [1,3] unit', stdout)
213      self.assertIn('RESULT metric_by_url: green_rect.html= [2,4] unit', stdout)
214      self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout)
215    finally:
216      results._output_stream.close()  # pylint: disable=W0212
217      os.remove(output_file)
218
219  def testCredentialsWhenLoginFails(self):
220    credentials_backend = StubCredentialsBackend(login_return_value=False)
221    did_run = self.runCredentialsTest(credentials_backend)
222    assert credentials_backend.did_get_login == True
223    assert credentials_backend.did_get_login_no_longer_needed == False
224    assert did_run == False
225
226  def testCredentialsWhenLoginSucceeds(self):
227    credentials_backend = StubCredentialsBackend(login_return_value=True)
228    did_run = self.runCredentialsTest(credentials_backend)
229    assert credentials_backend.did_get_login == True
230    assert credentials_backend.did_get_login_no_longer_needed == True
231    assert did_run
232
233  def runCredentialsTest(self, # pylint: disable=R0201
234                         credentials_backend):
235    ps = page_set.PageSet()
236    expectations = test_expectations.TestExpectations()
237    page = page_module.Page(
238        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
239    page.credentials = "test"
240    ps.pages.append(page)
241
242    did_run = [False]
243
244    try:
245      with tempfile.NamedTemporaryFile(delete=False) as f:
246        f.write(SIMPLE_CREDENTIALS_STRING)
247        ps.credentials_path = f.name
248
249      class TestThatInstallsCredentialsBackend(page_test.PageTest):
250        def __init__(self, credentials_backend):
251          super(TestThatInstallsCredentialsBackend, self).__init__('RunTest')
252          self._credentials_backend = credentials_backend
253
254        def DidStartBrowser(self, browser):
255          browser.credentials.AddBackend(self._credentials_backend)
256
257        def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
258          did_run[0] = True
259
260      test = TestThatInstallsCredentialsBackend(credentials_backend)
261      options = options_for_unittests.GetCopy()
262      options.output_format = 'none'
263      page_runner.Run(test, ps, expectations, options)
264    finally:
265      os.remove(f.name)
266
267    return did_run[0]
268
269  def testUserAgent(self):
270    ps = page_set.PageSet()
271    expectations = test_expectations.TestExpectations()
272    page = page_module.Page(
273        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
274    ps.pages.append(page)
275    ps.user_agent_type = 'tablet'
276
277    class TestUserAgent(page_test.PageTest):
278      def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
279        actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
280        expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
281        assert actual_user_agent.strip() == expected_user_agent
282
283        # This is so we can check later that the test actually made it into this
284        # function. Previously it was timing out before even getting here, which
285        # should fail, but since it skipped all the asserts, it slipped by.
286        self.hasRun = True # pylint: disable=W0201
287
288    test = TestUserAgent('RunTest')
289    options = options_for_unittests.GetCopy()
290    options.output_format = 'none'
291    page_runner.Run(test, ps, expectations, options)
292
293    self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
294
295  # Ensure that page_runner forces exactly 1 tab before running a page.
296  def testOneTab(self):
297    ps = page_set.PageSet()
298    expectations = test_expectations.TestExpectations()
299    page = page_module.Page(
300        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
301    ps.pages.append(page)
302
303    class TestOneTab(page_test.PageTest):
304      def __init__(self,
305                   test_method_name,
306                   action_name_to_run='',
307                   needs_browser_restart_after_each_run=False):
308        super(TestOneTab, self).__init__(test_method_name, action_name_to_run,
309                                         needs_browser_restart_after_each_run)
310        self._browser = None
311
312      def DidStartBrowser(self, browser):
313        self._browser = browser
314        if self._browser.supports_tab_control:
315          self._browser.tabs.New()
316
317      def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
318        if not self._browser.supports_tab_control:
319          logging.warning('Browser does not support tab control, skipping test')
320          return
321        assert len(self._browser.tabs) == 1
322
323    test = TestOneTab('RunTest')
324    options = options_for_unittests.GetCopy()
325    options.output_format = 'none'
326    page_runner.Run(test, ps, expectations, options)
327
328  # Ensure that page_runner allows the test to customize the browser before it
329  # launches.
330  def testBrowserBeforeLaunch(self):
331    ps = page_set.PageSet()
332    expectations = test_expectations.TestExpectations()
333    page = page_module.Page(
334        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
335    ps.pages.append(page)
336
337    class TestBeforeLaunch(page_test.PageTest):
338      def __init__(self,
339                   test_method_name,
340                   action_name_to_run=''):
341        super(TestBeforeLaunch, self).__init__(
342            test_method_name, action_name_to_run, False)
343        self._did_call_will_start = False
344        self._did_call_did_start = False
345
346      def WillStartBrowser(self, browser):
347        self._did_call_will_start = True
348        # TODO(simonjam): Test that the profile is available.
349
350      def DidStartBrowser(self, browser):
351        assert self._did_call_will_start
352        self._did_call_did_start = True
353
354      def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
355        assert self._did_call_did_start
356
357    test = TestBeforeLaunch('RunTest')
358    options = options_for_unittests.GetCopy()
359    options.output_format = 'none'
360    page_runner.Run(test, ps, expectations, options)
361