1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import tempfile
8import unittest
9
10from telemetry import benchmark
11from telemetry import decorators
12from telemetry.core import browser_finder
13from telemetry.core import exceptions
14from telemetry.core import user_agent
15from telemetry.core import util
16from telemetry.page import page as page_module
17from telemetry.page import page_runner
18from telemetry.page import page_set
19from telemetry.page import page_test
20from telemetry.page import test_expectations
21from telemetry.results import results_options
22from telemetry.unittest import options_for_unittests
23from telemetry.value import scalar
24from telemetry.value import string
25
26
27SIMPLE_CREDENTIALS_STRING = """
28{
29  "test": {
30    "username": "example",
31    "password": "asdf"
32  }
33}
34"""
35
36
37def SetUpPageRunnerArguments(options):
38  parser = options.CreateParser()
39  page_runner.AddCommandLineArgs(parser)
40  options.MergeDefaultValues(parser.get_default_values())
41  page_runner.ProcessCommandLineArgs(parser, options)
42
43class EmptyMetadataForTest(benchmark.BenchmarkMetadata):
44  def __init__(self):
45    super(EmptyMetadataForTest, self).__init__('')
46
47class StubCredentialsBackend(object):
48  def __init__(self, login_return_value):
49    self.did_get_login = False
50    self.did_get_login_no_longer_needed = False
51    self.login_return_value = login_return_value
52
53  @property
54  def credentials_type(self):
55    return 'test'
56
57  def LoginNeeded(self, *_):
58    self.did_get_login = True
59    return self.login_return_value
60
61  def LoginNoLongerNeeded(self, _):
62    self.did_get_login_no_longer_needed = True
63
64
65def GetSuccessfulPageRuns(results):
66  return [run for run in results.all_page_runs if run.ok or run.skipped]
67
68
69class PageRunnerTests(unittest.TestCase):
70  # TODO(nduca): Move the basic "test failed, test succeeded" tests from
71  # page_test_unittest to here.
72
73  def testHandlingOfCrashedTab(self):
74    ps = page_set.PageSet()
75    expectations = test_expectations.TestExpectations()
76    page1 = page_module.Page('chrome://crash', ps)
77    ps.pages.append(page1)
78
79    class Test(page_test.PageTest):
80      def ValidatePage(self, *args):
81        pass
82
83    options = options_for_unittests.GetCopy()
84    options.output_formats = ['none']
85    options.suppress_gtest_report = True
86    SetUpPageRunnerArguments(options)
87    results = results_options.CreateResults(EmptyMetadataForTest(), options)
88    page_runner.Run(Test(), ps, expectations, options, results)
89    self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
90    self.assertEquals(1, len(results.failures))
91
92  def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
93    ps = page_set.PageSet()
94    expectations = test_expectations.TestExpectations()
95    ps.pages.append(page_module.Page(
96        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
97    ps.pages.append(page_module.Page(
98        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
99
100    class ExpectedException(Exception):
101      pass
102
103    class Test(page_test.PageTest):
104      def __init__(self, *args):
105        super(Test, self).__init__(*args)
106        self.run_count = 0
107      def ValidatePage(self, *_):
108        old_run_count = self.run_count
109        self.run_count += 1
110        if old_run_count == 0:
111          raise ExpectedException()
112
113    options = options_for_unittests.GetCopy()
114    options.output_formats = ['none']
115    options.suppress_gtest_report = True
116    test = Test()
117    SetUpPageRunnerArguments(options)
118    results = results_options.CreateResults(EmptyMetadataForTest(), options)
119    page_runner.Run(test, ps, expectations, options, results)
120    self.assertEquals(2, test.run_count)
121    self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
122    self.assertEquals(1, len(results.failures))
123
124  def testHandlingOfCrashedTabWithExpectedFailure(self):
125    ps = page_set.PageSet()
126    expectations = test_expectations.TestExpectations()
127    expectations.Fail('chrome://crash')
128    page1 = page_module.Page('chrome://crash', ps)
129    ps.pages.append(page1)
130
131    class Test(page_test.PageTest):
132      def ValidatePage(self, *_):
133        pass
134
135    options = options_for_unittests.GetCopy()
136    options.output_formats = ['none']
137    options.suppress_gtest_report = True
138    SetUpPageRunnerArguments(options)
139    results = results_options.CreateResults(EmptyMetadataForTest(), options)
140    page_runner.Run(Test(), ps, expectations, options, results)
141    self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
142    self.assertEquals(0, len(results.failures))
143
144  def testRetryOnBrowserCrash(self):
145    ps = page_set.PageSet()
146    expectations = test_expectations.TestExpectations()
147    ps.pages.append(page_module.Page(
148        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
149
150    class CrashyMeasurement(page_test.PageTest):
151      has_crashed = False
152      def ValidateAndMeasurePage(self, page, tab, results):
153        # This value should be discarded on the first run when the
154        # browser crashed.
155        results.AddValue(
156            string.StringValue(page, 'test', 't', str(self.has_crashed)))
157        if not self.has_crashed:
158          self.has_crashed = True
159          raise exceptions.BrowserGoneException(tab.browser)
160
161    options = options_for_unittests.GetCopy()
162    options.output_formats = ['csv']
163    options.suppress_gtest_report = True
164
165    SetUpPageRunnerArguments(options)
166    results = results_options.CreateResults(EmptyMetadataForTest(), options)
167    page_runner.Run(CrashyMeasurement(), ps, expectations, options, results)
168
169    self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
170    self.assertEquals(0, len(results.failures))
171    self.assertEquals(1, len(results.all_page_specific_values))
172    self.assertEquals(
173        'True', results.all_page_specific_values[0].GetRepresentativeString())
174
175  @decorators.Disabled('xp')  # Flaky, http://crbug.com/390079.
176  def testDiscardFirstResult(self):
177    ps = page_set.PageSet()
178    expectations = test_expectations.TestExpectations()
179    ps.pages.append(page_module.Page(
180        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
181    ps.pages.append(page_module.Page(
182        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
183
184    class Measurement(page_test.PageTest):
185      @property
186      def discard_first_result(self):
187        return True
188
189      def ValidateAndMeasurePage(self, page, _, results):
190        results.AddValue(string.StringValue(page, 'test', 't', page.url))
191
192    options = options_for_unittests.GetCopy()
193    options.output_formats = ['none']
194    options.suppress_gtest_report = True
195    options.reset_results = None
196    options.upload_results = None
197    options.results_label = None
198
199    options.page_repeat = 1
200    options.pageset_repeat = 1
201    SetUpPageRunnerArguments(options)
202    results = results_options.CreateResults(EmptyMetadataForTest(), options)
203    page_runner.Run(Measurement(), ps, expectations, options, results)
204    self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
205    self.assertEquals(0, len(results.failures))
206    self.assertEquals(0, len(results.all_page_specific_values))
207
208    options.page_repeat = 1
209    options.pageset_repeat = 2
210    SetUpPageRunnerArguments(options)
211    results = results_options.CreateResults(EmptyMetadataForTest(), options)
212    page_runner.Run(Measurement(), ps, expectations, options, results)
213    self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
214    self.assertEquals(0, len(results.failures))
215    self.assertEquals(2, len(results.all_page_specific_values))
216
217    options.page_repeat = 2
218    options.pageset_repeat = 1
219    SetUpPageRunnerArguments(options)
220    results = results_options.CreateResults(EmptyMetadataForTest(), options)
221    page_runner.Run(Measurement(), ps, expectations, options, results)
222    self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
223    self.assertEquals(0, len(results.failures))
224    self.assertEquals(2, len(results.all_page_specific_values))
225
226    options.output_formats = ['html']
227    options.suppress_gtest_report = True
228    options.page_repeat = 1
229    options.pageset_repeat = 1
230    SetUpPageRunnerArguments(options)
231    results = results_options.CreateResults(EmptyMetadataForTest(), options)
232    page_runner.Run(Measurement(), ps, expectations, options, results)
233    self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
234    self.assertEquals(0, len(results.failures))
235    self.assertEquals(0, len(results.all_page_specific_values))
236
237  @decorators.Disabled('win')
238  def testPagesetRepeat(self):
239    ps = page_set.PageSet()
240    expectations = test_expectations.TestExpectations()
241    ps.pages.append(page_module.Page(
242        'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
243    ps.pages.append(page_module.Page(
244        'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir()))
245
246    class Measurement(page_test.PageTest):
247      i = 0
248      def ValidateAndMeasurePage(self, page, _, results):
249        self.i += 1
250        results.AddValue(scalar.ScalarValue(
251            page, 'metric', 'unit', self.i))
252
253    output_file = tempfile.NamedTemporaryFile(delete=False).name
254    try:
255      options = options_for_unittests.GetCopy()
256      options.output_formats = ['buildbot']
257      options.output_file = output_file
258      options.suppress_gtest_report = True
259      options.reset_results = None
260      options.upload_results = None
261      options.results_label = None
262
263      options.page_repeat = 1
264      options.pageset_repeat = 2
265      SetUpPageRunnerArguments(options)
266      results = results_options.CreateResults(EmptyMetadataForTest(), options)
267      page_runner.Run(Measurement(), ps, expectations, options, results)
268      results.PrintSummary()
269      self.assertEquals(4, len(GetSuccessfulPageRuns(results)))
270      self.assertEquals(0, len(results.failures))
271      with open(output_file) as f:
272        stdout = f.read()
273      self.assertIn('RESULT metric: blank.html= [1,3] unit', stdout)
274      self.assertIn('RESULT metric: green_rect.html= [2,4] unit', stdout)
275      self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout)
276    finally:
277      # TODO(chrishenry): This is a HACK!!1 Really, the right way to
278      # do this is for page_runner (or output formatter) to close any
279      # files it has opened.
280      for formatter in results._output_formatters:  # pylint: disable=W0212
281        formatter.output_stream.close()
282      os.remove(output_file)
283
284  def testCredentialsWhenLoginFails(self):
285    credentials_backend = StubCredentialsBackend(login_return_value=False)
286    did_run = self.runCredentialsTest(credentials_backend)
287    assert credentials_backend.did_get_login == True
288    assert credentials_backend.did_get_login_no_longer_needed == False
289    assert did_run == False
290
291  def testCredentialsWhenLoginSucceeds(self):
292    credentials_backend = StubCredentialsBackend(login_return_value=True)
293    did_run = self.runCredentialsTest(credentials_backend)
294    assert credentials_backend.did_get_login == True
295    assert credentials_backend.did_get_login_no_longer_needed == True
296    assert did_run
297
298  def runCredentialsTest(self, credentials_backend):
299    ps = page_set.PageSet()
300    expectations = test_expectations.TestExpectations()
301    page = page_module.Page(
302        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
303    page.credentials = "test"
304    ps.pages.append(page)
305
306    did_run = [False]
307
308    try:
309      with tempfile.NamedTemporaryFile(delete=False) as f:
310        f.write(SIMPLE_CREDENTIALS_STRING)
311        ps.credentials_path = f.name
312
313      class TestThatInstallsCredentialsBackend(page_test.PageTest):
314        def __init__(self, credentials_backend):
315          super(TestThatInstallsCredentialsBackend, self).__init__()
316          self._credentials_backend = credentials_backend
317
318        def DidStartBrowser(self, browser):
319          browser.credentials.AddBackend(self._credentials_backend)
320
321        def ValidatePage(self, *_):
322          did_run[0] = True
323
324      test = TestThatInstallsCredentialsBackend(credentials_backend)
325      options = options_for_unittests.GetCopy()
326      options.output_formats = ['none']
327      options.suppress_gtest_report = True
328      SetUpPageRunnerArguments(options)
329      results = results_options.CreateResults(EmptyMetadataForTest(), options)
330      page_runner.Run(test, ps, expectations, options, results)
331    finally:
332      os.remove(f.name)
333
334    return did_run[0]
335
336  def testUserAgent(self):
337    ps = page_set.PageSet()
338    expectations = test_expectations.TestExpectations()
339    page = page_module.Page(
340        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
341    ps.pages.append(page)
342    ps.user_agent_type = 'tablet'
343
344    class TestUserAgent(page_test.PageTest):
345      def ValidatePage(self, _1, tab, _2):
346        actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
347        expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
348        assert actual_user_agent.strip() == expected_user_agent
349
350        # This is so we can check later that the test actually made it into this
351        # function. Previously it was timing out before even getting here, which
352        # should fail, but since it skipped all the asserts, it slipped by.
353        self.hasRun = True # pylint: disable=W0201
354
355    test = TestUserAgent()
356    options = options_for_unittests.GetCopy()
357    options.output_formats = ['none']
358    options.suppress_gtest_report = True
359    SetUpPageRunnerArguments(options)
360    results = results_options.CreateResults(EmptyMetadataForTest(), options)
361    page_runner.Run(test, ps, expectations, options, results)
362
363    self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
364
365  # Ensure that page_runner forces exactly 1 tab before running a page.
366  @decorators.Enabled('has tabs')
367  def testOneTab(self):
368    ps = page_set.PageSet()
369    expectations = test_expectations.TestExpectations()
370    page = page_module.Page(
371        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
372    ps.pages.append(page)
373
374    class TestOneTab(page_test.PageTest):
375      def __init__(self):
376        super(TestOneTab, self).__init__()
377        self._browser = None
378
379      def DidStartBrowser(self, browser):
380        self._browser = browser
381        self._browser.tabs.New()
382
383      def ValidatePage(self, *_):
384        assert len(self._browser.tabs) == 1
385
386    test = TestOneTab()
387    options = options_for_unittests.GetCopy()
388    options.output_formats = ['none']
389    options.suppress_gtest_report = True
390    SetUpPageRunnerArguments(options)
391    results = results_options.CreateResults(EmptyMetadataForTest(), options)
392    page_runner.Run(test, ps, expectations, options, results)
393
394  # Ensure that page_runner allows the test to customize the browser before it
395  # launches.
396  def testBrowserBeforeLaunch(self):
397    ps = page_set.PageSet()
398    expectations = test_expectations.TestExpectations()
399    page = page_module.Page(
400        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
401    ps.pages.append(page)
402
403    class TestBeforeLaunch(page_test.PageTest):
404      def __init__(self):
405        super(TestBeforeLaunch, self).__init__()
406        self._did_call_will_start = False
407        self._did_call_did_start = False
408
409      def WillStartBrowser(self, platform):
410        self._did_call_will_start = True
411        # TODO(simonjam): Test that the profile is available.
412
413      def DidStartBrowser(self, browser):
414        assert self._did_call_will_start
415        self._did_call_did_start = True
416
417      def ValidatePage(self, *_):
418        assert self._did_call_did_start
419
420    test = TestBeforeLaunch()
421    options = options_for_unittests.GetCopy()
422    options.output_formats = ['none']
423    options.suppress_gtest_report = True
424    SetUpPageRunnerArguments(options)
425    results = results_options.CreateResults(EmptyMetadataForTest(), options)
426    page_runner.Run(test, ps, expectations, options, results)
427
428  def testRunPageWithStartupUrl(self):
429    ps = page_set.PageSet()
430    expectations = test_expectations.TestExpectations()
431    expectations = test_expectations.TestExpectations()
432    page = page_module.Page(
433        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
434    page.startup_url = 'about:blank'
435    ps.pages.append(page)
436
437    class Measurement(page_test.PageTest):
438      def __init__(self):
439        super(Measurement, self).__init__()
440        self.browser_restarted = False
441
442      def CustomizeBrowserOptionsForSinglePage(self, ps, options):
443        self.browser_restarted = True
444        super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps,
445                                                                      options)
446      def ValidateAndMeasurePage(self, page, tab, results):
447        pass
448
449    options = options_for_unittests.GetCopy()
450    options.page_repeat = 2
451    options.output_formats = ['none']
452    options.suppress_gtest_report = True
453    if not browser_finder.FindBrowser(options):
454      return
455    test = Measurement()
456    SetUpPageRunnerArguments(options)
457    results = results_options.CreateResults(EmptyMetadataForTest(), options)
458    page_runner.Run(test, ps, expectations, options, results)
459    self.assertEquals('about:blank', options.browser_options.startup_url)
460    self.assertTrue(test.browser_restarted)
461
462  # Ensure that page_runner calls cleanUp when a page run fails.
463  def testCleanUpPage(self):
464    ps = page_set.PageSet()
465    expectations = test_expectations.TestExpectations()
466    page = page_module.Page(
467        'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
468    ps.pages.append(page)
469
470    class Test(page_test.PageTest):
471      def __init__(self):
472        super(Test, self).__init__()
473        self.did_call_clean_up = False
474
475      def ValidatePage(self, *_):
476        raise exceptions.IntentionalException
477
478      def CleanUpAfterPage(self, page, tab):
479        self.did_call_clean_up = True
480
481
482    test = Test()
483    options = options_for_unittests.GetCopy()
484    options.output_formats = ['none']
485    options.suppress_gtest_report = True
486    SetUpPageRunnerArguments(options)
487    results = results_options.CreateResults(EmptyMetadataForTest(), options)
488    page_runner.Run(test, ps, expectations, options, results)
489    assert test.did_call_clean_up
490
491  # Ensure skipping the test if page cannot be run on the browser
492  def testPageCannotRunOnBrowser(self):
493    ps = page_set.PageSet()
494    expectations = test_expectations.TestExpectations()
495
496    class PageThatCannotRunOnBrowser(page_module.Page):
497
498      def __init__(self):
499        super(PageThatCannotRunOnBrowser, self).__init__(
500            url='file://blank.html', page_set=ps,
501            base_dir=util.GetUnittestDataDir())
502
503      def CanRunOnBrowser(self, _):
504        return False
505
506      def ValidatePage(self, _):
507        pass
508
509    class Test(page_test.PageTest):
510      def __init__(self, *args, **kwargs):
511        super(Test, self).__init__(*args, **kwargs)
512        self.will_navigate_to_page_called = False
513
514      def ValidatePage(self, *args):
515        pass
516
517      def WillNavigateToPage(self, _1, _2):
518        self.will_navigate_to_page_called = True
519
520    test = Test()
521    options = options_for_unittests.GetCopy()
522    options.output_formats = ['none']
523    options.suppress_gtest_report = True
524    SetUpPageRunnerArguments(options)
525    results = results_options.CreateResults(EmptyMetadataForTest(), options)
526    page_runner.Run(test, ps, expectations, options, results)
527    self.assertFalse(test.will_navigate_to_page_called)
528    self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
529    self.assertEquals(0, len(results.failures))
530
531  def TestUseLiveSitesFlag(self, options, expect_from_archive):
532    ps = page_set.PageSet(
533      file_path=util.GetUnittestDataDir(),
534      archive_data_file='data/archive_blank.json')
535    ps.pages.append(page_module.Page(
536      'file://blank.html', ps, base_dir=ps.base_dir))
537    expectations = test_expectations.TestExpectations()
538
539    class ArchiveTest(page_test.PageTest):
540      def __init__(self):
541        super(ArchiveTest, self).__init__()
542        self.is_page_from_archive = False
543        self.archive_path_exist = True
544
545      def WillNavigateToPage(self, page, tab):
546        self.archive_path_exist = (page.archive_path
547                                   and os.path.isfile(page.archive_path))
548        self.is_page_from_archive = (
549          tab.browser._wpr_server is not None) # pylint: disable=W0212
550
551      def ValidateAndMeasurePage(self, _, __, results):
552        pass
553
554    test = ArchiveTest()
555    results = results_options.CreateResults(EmptyMetadataForTest(), options)
556    try:
557      page_runner.Run(test, ps, expectations, options, results)
558      if expect_from_archive and not test.archive_path_exist:
559        logging.warning('archive path did not exist, asserting that page '
560                        'is from archive is skipped.')
561        return
562      self.assertEquals(expect_from_archive, test.is_page_from_archive)
563    finally:
564      for p in ps:
565        if os.path.isfile(p.archive_path):
566          os.remove(p.archive_path)
567
568
569  def testUseLiveSitesFlagSet(self):
570    options = options_for_unittests.GetCopy()
571    options.output_formats = ['none']
572    options.suppress_gtest_report = True
573    options.use_live_sites = True
574    SetUpPageRunnerArguments(options)
575    self.TestUseLiveSitesFlag(options, expect_from_archive=False)
576
577  def testUseLiveSitesFlagUnset(self):
578    options = options_for_unittests.GetCopy()
579    options.output_formats = ['none']
580    options.suppress_gtest_report = True
581    SetUpPageRunnerArguments(options)
582    self.TestUseLiveSitesFlag(options, expect_from_archive=True)
583
584  def _testMaxFailuresOptionIsRespectedAndOverridable(self, max_failures=None):
585    class TestPage(page_module.Page):
586      def __init__(self, *args, **kwargs):
587        super(TestPage, self).__init__(*args, **kwargs)
588        self.was_run = False
589
590      def RunNavigateSteps(self, action_runner): # pylint: disable=W0613
591        self.was_run = True
592        raise Exception('Test exception')
593
594    class Test(page_test.PageTest):
595      def ValidatePage(self, *args):
596        pass
597
598    ps = page_set.PageSet()
599    expectations = test_expectations.TestExpectations()
600    for ii in range(5):
601      ps.pages.append(TestPage(
602          'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
603
604    options = options_for_unittests.GetCopy()
605    options.output_formats = ['none']
606    options.suppress_gtest_report = True
607    expected_max_failures = 2
608    if not max_failures is None:
609      options.max_failures = max_failures
610      expected_max_failures = max_failures
611    SetUpPageRunnerArguments(options)
612    results = results_options.CreateResults(EmptyMetadataForTest(), options)
613    page_runner.Run(Test(max_failures=2),
614                    ps, expectations, options, results)
615    self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
616    # Runs up to max_failures+1 failing tests before stopping, since
617    # every tests after max_failures failures have been encountered
618    # may all be passing.
619    self.assertEquals(expected_max_failures + 1, len(results.failures))
620    for ii in range(len(ps.pages)):
621      if ii <= expected_max_failures:
622        self.assertTrue(ps.pages[ii].was_run)
623      else:
624        self.assertFalse(ps.pages[ii].was_run)
625
626  def testMaxFailuresOptionIsRespected(self):
627    self._testMaxFailuresOptionIsRespectedAndOverridable()
628
629  def testMaxFailuresOptionIsOverridable(self):
630    self._testMaxFailuresOptionIsRespectedAndOverridable(1)
631