1a5728872c7702ddd09537c95bc3cbd20e1f2fb09Daniel Dunbar# Copyright 2014 The Chromium Authors. All rights reserved.
28e8fb3be5bd78f0564444eca02b404566a5f3b5dAndy Gibbs# Use of this source code is governed by a BSD-style license that can be
3bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman# found in the LICENSE file.
4bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
5bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanimport os
6bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanimport traceback
7bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
8bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.page import page_set
9bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.results import base_test_results_unittest
10bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.results import gtest_progress_reporter
11bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.results import page_test_results
12bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.unittest import simple_mock
13bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.value import failure
14bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.value import skip
15bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
16bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
17bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmandef _MakePageSet():
18bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  ps = page_set.PageSet(file_path=os.path.dirname(__file__))
19bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  ps.AddPageWithDefaultRunNavigate('http://www.foo.com/')
20bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  ps.AddPageWithDefaultRunNavigate('http://www.bar.com/')
21bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  ps.AddPageWithDefaultRunNavigate('http://www.baz.com/')
22bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  ps.AddPageWithDefaultRunNavigate('http://www.roz.com/')
23bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  return ps
24bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
25bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
26bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanclass GTestProgressReporterTest(
27bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman    base_test_results_unittest.BaseTestResultsUnittest):
28bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman
29bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman  def setUp(self):
30bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman    super(GTestProgressReporterTest, self).setUp()
31    self._mock_timer = simple_mock.MockTimer(gtest_progress_reporter)
32
33    self._output_stream = base_test_results_unittest.TestOutputStream()
34    self._reporter = gtest_progress_reporter.GTestProgressReporter(
35        self._output_stream)
36
37  def tearDown(self):
38    self._mock_timer.Restore()
39
40  def testSingleSuccessPage(self):
41    test_page_set = _MakePageSet()
42
43    results = page_test_results.PageTestResults(
44        progress_reporter=self._reporter)
45    results.WillRunPage(test_page_set.pages[0])
46    self._mock_timer.SetTime(0.007)
47    results.DidRunPage(test_page_set.pages[0])
48
49    results.PrintSummary()
50    expected = ('[ RUN      ] http://www.foo.com/\n'
51                '[       OK ] http://www.foo.com/ (7 ms)\n'
52                '[  PASSED  ] 1 test.\n\n')
53    self.assertEquals(expected, ''.join(self._output_stream.output_data))
54
55  def testSingleFailedPage(self):
56    test_page_set = _MakePageSet()
57
58    results = page_test_results.PageTestResults(
59        progress_reporter=self._reporter)
60    results.WillRunPage(test_page_set.pages[0])
61    exc_info = self.CreateException()
62    results.AddValue(failure.FailureValue(test_page_set.pages[0], exc_info))
63    results.DidRunPage(test_page_set.pages[0])
64
65    results.PrintSummary()
66    exception_trace = ''.join(traceback.format_exception(*exc_info))
67    expected = ('[ RUN      ] http://www.foo.com/\n'
68                '%s\n'
69                '[  FAILED  ] http://www.foo.com/ (0 ms)\n'
70                '[  PASSED  ] 0 tests.\n'
71                '[  FAILED  ] 1 test, listed below:\n'
72                '[  FAILED  ]  http://www.foo.com/\n\n'
73                '1 FAILED TEST\n\n' % exception_trace)
74    self.assertEquals(expected, ''.join(self._output_stream.output_data))
75
76  def testSingleSkippedPage(self):
77    test_page_set = _MakePageSet()
78    results = page_test_results.PageTestResults(
79        progress_reporter=self._reporter)
80    results.WillRunPage(test_page_set.pages[0])
81    self._mock_timer.SetTime(0.007)
82    results.AddValue(skip.SkipValue(test_page_set.pages[0],
83        'Page skipped for testing reason'))
84    results.DidRunPage(test_page_set.pages[0])
85
86    results.PrintSummary()
87    expected = ('[ RUN      ] http://www.foo.com/\n'
88                '===== SKIPPING TEST http://www.foo.com/:'
89                ' Page skipped for testing reason =====\n'
90                '[       OK ] http://www.foo.com/ (7 ms)\n'
91                '[  PASSED  ] 1 test.\n\n')
92    self.assertEquals(expected, ''.join(self._output_stream.output_data))
93
94  def testPassAndFailedPages(self):
95    test_page_set = _MakePageSet()
96    results = page_test_results.PageTestResults(
97        progress_reporter=self._reporter)
98    exc_info = self.CreateException()
99
100    results.WillRunPage(test_page_set.pages[0])
101    self._mock_timer.SetTime(0.007)
102    results.DidRunPage(test_page_set.pages[0])
103
104    results.WillRunPage(test_page_set.pages[1])
105    self._mock_timer.SetTime(0.009)
106    results.AddValue(failure.FailureValue(test_page_set.pages[1], exc_info))
107    results.DidRunPage(test_page_set.pages[1])
108
109    results.WillRunPage(test_page_set.pages[2])
110    self._mock_timer.SetTime(0.015)
111    results.AddValue(failure.FailureValue(test_page_set.pages[2], exc_info))
112    results.DidRunPage(test_page_set.pages[2])
113
114    results.WillRunPage(test_page_set.pages[3])
115    self._mock_timer.SetTime(0.020)
116    results.DidRunPage(test_page_set.pages[3])
117
118    results.PrintSummary()
119    exception_trace = ''.join(traceback.format_exception(*exc_info))
120    expected = ('[ RUN      ] http://www.foo.com/\n'
121                '[       OK ] http://www.foo.com/ (7 ms)\n'
122                '[ RUN      ] http://www.bar.com/\n'
123                '%s\n'
124                '[  FAILED  ] http://www.bar.com/ (2 ms)\n'
125                '[ RUN      ] http://www.baz.com/\n'
126                '%s\n'
127                '[  FAILED  ] http://www.baz.com/ (6 ms)\n'
128                '[ RUN      ] http://www.roz.com/\n'
129                '[       OK ] http://www.roz.com/ (5 ms)\n'
130                '[  PASSED  ] 2 tests.\n'
131                '[  FAILED  ] 2 tests, listed below:\n'
132                '[  FAILED  ]  http://www.bar.com/\n'
133                '[  FAILED  ]  http://www.baz.com/\n\n'
134                '2 FAILED TESTS\n\n' % (exception_trace, exception_trace))
135    self.assertEquals(expected, ''.join(self._output_stream.output_data))
136
137  def testWillAttemptPageRun(self):
138    test_page_set = _MakePageSet()
139
140    results = page_test_results.PageTestResults(
141        progress_reporter=self._reporter)
142    results.WillRunPage(test_page_set.pages[0])
143    results.WillAttemptPageRun(1, 5)
144    results.WillAttemptPageRun(2, 5)
145    results.WillAttemptPageRun(3, 5)
146    self._mock_timer.SetTime(0.007)
147    results.DidRunPage(test_page_set.pages[0])
148
149    results.PrintSummary()
150    expected = ('[ RUN      ] http://www.foo.com/\n'
151                '===== RETRYING PAGE RUN (attempt 2 out of 5 allowed) =====\n'
152                'Page run attempt failed and will be retried.'
153                ' Discarding previous results.\n'
154                '===== RETRYING PAGE RUN (attempt 3 out of 5 allowed) =====\n'
155                'Page run attempt failed and will be retried.'
156                ' Discarding previous results.\n'
157                '[       OK ] http://www.foo.com/ (7 ms)\n'
158                '[  PASSED  ] 1 test.\n\n')
159    self.assertEquals(expected, ''.join(self._output_stream.output_data))
160
161  def testStreamingResults(self):
162    test_page_set = _MakePageSet()
163    results = page_test_results.PageTestResults(
164        progress_reporter=self._reporter)
165    exc_info = self.CreateException()
166
167    results.WillRunPage(test_page_set.pages[0])
168    self._mock_timer.SetTime(0.007)
169    results.DidRunPage(test_page_set.pages[0])
170    expected = ('[ RUN      ] http://www.foo.com/\n'
171                '[       OK ] http://www.foo.com/ (7 ms)\n')
172    self.assertEquals(expected, ''.join(self._output_stream.output_data))
173
174    results.WillRunPage(test_page_set.pages[1])
175    self._mock_timer.SetTime(0.009)
176    exception_trace = ''.join(traceback.format_exception(*exc_info))
177    results.AddValue(failure.FailureValue(test_page_set.pages[1], exc_info))
178    results.DidRunPage(test_page_set.pages[1])
179    expected = ('[ RUN      ] http://www.foo.com/\n'
180                '[       OK ] http://www.foo.com/ (7 ms)\n'
181                '[ RUN      ] http://www.bar.com/\n'
182                '%s\n'
183                '[  FAILED  ] http://www.bar.com/ (2 ms)\n' % exception_trace)
184
185  def testOutputSkipInformation(self):
186    test_page_set = _MakePageSet()
187    self._reporter = gtest_progress_reporter.GTestProgressReporter(
188        self._output_stream, output_skipped_tests_summary=True)
189    results = page_test_results.PageTestResults(
190        progress_reporter=self._reporter)
191    results.WillRunPage(test_page_set.pages[0])
192    self._mock_timer.SetTime(0.007)
193    results.AddValue(skip.SkipValue(test_page_set.pages[0],
194        'Page skipped for testing reason'))
195    results.DidRunPage(test_page_set.pages[0])
196
197    results.PrintSummary()
198    expected = ('[ RUN      ] http://www.foo.com/\n'
199                '===== SKIPPING TEST http://www.foo.com/:'
200                ' Page skipped for testing reason =====\n'
201                '[       OK ] http://www.foo.com/ (7 ms)\n'
202                '[  PASSED  ] 1 test.\n'
203                '\n'
204                'Skipped pages:\n'
205                'http://www.foo.com/\n'
206                '\n')
207    self.assertEquals(expected, ''.join(self._output_stream.output_data))
208