1a5728872c7702ddd09537c95bc3cbd20e1f2fb09Daniel Dunbar# Copyright 2014 The Chromium Authors. All rights reserved. 28e8fb3be5bd78f0564444eca02b404566a5f3b5dAndy Gibbs# Use of this source code is governed by a BSD-style license that can be 3bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman# found in the LICENSE file. 4bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 5bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanimport os 6bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanimport traceback 7bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 8bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.page import page_set 9bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.results import base_test_results_unittest 10bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.results import gtest_progress_reporter 11bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.results import page_test_results 12bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.unittest import simple_mock 13bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.value import failure 14bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanfrom telemetry.value import skip 15bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 16bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 17bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmandef _MakePageSet(): 18bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman ps = page_set.PageSet(file_path=os.path.dirname(__file__)) 19bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman ps.AddPageWithDefaultRunNavigate('http://www.foo.com/') 20bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman ps.AddPageWithDefaultRunNavigate('http://www.bar.com/') 21bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman ps.AddPageWithDefaultRunNavigate('http://www.baz.com/') 22bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman ps.AddPageWithDefaultRunNavigate('http://www.roz.com/') 23bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman return ps 24bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 25bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 26bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedmanclass GTestProgressReporterTest( 27bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman base_test_results_unittest.BaseTestResultsUnittest): 28bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman 29bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman def setUp(self): 30bc4e29f307f86ddbc2f31d9530da79ad9b0c6b7bEli Friedman super(GTestProgressReporterTest, self).setUp() 31 self._mock_timer = simple_mock.MockTimer(gtest_progress_reporter) 32 33 self._output_stream = base_test_results_unittest.TestOutputStream() 34 self._reporter = gtest_progress_reporter.GTestProgressReporter( 35 self._output_stream) 36 37 def tearDown(self): 38 self._mock_timer.Restore() 39 40 def testSingleSuccessPage(self): 41 test_page_set = _MakePageSet() 42 43 results = page_test_results.PageTestResults( 44 progress_reporter=self._reporter) 45 results.WillRunPage(test_page_set.pages[0]) 46 self._mock_timer.SetTime(0.007) 47 results.DidRunPage(test_page_set.pages[0]) 48 49 results.PrintSummary() 50 expected = ('[ RUN ] http://www.foo.com/\n' 51 '[ OK ] http://www.foo.com/ (7 ms)\n' 52 '[ PASSED ] 1 test.\n\n') 53 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 54 55 def testSingleFailedPage(self): 56 test_page_set = _MakePageSet() 57 58 results = page_test_results.PageTestResults( 59 progress_reporter=self._reporter) 60 results.WillRunPage(test_page_set.pages[0]) 61 exc_info = self.CreateException() 62 results.AddValue(failure.FailureValue(test_page_set.pages[0], exc_info)) 63 results.DidRunPage(test_page_set.pages[0]) 64 65 results.PrintSummary() 66 exception_trace = ''.join(traceback.format_exception(*exc_info)) 67 expected = ('[ RUN ] http://www.foo.com/\n' 68 '%s\n' 69 '[ FAILED ] http://www.foo.com/ (0 ms)\n' 70 '[ PASSED ] 0 tests.\n' 71 '[ FAILED ] 1 test, listed below:\n' 72 '[ FAILED ] http://www.foo.com/\n\n' 73 '1 FAILED TEST\n\n' % exception_trace) 74 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 75 76 def testSingleSkippedPage(self): 77 test_page_set = _MakePageSet() 78 results = page_test_results.PageTestResults( 79 progress_reporter=self._reporter) 80 results.WillRunPage(test_page_set.pages[0]) 81 self._mock_timer.SetTime(0.007) 82 results.AddValue(skip.SkipValue(test_page_set.pages[0], 83 'Page skipped for testing reason')) 84 results.DidRunPage(test_page_set.pages[0]) 85 86 results.PrintSummary() 87 expected = ('[ RUN ] http://www.foo.com/\n' 88 '===== SKIPPING TEST http://www.foo.com/:' 89 ' Page skipped for testing reason =====\n' 90 '[ OK ] http://www.foo.com/ (7 ms)\n' 91 '[ PASSED ] 1 test.\n\n') 92 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 93 94 def testPassAndFailedPages(self): 95 test_page_set = _MakePageSet() 96 results = page_test_results.PageTestResults( 97 progress_reporter=self._reporter) 98 exc_info = self.CreateException() 99 100 results.WillRunPage(test_page_set.pages[0]) 101 self._mock_timer.SetTime(0.007) 102 results.DidRunPage(test_page_set.pages[0]) 103 104 results.WillRunPage(test_page_set.pages[1]) 105 self._mock_timer.SetTime(0.009) 106 results.AddValue(failure.FailureValue(test_page_set.pages[1], exc_info)) 107 results.DidRunPage(test_page_set.pages[1]) 108 109 results.WillRunPage(test_page_set.pages[2]) 110 self._mock_timer.SetTime(0.015) 111 results.AddValue(failure.FailureValue(test_page_set.pages[2], exc_info)) 112 results.DidRunPage(test_page_set.pages[2]) 113 114 results.WillRunPage(test_page_set.pages[3]) 115 self._mock_timer.SetTime(0.020) 116 results.DidRunPage(test_page_set.pages[3]) 117 118 results.PrintSummary() 119 exception_trace = ''.join(traceback.format_exception(*exc_info)) 120 expected = ('[ RUN ] http://www.foo.com/\n' 121 '[ OK ] http://www.foo.com/ (7 ms)\n' 122 '[ RUN ] http://www.bar.com/\n' 123 '%s\n' 124 '[ FAILED ] http://www.bar.com/ (2 ms)\n' 125 '[ RUN ] http://www.baz.com/\n' 126 '%s\n' 127 '[ FAILED ] http://www.baz.com/ (6 ms)\n' 128 '[ RUN ] http://www.roz.com/\n' 129 '[ OK ] http://www.roz.com/ (5 ms)\n' 130 '[ PASSED ] 2 tests.\n' 131 '[ FAILED ] 2 tests, listed below:\n' 132 '[ FAILED ] http://www.bar.com/\n' 133 '[ FAILED ] http://www.baz.com/\n\n' 134 '2 FAILED TESTS\n\n' % (exception_trace, exception_trace)) 135 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 136 137 def testWillAttemptPageRun(self): 138 test_page_set = _MakePageSet() 139 140 results = page_test_results.PageTestResults( 141 progress_reporter=self._reporter) 142 results.WillRunPage(test_page_set.pages[0]) 143 results.WillAttemptPageRun(1, 5) 144 results.WillAttemptPageRun(2, 5) 145 results.WillAttemptPageRun(3, 5) 146 self._mock_timer.SetTime(0.007) 147 results.DidRunPage(test_page_set.pages[0]) 148 149 results.PrintSummary() 150 expected = ('[ RUN ] http://www.foo.com/\n' 151 '===== RETRYING PAGE RUN (attempt 2 out of 5 allowed) =====\n' 152 'Page run attempt failed and will be retried.' 153 ' Discarding previous results.\n' 154 '===== RETRYING PAGE RUN (attempt 3 out of 5 allowed) =====\n' 155 'Page run attempt failed and will be retried.' 156 ' Discarding previous results.\n' 157 '[ OK ] http://www.foo.com/ (7 ms)\n' 158 '[ PASSED ] 1 test.\n\n') 159 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 160 161 def testStreamingResults(self): 162 test_page_set = _MakePageSet() 163 results = page_test_results.PageTestResults( 164 progress_reporter=self._reporter) 165 exc_info = self.CreateException() 166 167 results.WillRunPage(test_page_set.pages[0]) 168 self._mock_timer.SetTime(0.007) 169 results.DidRunPage(test_page_set.pages[0]) 170 expected = ('[ RUN ] http://www.foo.com/\n' 171 '[ OK ] http://www.foo.com/ (7 ms)\n') 172 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 173 174 results.WillRunPage(test_page_set.pages[1]) 175 self._mock_timer.SetTime(0.009) 176 exception_trace = ''.join(traceback.format_exception(*exc_info)) 177 results.AddValue(failure.FailureValue(test_page_set.pages[1], exc_info)) 178 results.DidRunPage(test_page_set.pages[1]) 179 expected = ('[ RUN ] http://www.foo.com/\n' 180 '[ OK ] http://www.foo.com/ (7 ms)\n' 181 '[ RUN ] http://www.bar.com/\n' 182 '%s\n' 183 '[ FAILED ] http://www.bar.com/ (2 ms)\n' % exception_trace) 184 185 def testOutputSkipInformation(self): 186 test_page_set = _MakePageSet() 187 self._reporter = gtest_progress_reporter.GTestProgressReporter( 188 self._output_stream, output_skipped_tests_summary=True) 189 results = page_test_results.PageTestResults( 190 progress_reporter=self._reporter) 191 results.WillRunPage(test_page_set.pages[0]) 192 self._mock_timer.SetTime(0.007) 193 results.AddValue(skip.SkipValue(test_page_set.pages[0], 194 'Page skipped for testing reason')) 195 results.DidRunPage(test_page_set.pages[0]) 196 197 results.PrintSummary() 198 expected = ('[ RUN ] http://www.foo.com/\n' 199 '===== SKIPPING TEST http://www.foo.com/:' 200 ' Page skipped for testing reason =====\n' 201 '[ OK ] http://www.foo.com/ (7 ms)\n' 202 '[ PASSED ] 1 test.\n' 203 '\n' 204 'Skipped pages:\n' 205 'http://www.foo.com/\n' 206 '\n') 207 self.assertEquals(expected, ''.join(self._output_stream.output_data)) 208