1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import datetime 6import unittest 7 8import mock 9import webapp2 10import webtest 11 12from dashboard import auto_bisect 13from dashboard import request_handler 14from dashboard import testing_common 15from dashboard import utils 16from dashboard.models import anomaly 17from dashboard.models import try_job 18 19 20@mock.patch.object(utils, 'TickMonitoringCustomMetric', mock.MagicMock()) 21class AutoBisectTest(testing_common.TestCase): 22 23 def setUp(self): 24 super(AutoBisectTest, self).setUp() 25 app = webapp2.WSGIApplication( 26 [('/auto_bisect', auto_bisect.AutoBisectHandler)]) 27 testing_common.SetIsInternalUser('internal@chromium.org', True) 28 self.testapp = webtest.TestApp(app) 29 self.SetCurrentUser('internal@chromium.org') 30 31 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 32 def testPost_FailedJobRunTwice_JobRestarted(self, mock_perform_bisect): 33 testing_common.AddTests( 34 ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}}) 35 test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score') 36 anomaly.Anomaly( 37 bug_id=111, test=test_key, 38 start_revision=300100, end_revision=300200, 39 median_before_anomaly=100, median_after_anomaly=200).put() 40 try_job.TryJob( 41 bug_id=111, status='failed', 42 last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8), 43 run_count=2).put() 44 self.testapp.post('/auto_bisect') 45 mock_perform_bisect.assert_called_once_with( 46 try_job.TryJob.query(try_job.TryJob.bug_id == 111).get()) 47 48 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 49 def testPost_FailedJobRunOnce_JobRestarted(self, mock_perform_bisect): 50 try_job.TryJob( 51 bug_id=222, status='failed', 52 last_ran_timestamp=datetime.datetime.now(), 53 run_count=1).put() 54 self.testapp.post('/auto_bisect') 55 mock_perform_bisect.assert_called_once_with( 56 try_job.TryJob.query(try_job.TryJob.bug_id == 222).get()) 57 58 @mock.patch.object(auto_bisect.start_try_job, 'LogBisectResult') 59 def testPost_JobRunTooManyTimes_LogsMessage(self, mock_log_result): 60 job_key = try_job.TryJob( 61 bug_id=333, status='failed', 62 last_ran_timestamp=datetime.datetime.now(), 63 run_count=len(auto_bisect._BISECT_RESTART_PERIOD_DAYS) + 1).put() 64 self.testapp.post('/auto_bisect') 65 self.assertIsNone(job_key.get()) 66 mock_log_result.assert_called_once_with(333, mock.ANY) 67 68 def testGet_WithStatsParameter_ListsTryJobs(self): 69 now = datetime.datetime.now() 70 try_job.TryJob( 71 bug_id=222, status='failed', 72 last_ran_timestamp=now, run_count=2).put() 73 try_job.TryJob( 74 bug_id=444, status='started', 75 last_ran_timestamp=now, run_count=1).put() 76 try_job.TryJob( 77 bug_id=777, status='started', 78 last_ran_timestamp=now, run_count=1).put() 79 try_job.TryJob( 80 bug_id=555, status=None, 81 last_ran_timestamp=now, run_count=1).put() 82 response = self.testapp.get('/auto_bisect?stats') 83 self.assertIn('Failed jobs: 1', response.body) 84 self.assertIn('Started jobs: 2', response.body) 85 86 87class StartNewBisectForBugTest(testing_common.TestCase): 88 89 def setUp(self): 90 super(StartNewBisectForBugTest, self).setUp() 91 self.SetCurrentUser('internal@chromium.org') 92 93 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 94 def testStartNewBisectForBug_StartsBisect(self, mock_perform_bisect): 95 testing_common.AddTests( 96 ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}}) 97 test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score') 98 anomaly.Anomaly( 99 bug_id=111, test=test_key, 100 start_revision=300100, end_revision=300200, 101 median_before_anomaly=100, median_after_anomaly=200).put() 102 auto_bisect.StartNewBisectForBug(111) 103 job = try_job.TryJob.query(try_job.TryJob.bug_id == 111).get() 104 mock_perform_bisect.assert_called_once_with(job) 105 106 def testStartNewBisectForBug_RevisionTooLow_ReturnsError(self): 107 testing_common.AddTests( 108 ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}}) 109 test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score') 110 anomaly.Anomaly( 111 bug_id=222, test=test_key, 112 start_revision=1200, end_revision=1250, 113 median_before_anomaly=100, median_after_anomaly=200).put() 114 result = auto_bisect.StartNewBisectForBug(222) 115 self.assertEqual({'error': 'Invalid "good" revision: 1199.'}, result) 116 117 @mock.patch.object( 118 auto_bisect.start_try_job, 'PerformBisect', 119 mock.MagicMock(side_effect=request_handler.InvalidInputError( 120 'Some reason'))) 121 def testStartNewBisectForBug_InvalidInputErrorRaised_ReturnsError(self): 122 testing_common.AddTests(['Foo'], ['bar'], {'sunspider': {'score': {}}}) 123 test_key = utils.TestKey('Foo/bar/sunspider/score') 124 anomaly.Anomaly( 125 bug_id=345, test=test_key, 126 start_revision=300100, end_revision=300200, 127 median_before_anomaly=100, median_after_anomaly=200).put() 128 result = auto_bisect.StartNewBisectForBug(345) 129 self.assertEqual({'error': 'Some reason'}, result) 130 131 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 132 def testStartNewBisectForBug_WithDefaultRevs_StartsBisect( 133 self, mock_perform_bisect): 134 testing_common.AddTests( 135 ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}}) 136 test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score') 137 testing_common.AddRows( 138 'ChromiumPerf/linux-release/sunspider/score', 139 { 140 1199: { 141 'a_default_rev': 'r_foo', 142 'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857' 143 }, 144 1250: { 145 'a_default_rev': 'r_foo', 146 'r_foo': 'fc34e5346446854637311ad7793a95d56e314042' 147 } 148 }) 149 anomaly.Anomaly( 150 bug_id=333, test=test_key, 151 start_revision=1200, end_revision=1250, 152 median_before_anomaly=100, median_after_anomaly=200).put() 153 auto_bisect.StartNewBisectForBug(333) 154 job = try_job.TryJob.query(try_job.TryJob.bug_id == 333).get() 155 mock_perform_bisect.assert_called_once_with(job) 156 157 def testStartNewBisectForBug_UnbisectableTest_ReturnsError(self): 158 testing_common.AddTests(['V8'], ['x86'], {'v8': {'sunspider': {}}}) 159 # The test suite "v8" is in the black-list of test suite names. 160 test_key = utils.TestKey('V8/x86/v8/sunspider') 161 anomaly.Anomaly( 162 bug_id=444, test=test_key, 163 start_revision=155000, end_revision=155100, 164 median_before_anomaly=100, median_after_anomaly=200).put() 165 result = auto_bisect.StartNewBisectForBug(444) 166 self.assertEqual({'error': 'Could not select a test.'}, result) 167 168 169class TickMonitoringCustomMetricTest(testing_common.TestCase): 170 171 def setUp(self): 172 super(TickMonitoringCustomMetricTest, self).setUp() 173 app = webapp2.WSGIApplication( 174 [('/auto_bisect', auto_bisect.AutoBisectHandler)]) 175 self.testapp = webtest.TestApp(app) 176 177 @mock.patch.object(utils, 'TickMonitoringCustomMetric') 178 def testPost_NoTryJobs_CustomMetricTicked(self, mock_tick): 179 self.testapp.post('/auto_bisect') 180 mock_tick.assert_called_once_with('RestartFailedBisectJobs') 181 182 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 183 @mock.patch.object(utils, 'TickMonitoringCustomMetric') 184 def testPost_RunCount1_ExceptionInPerformBisect_CustomMetricNotTicked( 185 self, mock_tick, mock_perform_bisect): 186 mock_perform_bisect.side_effect = request_handler.InvalidInputError() 187 try_job.TryJob( 188 bug_id=222, status='failed', 189 last_ran_timestamp=datetime.datetime.now(), 190 run_count=1).put() 191 self.testapp.post('/auto_bisect') 192 self.assertEqual(0, mock_tick.call_count) 193 194 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 195 @mock.patch.object(utils, 'TickMonitoringCustomMetric') 196 def testPost_RunCount2_ExceptionInPerformBisect_CustomMetricNotTicked( 197 self, mock_tick, mock_perform_bisect): 198 mock_perform_bisect.side_effect = request_handler.InvalidInputError() 199 try_job.TryJob( 200 bug_id=111, status='failed', 201 last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8), 202 run_count=2).put() 203 self.testapp.post('/auto_bisect') 204 self.assertEqual(0, mock_tick.call_count) 205 206 @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect') 207 @mock.patch.object(utils, 'TickMonitoringCustomMetric') 208 def testPost_NoExceptionInPerformBisect_CustomMetricTicked( 209 self, mock_tick, mock_perform_bisect): 210 try_job.TryJob( 211 bug_id=222, status='failed', 212 last_ran_timestamp=datetime.datetime.now(), 213 run_count=1).put() 214 self.testapp.post('/auto_bisect') 215 self.assertEqual(1, mock_perform_bisect.call_count) 216 mock_tick.assert_called_once_with('RestartFailedBisectJobs') 217 218 219if __name__ == '__main__': 220 unittest.main() 221