1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import datetime
6import unittest
7
8import mock
9import webapp2
10import webtest
11
12from dashboard import auto_bisect
13from dashboard import request_handler
14from dashboard import testing_common
15from dashboard import utils
16from dashboard.models import anomaly
17from dashboard.models import try_job
18
19
20@mock.patch.object(utils, 'TickMonitoringCustomMetric', mock.MagicMock())
21class AutoBisectTest(testing_common.TestCase):
22
23  def setUp(self):
24    super(AutoBisectTest, self).setUp()
25    app = webapp2.WSGIApplication(
26        [('/auto_bisect', auto_bisect.AutoBisectHandler)])
27    testing_common.SetIsInternalUser('internal@chromium.org', True)
28    self.testapp = webtest.TestApp(app)
29    self.SetCurrentUser('internal@chromium.org')
30
31  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
32  def testPost_FailedJobRunTwice_JobRestarted(self, mock_perform_bisect):
33    testing_common.AddTests(
34        ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
35    test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
36    anomaly.Anomaly(
37        bug_id=111, test=test_key,
38        start_revision=300100, end_revision=300200,
39        median_before_anomaly=100, median_after_anomaly=200).put()
40    try_job.TryJob(
41        bug_id=111, status='failed',
42        last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8),
43        run_count=2).put()
44    self.testapp.post('/auto_bisect')
45    mock_perform_bisect.assert_called_once_with(
46        try_job.TryJob.query(try_job.TryJob.bug_id == 111).get())
47
48  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
49  def testPost_FailedJobRunOnce_JobRestarted(self, mock_perform_bisect):
50    try_job.TryJob(
51        bug_id=222, status='failed',
52        last_ran_timestamp=datetime.datetime.now(),
53        run_count=1).put()
54    self.testapp.post('/auto_bisect')
55    mock_perform_bisect.assert_called_once_with(
56        try_job.TryJob.query(try_job.TryJob.bug_id == 222).get())
57
58  @mock.patch.object(auto_bisect.start_try_job, 'LogBisectResult')
59  def testPost_JobRunTooManyTimes_LogsMessage(self, mock_log_result):
60    job_key = try_job.TryJob(
61        bug_id=333, status='failed',
62        last_ran_timestamp=datetime.datetime.now(),
63        run_count=len(auto_bisect._BISECT_RESTART_PERIOD_DAYS) + 1).put()
64    self.testapp.post('/auto_bisect')
65    self.assertIsNone(job_key.get())
66    mock_log_result.assert_called_once_with(333, mock.ANY)
67
68  def testGet_WithStatsParameter_ListsTryJobs(self):
69    now = datetime.datetime.now()
70    try_job.TryJob(
71        bug_id=222, status='failed',
72        last_ran_timestamp=now, run_count=2).put()
73    try_job.TryJob(
74        bug_id=444, status='started',
75        last_ran_timestamp=now, run_count=1).put()
76    try_job.TryJob(
77        bug_id=777, status='started',
78        last_ran_timestamp=now, run_count=1).put()
79    try_job.TryJob(
80        bug_id=555, status=None,
81        last_ran_timestamp=now, run_count=1).put()
82    response = self.testapp.get('/auto_bisect?stats')
83    self.assertIn('Failed jobs: 1', response.body)
84    self.assertIn('Started jobs: 2', response.body)
85
86
87class StartNewBisectForBugTest(testing_common.TestCase):
88
89  def setUp(self):
90    super(StartNewBisectForBugTest, self).setUp()
91    self.SetCurrentUser('internal@chromium.org')
92
93  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
94  def testStartNewBisectForBug_StartsBisect(self, mock_perform_bisect):
95    testing_common.AddTests(
96        ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
97    test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
98    anomaly.Anomaly(
99        bug_id=111, test=test_key,
100        start_revision=300100, end_revision=300200,
101        median_before_anomaly=100, median_after_anomaly=200).put()
102    auto_bisect.StartNewBisectForBug(111)
103    job = try_job.TryJob.query(try_job.TryJob.bug_id == 111).get()
104    mock_perform_bisect.assert_called_once_with(job)
105
106  def testStartNewBisectForBug_RevisionTooLow_ReturnsError(self):
107    testing_common.AddTests(
108        ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
109    test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
110    anomaly.Anomaly(
111        bug_id=222, test=test_key,
112        start_revision=1200, end_revision=1250,
113        median_before_anomaly=100, median_after_anomaly=200).put()
114    result = auto_bisect.StartNewBisectForBug(222)
115    self.assertEqual({'error': 'Invalid "good" revision: 1199.'}, result)
116
117  @mock.patch.object(
118      auto_bisect.start_try_job, 'PerformBisect',
119      mock.MagicMock(side_effect=request_handler.InvalidInputError(
120          'Some reason')))
121  def testStartNewBisectForBug_InvalidInputErrorRaised_ReturnsError(self):
122    testing_common.AddTests(['Foo'], ['bar'], {'sunspider': {'score': {}}})
123    test_key = utils.TestKey('Foo/bar/sunspider/score')
124    anomaly.Anomaly(
125        bug_id=345, test=test_key,
126        start_revision=300100, end_revision=300200,
127        median_before_anomaly=100, median_after_anomaly=200).put()
128    result = auto_bisect.StartNewBisectForBug(345)
129    self.assertEqual({'error': 'Some reason'}, result)
130
131  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
132  def testStartNewBisectForBug_WithDefaultRevs_StartsBisect(
133      self, mock_perform_bisect):
134    testing_common.AddTests(
135        ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
136    test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
137    testing_common.AddRows(
138        'ChromiumPerf/linux-release/sunspider/score',
139        {
140            1199: {
141                'a_default_rev': 'r_foo',
142                'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857'
143            },
144            1250: {
145                'a_default_rev': 'r_foo',
146                'r_foo': 'fc34e5346446854637311ad7793a95d56e314042'
147            }
148        })
149    anomaly.Anomaly(
150        bug_id=333, test=test_key,
151        start_revision=1200, end_revision=1250,
152        median_before_anomaly=100, median_after_anomaly=200).put()
153    auto_bisect.StartNewBisectForBug(333)
154    job = try_job.TryJob.query(try_job.TryJob.bug_id == 333).get()
155    mock_perform_bisect.assert_called_once_with(job)
156
157  def testStartNewBisectForBug_UnbisectableTest_ReturnsError(self):
158    testing_common.AddTests(['V8'], ['x86'], {'v8': {'sunspider': {}}})
159    # The test suite "v8" is in the black-list of test suite names.
160    test_key = utils.TestKey('V8/x86/v8/sunspider')
161    anomaly.Anomaly(
162        bug_id=444, test=test_key,
163        start_revision=155000, end_revision=155100,
164        median_before_anomaly=100, median_after_anomaly=200).put()
165    result = auto_bisect.StartNewBisectForBug(444)
166    self.assertEqual({'error': 'Could not select a test.'}, result)
167
168
169class TickMonitoringCustomMetricTest(testing_common.TestCase):
170
171  def setUp(self):
172    super(TickMonitoringCustomMetricTest, self).setUp()
173    app = webapp2.WSGIApplication(
174        [('/auto_bisect', auto_bisect.AutoBisectHandler)])
175    self.testapp = webtest.TestApp(app)
176
177  @mock.patch.object(utils, 'TickMonitoringCustomMetric')
178  def testPost_NoTryJobs_CustomMetricTicked(self, mock_tick):
179    self.testapp.post('/auto_bisect')
180    mock_tick.assert_called_once_with('RestartFailedBisectJobs')
181
182  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
183  @mock.patch.object(utils, 'TickMonitoringCustomMetric')
184  def testPost_RunCount1_ExceptionInPerformBisect_CustomMetricNotTicked(
185      self, mock_tick, mock_perform_bisect):
186    mock_perform_bisect.side_effect = request_handler.InvalidInputError()
187    try_job.TryJob(
188        bug_id=222, status='failed',
189        last_ran_timestamp=datetime.datetime.now(),
190        run_count=1).put()
191    self.testapp.post('/auto_bisect')
192    self.assertEqual(0, mock_tick.call_count)
193
194  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
195  @mock.patch.object(utils, 'TickMonitoringCustomMetric')
196  def testPost_RunCount2_ExceptionInPerformBisect_CustomMetricNotTicked(
197      self, mock_tick, mock_perform_bisect):
198    mock_perform_bisect.side_effect = request_handler.InvalidInputError()
199    try_job.TryJob(
200        bug_id=111, status='failed',
201        last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8),
202        run_count=2).put()
203    self.testapp.post('/auto_bisect')
204    self.assertEqual(0, mock_tick.call_count)
205
206  @mock.patch.object(auto_bisect.start_try_job, 'PerformBisect')
207  @mock.patch.object(utils, 'TickMonitoringCustomMetric')
208  def testPost_NoExceptionInPerformBisect_CustomMetricTicked(
209      self, mock_tick, mock_perform_bisect):
210    try_job.TryJob(
211        bug_id=222, status='failed',
212        last_ran_timestamp=datetime.datetime.now(),
213        run_count=1).put()
214    self.testapp.post('/auto_bisect')
215    self.assertEqual(1, mock_perform_bisect.call_count)
216    mock_tick.assert_called_once_with('RestartFailedBisectJobs')
217
218
219if __name__ == '__main__':
220  unittest.main()
221