1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import unittest
6
7from mapreduce import test_support
8import mock
9
10from google.appengine.ext import testbed
11
12from dashboard import bench_find_anomalies
13from dashboard import find_change_points_exp
14from dashboard import layered_cache
15from dashboard import testing_common
16from dashboard import utils
17from dashboard.models import anomaly
18from dashboard.models import anomaly_config
19from dashboard.models import graph_data
20from dashboard.models import sheriff
21
22_SAMPLE_SERIES = [
23    (1, 50.0), (2, 50.0), (3, 50.0),
24    (4, 60.0), (5, 60.0), (6, 60.0),
25    (7, 50.0), (8, 50.0), (9, 50.0),
26    (10, 50.0), (11, 50.0), (12, 50.0)]
27
28_LARGE_SAMPLE_SERIES = [
29    (1, 50.0), (2, 50.0), (3, 50.0), (4, 60.0), (5, 60.0), (6, 60.0),
30    (7, 80.0), (8, 80.0), (9, 80.0), (10, 80.0), (11, 80.0), (12, 80.0),
31    (13, 90.0), (14, 90.0), (15, 90.0), (16, 50.0), (17, 50.0), (18, 50.0)]
32
33
34class BenchFindChangePointsTest(testing_common.TestCase):
35
36  def setUp(self):
37    super(BenchFindChangePointsTest, self).setUp()
38    self.sheriff = sheriff.Sheriff(
39        email='a@google.com', id=bench_find_anomalies._TEST_DATA_SHERIFF).put()
40
41  def _AddTestData(self, test_name, rows,
42                   improvement_direction=anomaly.UNKNOWN, config=None):
43    testing_common.AddTests(
44        ['ChromiumGPU'],
45        ['linux-release'], {
46            'scrolling_benchmark': {
47                test_name: {},
48            },
49        })
50    test = utils.TestKey(
51        'ChromiumGPU/linux-release/scrolling_benchmark/' + test_name).get()
52    test.improvement_direction = improvement_direction
53    test_container_key = utils.GetTestContainerKey(test.key)
54
55    sheriff_entity = self.sheriff.get()
56    if sheriff_entity.patterns:
57      sheriff_entity.patterns.append(test.test_path)
58    else:
59      sheriff_entity.patterns = [test.test_path]
60    sheriff_entity.put()
61
62    for row in rows:
63      graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put()
64
65    # Add test config.
66    if not config:
67      config = {
68          'max_window_size': 50,
69          'multiple_of_std_dev': 3.5,
70          'min_relative_change': 0.1,
71          'min_absolute_change': 1.0,
72          'min_segment_size': 3,
73      }
74    anomaly_config.AnomalyConfig(
75        id='config_' + test_name, config=config,
76        patterns=[test.test_path]).put()
77
78    test.put()
79    return test
80
81  def _AddAnomalyForTest(self, end_revision, bug_id, test_key):
82    anomaly_key = anomaly.Anomaly(
83        start_revision=end_revision - 1,
84        end_revision=end_revision,
85        test=test_key,
86        median_before_anomaly=50,
87        segment_size_after=3,
88        window_end_revision=6,
89        std_dev_before_anomaly=10,
90        bug_id=bug_id,
91        sheriff=self.sheriff).put()
92    return anomaly_key
93
94  @mock.patch.object(bench_find_anomalies, '_AddReportToLog')
95  def testBenchFindChangePoints_Basic(self, add_report_to_log_mock):
96    test = self._AddTestData('test', _SAMPLE_SERIES, anomaly.DOWN)
97
98    # Add untriaged anomalies.
99    self._AddAnomalyForTest(7, None, test.key)
100
101    # Add confirmed anomalies.
102    self._AddAnomalyForTest(4, 123, test.key)
103
104    # Add invalid anomalies.
105    self._AddAnomalyForTest(10, -1, test.key)
106
107    bench_find_anomalies.SetupBaseDataForBench()
108
109    self.ExecuteDeferredTasks(bench_find_anomalies._TASK_QUEUE_NAME)
110
111    test_benches = bench_find_anomalies.TestBench.query().fetch()
112    self.assertEqual(1, len(test_benches))
113
114    self.assertEqual(_SAMPLE_SERIES, test_benches[0].data_series)
115    self.assertEqual([[1, 2, 3, 4, 5, 6, 7, 8]],
116                     test_benches[0].base_anomaly_revs)
117    self.assertEqual([[6, 7, 8, 9, 10, 11, 12]],
118                     test_benches[0].invalid_anomaly_revs)
119    self.assertEqual([[1, 2, 3, 4, 5, 6, 7, 8]],
120                     test_benches[0].confirmed_anomaly_revs)
121
122    bench_name = 'find_change_points_default'
123    bench_description = 'A description.'
124    bench_find_anomalies.BenchFindChangePoints(bench_name, bench_description)
125
126    # Layered cache set.
127    bench_key = '%s.%s' % (bench_name, bench_description)
128    self.assertEqual({bench_key: True}, layered_cache.Get(
129        bench_find_anomalies._FIND_ANOMALIES_BENCH_CACHE_KEY))
130
131    task_queue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
132    test_support.execute_until_empty(task_queue,
133                                     bench_find_anomalies._TASK_QUEUE_NAME)
134
135    expected_result_dict = {
136        'bench_name': bench_name,
137        'description': bench_description,
138        'invalid_alerts': '0/1',
139        'confirmed_alerts': '1/1',
140        'new_alerts': 0,
141        'total_alerts': '1/1',
142        'unconfirmed_alert_links': '',
143        'extra_alert_links': '',
144    }
145    add_report_to_log_mock.assert_called_once_with(expected_result_dict)
146
147  def testBenchFindChangePoints_UniqueBenchRunsOnce(self):
148    test = self._AddTestData('test', _SAMPLE_SERIES, anomaly.DOWN)
149    self._AddAnomalyForTest(4, 123, test.key)
150
151    bench_find_anomalies.SetupBaseDataForBench()
152
153    self.ExecuteDeferredTasks(bench_find_anomalies._TASK_QUEUE_NAME)
154
155    bench_name = 'a_bench_name1'
156    bench_description = 'Test find_change_points v1'
157    bench_find_anomalies._EXPERIMENTAL_FUNCTIONS = {
158        bench_name: find_change_points_exp.RunFindChangePoints
159    }
160
161    bench_find_anomalies.BenchFindChangePoints(bench_name, bench_description)
162
163    # A task should be added.
164    task_queue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME)
165    tasks = task_queue.GetTasks(bench_find_anomalies._TASK_QUEUE_NAME)
166    self.assertEqual(1, len(tasks))
167
168    with self.assertRaises(ValueError):
169      bench_find_anomalies.BenchFindChangePoints(bench_name, bench_description)
170
171
172if __name__ == '__main__':
173  unittest.main()
174