1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import unittest 6 7from mapreduce import test_support 8import mock 9 10from google.appengine.ext import testbed 11 12from dashboard import bench_find_anomalies 13from dashboard import find_change_points_exp 14from dashboard import layered_cache 15from dashboard import testing_common 16from dashboard import utils 17from dashboard.models import anomaly 18from dashboard.models import anomaly_config 19from dashboard.models import graph_data 20from dashboard.models import sheriff 21 22_SAMPLE_SERIES = [ 23 (1, 50.0), (2, 50.0), (3, 50.0), 24 (4, 60.0), (5, 60.0), (6, 60.0), 25 (7, 50.0), (8, 50.0), (9, 50.0), 26 (10, 50.0), (11, 50.0), (12, 50.0)] 27 28_LARGE_SAMPLE_SERIES = [ 29 (1, 50.0), (2, 50.0), (3, 50.0), (4, 60.0), (5, 60.0), (6, 60.0), 30 (7, 80.0), (8, 80.0), (9, 80.0), (10, 80.0), (11, 80.0), (12, 80.0), 31 (13, 90.0), (14, 90.0), (15, 90.0), (16, 50.0), (17, 50.0), (18, 50.0)] 32 33 34class BenchFindChangePointsTest(testing_common.TestCase): 35 36 def setUp(self): 37 super(BenchFindChangePointsTest, self).setUp() 38 self.sheriff = sheriff.Sheriff( 39 email='a@google.com', id=bench_find_anomalies._TEST_DATA_SHERIFF).put() 40 41 def _AddTestData(self, test_name, rows, 42 improvement_direction=anomaly.UNKNOWN, config=None): 43 testing_common.AddTests( 44 ['ChromiumGPU'], 45 ['linux-release'], { 46 'scrolling_benchmark': { 47 test_name: {}, 48 }, 49 }) 50 test = utils.TestKey( 51 'ChromiumGPU/linux-release/scrolling_benchmark/' + test_name).get() 52 test.improvement_direction = improvement_direction 53 test_container_key = utils.GetTestContainerKey(test.key) 54 55 sheriff_entity = self.sheriff.get() 56 if sheriff_entity.patterns: 57 sheriff_entity.patterns.append(test.test_path) 58 else: 59 sheriff_entity.patterns = [test.test_path] 60 sheriff_entity.put() 61 62 for row in rows: 63 graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() 64 65 # Add test config. 66 if not config: 67 config = { 68 'max_window_size': 50, 69 'multiple_of_std_dev': 3.5, 70 'min_relative_change': 0.1, 71 'min_absolute_change': 1.0, 72 'min_segment_size': 3, 73 } 74 anomaly_config.AnomalyConfig( 75 id='config_' + test_name, config=config, 76 patterns=[test.test_path]).put() 77 78 test.put() 79 return test 80 81 def _AddAnomalyForTest(self, end_revision, bug_id, test_key): 82 anomaly_key = anomaly.Anomaly( 83 start_revision=end_revision - 1, 84 end_revision=end_revision, 85 test=test_key, 86 median_before_anomaly=50, 87 segment_size_after=3, 88 window_end_revision=6, 89 std_dev_before_anomaly=10, 90 bug_id=bug_id, 91 sheriff=self.sheriff).put() 92 return anomaly_key 93 94 @mock.patch.object(bench_find_anomalies, '_AddReportToLog') 95 def testBenchFindChangePoints_Basic(self, add_report_to_log_mock): 96 test = self._AddTestData('test', _SAMPLE_SERIES, anomaly.DOWN) 97 98 # Add untriaged anomalies. 99 self._AddAnomalyForTest(7, None, test.key) 100 101 # Add confirmed anomalies. 102 self._AddAnomalyForTest(4, 123, test.key) 103 104 # Add invalid anomalies. 105 self._AddAnomalyForTest(10, -1, test.key) 106 107 bench_find_anomalies.SetupBaseDataForBench() 108 109 self.ExecuteDeferredTasks(bench_find_anomalies._TASK_QUEUE_NAME) 110 111 test_benches = bench_find_anomalies.TestBench.query().fetch() 112 self.assertEqual(1, len(test_benches)) 113 114 self.assertEqual(_SAMPLE_SERIES, test_benches[0].data_series) 115 self.assertEqual([[1, 2, 3, 4, 5, 6, 7, 8]], 116 test_benches[0].base_anomaly_revs) 117 self.assertEqual([[6, 7, 8, 9, 10, 11, 12]], 118 test_benches[0].invalid_anomaly_revs) 119 self.assertEqual([[1, 2, 3, 4, 5, 6, 7, 8]], 120 test_benches[0].confirmed_anomaly_revs) 121 122 bench_name = 'find_change_points_default' 123 bench_description = 'A description.' 124 bench_find_anomalies.BenchFindChangePoints(bench_name, bench_description) 125 126 # Layered cache set. 127 bench_key = '%s.%s' % (bench_name, bench_description) 128 self.assertEqual({bench_key: True}, layered_cache.Get( 129 bench_find_anomalies._FIND_ANOMALIES_BENCH_CACHE_KEY)) 130 131 task_queue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) 132 test_support.execute_until_empty(task_queue, 133 bench_find_anomalies._TASK_QUEUE_NAME) 134 135 expected_result_dict = { 136 'bench_name': bench_name, 137 'description': bench_description, 138 'invalid_alerts': '0/1', 139 'confirmed_alerts': '1/1', 140 'new_alerts': 0, 141 'total_alerts': '1/1', 142 'unconfirmed_alert_links': '', 143 'extra_alert_links': '', 144 } 145 add_report_to_log_mock.assert_called_once_with(expected_result_dict) 146 147 def testBenchFindChangePoints_UniqueBenchRunsOnce(self): 148 test = self._AddTestData('test', _SAMPLE_SERIES, anomaly.DOWN) 149 self._AddAnomalyForTest(4, 123, test.key) 150 151 bench_find_anomalies.SetupBaseDataForBench() 152 153 self.ExecuteDeferredTasks(bench_find_anomalies._TASK_QUEUE_NAME) 154 155 bench_name = 'a_bench_name1' 156 bench_description = 'Test find_change_points v1' 157 bench_find_anomalies._EXPERIMENTAL_FUNCTIONS = { 158 bench_name: find_change_points_exp.RunFindChangePoints 159 } 160 161 bench_find_anomalies.BenchFindChangePoints(bench_name, bench_description) 162 163 # A task should be added. 164 task_queue = self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) 165 tasks = task_queue.GetTasks(bench_find_anomalies._TASK_QUEUE_NAME) 166 self.assertEqual(1, len(tasks)) 167 168 with self.assertRaises(ValueError): 169 bench_find_anomalies.BenchFindChangePoints(bench_name, bench_description) 170 171 172if __name__ == '__main__': 173 unittest.main() 174