schedv2_unittest.py revision 7057cf67ba1dbdd4387f53e5fe47b43c955b1a53
1#!/usr/bin/python 2 3# Copyright 2015 Google Inc. All Rights Reserved. 4 5import mock 6import unittest 7import StringIO 8 9import benchmark_run 10import machine_manager 11import schedv2 12import test_flag 13from benchmark_run import MockBenchmarkRun 14from experiment_factory import ExperimentFactory 15from experiment_file import ExperimentFile 16from experiment_runner import ExperimentRunner 17from machine_manager import MockCrosMachine 18from cros_utils import command_executer 19from cros_utils.command_executer import CommandExecuter 20from experiment_runner_unittest import FakeLogger 21from schedv2 import Schedv2 22 23 24EXPERIMENT_FILE_1 = """\ 25board: daisy 26remote: chromeos-daisy1.cros chromeos-daisy2.cros 27 28benchmark: kraken { 29 suite: telemetry_Crosperf 30 iterations: 3 31} 32 33image1 { 34 chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin 35 remote: chromeos-daisy3.cros 36} 37 38image2 { 39 chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin 40 remote: chromeos-daisy4.cros chromeos-daisy5.cros 41} 42""" 43 44 45EXPERIMENT_FILE_WITH_FORMAT = """\ 46board: daisy 47remote: chromeos-daisy1.cros chromeos-daisy2.cros 48 49benchmark: kraken {{ 50 suite: telemetry_Crosperf 51 iterations: {kraken_iterations} 52}} 53 54image1 {{ 55 chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin 56 remote: chromeos-daisy3.cros 57}} 58 59image2 {{ 60 chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin 61 remote: chromeos-daisy4.cros chromeos-daisy5.cros 62}} 63""" 64 65 66class Schedv2Test(unittest.TestCase): 67 68 mock_logger = FakeLogger() 69 mock_cmd_exec = mock.Mock(spec=CommandExecuter) 70 71 @mock.patch('benchmark_run.BenchmarkRun', 72 new=benchmark_run.MockBenchmarkRun) 73 def _make_fake_experiment(self, expstr): 74 """Create fake experiment from string. 75 76 Note - we mock out BenchmarkRun in this step. 77 """ 78 experiment_file = ExperimentFile(StringIO.StringIO(expstr)) 79 experiment = ExperimentFactory().GetExperiment( 80 experiment_file, working_directory="", log_dir="") 81 return experiment 82 83 def test_remote(self): 84 """Test that remotes in labels are aggregated into experiment.remote.""" 85 86 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) 87 self.exp.log_level = 'verbose' 88 schedv2 = Schedv2(self.exp) 89 self.assertIn('chromeos-daisy1.cros', self.exp.remote) 90 self.assertIn('chromeos-daisy2.cros', self.exp.remote) 91 self.assertIn('chromeos-daisy3.cros', self.exp.remote) 92 self.assertIn('chromeos-daisy4.cros', self.exp.remote) 93 self.assertIn('chromeos-daisy5.cros', self.exp.remote) 94 95 def test_unreachable_remote(self): 96 """Test unreachable remotes are removed from experiment remote and 97 label.remote.""" 98 99 def MockIsReachable(cm): 100 return (cm.name != 'chromeos-daisy3.cros' and 101 cm.name != 'chromeos-daisy5.cros') 102 103 with mock.patch('machine_manager.MockCrosMachine.IsReachable', 104 new=MockIsReachable) as f: 105 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) 106 self.assertIn('chromeos-daisy1.cros', self.exp.remote) 107 self.assertIn('chromeos-daisy2.cros', self.exp.remote) 108 self.assertNotIn('chromeos-daisy3.cros', self.exp.remote) 109 self.assertIn('chromeos-daisy4.cros', self.exp.remote) 110 self.assertNotIn('chromeos-daisy5.cros', self.exp.remote) 111 112 for l in self.exp.labels: 113 if l.name == 'image2': 114 self.assertNotIn('chromeos-daisy5.cros', l.remote) 115 self.assertIn('chromeos-daisy4.cros', l.remote) 116 elif l.name == 'image1': 117 self.assertNotIn('chromeos-daisy3.cros', l.remote) 118 119 @mock.patch('schedv2.BenchmarkRunCacheReader') 120 def test_BenchmarkRunCacheReader_1(self, reader): 121 """Test benchmarkrun set is split into 5 segments.""" 122 123 self.exp = self._make_fake_experiment( 124 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=9)) 125 schedv2 = Schedv2(self.exp) 126 # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4, 127 # 4, 2 brs respectively. 128 # Assert that BenchmarkRunCacheReader() is called 5 times. 129 self.assertEquals(reader.call_count, 5) 130 # reader.call_args_list[n] - nth call. 131 # reader.call_args_list[n][0] - positioned args in nth call. 132 # reader.call_args_list[n][0][1] - the 2nd arg in nth call, 133 # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'. 134 self.assertEquals(len(reader.call_args_list[0][0][1]), 4) 135 self.assertEquals(len(reader.call_args_list[1][0][1]), 4) 136 self.assertEquals(len(reader.call_args_list[2][0][1]), 4) 137 self.assertEquals(len(reader.call_args_list[3][0][1]), 4) 138 self.assertEquals(len(reader.call_args_list[4][0][1]), 2) 139 140 @mock.patch('schedv2.BenchmarkRunCacheReader') 141 def test_BenchmarkRunCacheReader_2(self, reader): 142 """Test benchmarkrun set is split into 4 segments.""" 143 144 self.exp = self._make_fake_experiment( 145 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=8)) 146 schedv2 = Schedv2(self.exp) 147 # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs. 148 self.assertEquals(reader.call_count, 4) 149 self.assertEquals(len(reader.call_args_list[0][0][1]), 4) 150 self.assertEquals(len(reader.call_args_list[1][0][1]), 4) 151 self.assertEquals(len(reader.call_args_list[2][0][1]), 4) 152 self.assertEquals(len(reader.call_args_list[3][0][1]), 4) 153 154 @mock.patch('schedv2.BenchmarkRunCacheReader') 155 def test_BenchmarkRunCacheReader_3(self, reader): 156 """Test benchmarkrun set is split into 2 segments.""" 157 158 self.exp = self._make_fake_experiment( 159 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=3)) 160 schedv2 = Schedv2(self.exp) 161 # We have 3 * 2 == 6 brs, we use 2 threads. 162 self.assertEquals(reader.call_count, 2) 163 self.assertEquals(len(reader.call_args_list[0][0][1]), 3) 164 self.assertEquals(len(reader.call_args_list[1][0][1]), 3) 165 166 @mock.patch('schedv2.BenchmarkRunCacheReader') 167 def test_BenchmarkRunCacheReader_4(self, reader): 168 """Test benchmarkrun set is not splitted.""" 169 170 self.exp = self._make_fake_experiment( 171 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=1)) 172 schedv2 = Schedv2(self.exp) 173 # We have 1 * 2 == 2 br, so only 1 instance. 174 self.assertEquals(reader.call_count, 1) 175 self.assertEquals(len(reader.call_args_list[0][0][1]), 2) 176 177 def test_cachehit(self): 178 """Test cache-hit and none-cache-hit brs are properly organized.""" 179 180 def MockReadCache(br): 181 br.cache_hit = (br.label.name == 'image2') 182 183 with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache', 184 new=MockReadCache) as f: 185 # We have 2 * 30 brs, half of which are put into _cached_br_list. 186 self.exp = self._make_fake_experiment( 187 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30)) 188 schedv2 = Schedv2(self.exp) 189 self.assertEquals(len(schedv2._cached_br_list), 30) 190 # The non-cache-hit brs are put into Schedv2._label_brl_map. 191 self.assertEquals(reduce(lambda a, x: a + len(x[1]), 192 schedv2._label_brl_map.iteritems(), 0), 193 30) 194 195 def test_nocachehit(self): 196 """Test no cache-hit.""" 197 198 def MockReadCache(br): 199 br.cache_hit = False 200 201 with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache', 202 new=MockReadCache) as f: 203 # We have 2 * 30 brs, none of which are put into _cached_br_list. 204 self.exp = self._make_fake_experiment( 205 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30)) 206 schedv2 = Schedv2(self.exp) 207 self.assertEquals(len(schedv2._cached_br_list), 0) 208 # The non-cache-hit brs are put into Schedv2._label_brl_map. 209 self.assertEquals(reduce(lambda a, x: a + len(x[1]), 210 schedv2._label_brl_map.iteritems(), 0), 211 60) 212 213 214if __name__ == '__main__': 215 test_flag.SetTestMode(True) 216 unittest.main() 217 218