schedv2_unittest.py revision e627fd61c2edba668eb2af8221892286b13f05a3 
1#!/usr/bin/python
2
3# Copyright 2015 Google Inc. All Rights Reserved.
4
5import mock
6import unittest
7import StringIO
8
9import benchmark_run
10import machine_manager
11import schedv2
12import test_flag
13from benchmark_run import MockBenchmarkRun
14from experiment_factory import ExperimentFactory
15from experiment_file import ExperimentFile
16from experiment_runner import ExperimentRunner
17from machine_manager import MockCrosMachine
18from cros_utils import command_executer
19from cros_utils.command_executer import CommandExecuter
20from experiment_runner_unittest import FakeLogger
21from schedv2 import Schedv2
22
23
24EXPERIMENT_FILE_1 = """\
25board: daisy
26remote: chromeos-daisy1.cros chromeos-daisy2.cros
27
28benchmark: kraken {
29  suite: telemetry_Crosperf
30  iterations: 3
31}
32
33image1 {
34  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
35  remote: chromeos-daisy3.cros
36}
37
38image2 {
39  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
40  remote: chromeos-daisy4.cros chromeos-daisy5.cros
41}
42"""
43
44
45EXPERIMENT_FILE_WITH_FORMAT = """\
46board: daisy
47remote: chromeos-daisy1.cros chromeos-daisy2.cros
48
49benchmark: kraken {{
50  suite: telemetry_Crosperf
51  iterations: {kraken_iterations}
52}}
53
54image1 {{
55  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
56  remote: chromeos-daisy3.cros
57}}
58
59image2 {{
60  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
61  remote: chromeos-daisy4.cros chromeos-daisy5.cros
62}}
63"""
64
65
66class Schedv2Test(unittest.TestCase):
67
68    mock_logger = FakeLogger()
69    mock_cmd_exec = mock.Mock(spec=CommandExecuter)
70
71    @mock.patch('benchmark_run.BenchmarkRun',
72                new=benchmark_run.MockBenchmarkRun)
73    def _make_fake_experiment(self, expstr):
74        """Create fake experiment from string.
75
76        Note - we mock out BenchmarkRun in this step.
77        """
78        experiment_file = ExperimentFile(StringIO.StringIO(expstr))
79        experiment = ExperimentFactory().GetExperiment(
80            experiment_file, working_directory="", log_dir="")
81        return experiment
82
83    def test_remote(self):
84        """Test that remotes in labels are aggregated into experiment.remote."""
85
86        self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
87        self.exp.log_level = 'verbose'
88        schedv2 = Schedv2(self.exp)
89        self.assertIn('chromeos-daisy1.cros', self.exp.remote)
90        self.assertIn('chromeos-daisy2.cros', self.exp.remote)
91        self.assertIn('chromeos-daisy3.cros', self.exp.remote)
92        self.assertIn('chromeos-daisy4.cros', self.exp.remote)
93        self.assertIn('chromeos-daisy5.cros', self.exp.remote)
94
95    def test_unreachable_remote(self):
96        """Test unreachable remotes are removed from experiment remote and
97        label.remote."""
98
99        def MockIsReachable(cm):
100            return (cm.name != 'chromeos-daisy3.cros' and
101                    cm.name != 'chromeos-daisy5.cros')
102
103        with mock.patch('machine_manager.MockCrosMachine.IsReachable',
104                        new=MockIsReachable) as f:
105            self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
106            self.assertIn('chromeos-daisy1.cros', self.exp.remote)
107            self.assertIn('chromeos-daisy2.cros', self.exp.remote)
108            self.assertNotIn('chromeos-daisy3.cros', self.exp.remote)
109            self.assertIn('chromeos-daisy4.cros', self.exp.remote)
110            self.assertNotIn('chromeos-daisy5.cros', self.exp.remote)
111
112            for l in self.exp.labels:
113                if l.name == 'image2':
114                    self.assertNotIn('chromeos-daisy5.cros', l.remote)
115                    self.assertIn('chromeos-daisy4.cros', l.remote)
116                elif l.name == 'image1':
117                    self.assertNotIn('chromeos-daisy3.cros', l.remote)
118
119    @mock.patch('schedv2.BenchmarkRunCacheReader')
120    def test_BenchmarkRunCacheReader_1(self, reader):
121        """Test benchmarkrun set is split into 5 segments."""
122
123        self.exp = self._make_fake_experiment(
124            EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=9))
125        schedv2 = Schedv2(self.exp)
126        # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4,
127        # 4, 2 brs respectively.
128        # Assert that BenchmarkRunCacheReader() is called 5 times.
129        self.assertEquals(reader.call_count, 5)
130        # reader.call_args_list[n] - nth call.
131        # reader.call_args_list[n][0] - positioned args in nth call.
132        # reader.call_args_list[n][0][1] - the 2nd arg in nth call,
133        # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'.
134        self.assertEquals(len(reader.call_args_list[0][0][1]), 4)
135        self.assertEquals(len(reader.call_args_list[1][0][1]), 4)
136        self.assertEquals(len(reader.call_args_list[2][0][1]), 4)
137        self.assertEquals(len(reader.call_args_list[3][0][1]), 4)
138        self.assertEquals(len(reader.call_args_list[4][0][1]), 2)
139
140    @mock.patch('schedv2.BenchmarkRunCacheReader')
141    def test_BenchmarkRunCacheReader_2(self, reader):
142        """Test benchmarkrun set is split into 4 segments."""
143
144        self.exp = self._make_fake_experiment(
145            EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=8))
146        schedv2 = Schedv2(self.exp)
147        # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs.
148        self.assertEquals(reader.call_count, 4)
149        self.assertEquals(len(reader.call_args_list[0][0][1]), 4)
150        self.assertEquals(len(reader.call_args_list[1][0][1]), 4)
151        self.assertEquals(len(reader.call_args_list[2][0][1]), 4)
152        self.assertEquals(len(reader.call_args_list[3][0][1]), 4)
153
154    @mock.patch('schedv2.BenchmarkRunCacheReader')
155    def test_BenchmarkRunCacheReader_3(self, reader):
156        """Test benchmarkrun set is split into 2 segments."""
157
158        self.exp = self._make_fake_experiment(
159            EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=3))
160        schedv2 = Schedv2(self.exp)
161        # We have 3 * 2 == 6 brs, we use 2 threads.
162        self.assertEquals(reader.call_count, 2)
163        self.assertEquals(len(reader.call_args_list[0][0][1]), 3)
164        self.assertEquals(len(reader.call_args_list[1][0][1]), 3)
165
166    @mock.patch('schedv2.BenchmarkRunCacheReader')
167    def test_BenchmarkRunCacheReader_4(self, reader):
168        """Test benchmarkrun set is not splitted."""
169
170        self.exp = self._make_fake_experiment(
171            EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=1))
172        schedv2 = Schedv2(self.exp)
173        # We have 1 * 2 == 2 br, so only 1 instance.
174        self.assertEquals(reader.call_count, 1)
175        self.assertEquals(len(reader.call_args_list[0][0][1]), 2)
176
177    def test_cachehit(self):
178        """Test cache-hit and none-cache-hit brs are properly organized."""
179
180        def MockReadCache(br):
181            br.cache_hit = (br.label.name == 'image2')
182
183        with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
184                        new=MockReadCache) as f:
185            # We have 2 * 30 brs, half of which are put into _cached_br_list.
186            self.exp = self._make_fake_experiment(
187                EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30))
188            schedv2 = Schedv2(self.exp)
189            self.assertEquals(len(schedv2._cached_br_list), 30)
190            # The non-cache-hit brs are put into Schedv2._label_brl_map.
191            self.assertEquals(reduce(lambda a, x: a + len(x[1]),
192                                     schedv2._label_brl_map.iteritems(), 0),
193                              30)
194
195    def test_nocachehit(self):
196        """Test no cache-hit."""
197
198        def MockReadCache(br):
199            br.cache_hit = False
200
201        with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
202                        new=MockReadCache) as f:
203            # We have 2 * 30 brs, none of which are put into _cached_br_list.
204            self.exp = self._make_fake_experiment(
205                EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30))
206            schedv2 = Schedv2(self.exp)
207            self.assertEquals(len(schedv2._cached_br_list), 0)
208            # The non-cache-hit brs are put into Schedv2._label_brl_map.
209            self.assertEquals(reduce(lambda a, x: a + len(x[1]),
210                                     schedv2._label_brl_map.iteritems(), 0),
211                              60)
212
213
214if __name__ == '__main__':
215    test_flag.SetTestMode(True)
216    unittest.main()
217
218