schedv2_unittest.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python
2
3# Copyright 2015 Google Inc. All Rights Reserved.
4
5import mock
6import unittest
7import StringIO
8
9import benchmark_run
10import machine_manager
11import schedv2
12import test_flag
13from benchmark_run import MockBenchmarkRun
14from experiment_factory import ExperimentFactory
15from experiment_file import ExperimentFile
16from experiment_runner import ExperimentRunner
17from machine_manager import MockCrosMachine
18from cros_utils import command_executer
19from cros_utils.command_executer import CommandExecuter
20from experiment_runner_unittest import FakeLogger
21from schedv2 import Schedv2
22
23EXPERIMENT_FILE_1 = """\
24board: daisy
25remote: chromeos-daisy1.cros chromeos-daisy2.cros
26
27benchmark: kraken {
28  suite: telemetry_Crosperf
29  iterations: 3
30}
31
32image1 {
33  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
34  remote: chromeos-daisy3.cros
35}
36
37image2 {
38  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
39  remote: chromeos-daisy4.cros chromeos-daisy5.cros
40}
41"""
42
43EXPERIMENT_FILE_WITH_FORMAT = """\
44board: daisy
45remote: chromeos-daisy1.cros chromeos-daisy2.cros
46
47benchmark: kraken {{
48  suite: telemetry_Crosperf
49  iterations: {kraken_iterations}
50}}
51
52image1 {{
53  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
54  remote: chromeos-daisy3.cros
55}}
56
57image2 {{
58  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
59  remote: chromeos-daisy4.cros chromeos-daisy5.cros
60}}
61"""
62
63
64class Schedv2Test(unittest.TestCase):
65
66  mock_logger = FakeLogger()
67  mock_cmd_exec = mock.Mock(spec=CommandExecuter)
68
69  @mock.patch('benchmark_run.BenchmarkRun', new=benchmark_run.MockBenchmarkRun)
70  def _make_fake_experiment(self, expstr):
71    """Create fake experiment from string.
72
73        Note - we mock out BenchmarkRun in this step.
74        """
75    experiment_file = ExperimentFile(StringIO.StringIO(expstr))
76    experiment = ExperimentFactory().GetExperiment(experiment_file,
77                                                   working_directory='',
78                                                   log_dir='')
79    return experiment
80
81  def test_remote(self):
82    """Test that remotes in labels are aggregated into experiment.remote."""
83
84    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
85    self.exp.log_level = 'verbose'
86    schedv2 = Schedv2(self.exp)
87    self.assertIn('chromeos-daisy1.cros', self.exp.remote)
88    self.assertIn('chromeos-daisy2.cros', self.exp.remote)
89    self.assertIn('chromeos-daisy3.cros', self.exp.remote)
90    self.assertIn('chromeos-daisy4.cros', self.exp.remote)
91    self.assertIn('chromeos-daisy5.cros', self.exp.remote)
92
93  def test_unreachable_remote(self):
94    """Test unreachable remotes are removed from experiment remote and
95        label.remote."""
96
97    def MockIsReachable(cm):
98      return (cm.name != 'chromeos-daisy3.cros' and
99              cm.name != 'chromeos-daisy5.cros')
100
101    with mock.patch('machine_manager.MockCrosMachine.IsReachable',
102                    new=MockIsReachable) as f:
103      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
104      self.assertIn('chromeos-daisy1.cros', self.exp.remote)
105      self.assertIn('chromeos-daisy2.cros', self.exp.remote)
106      self.assertNotIn('chromeos-daisy3.cros', self.exp.remote)
107      self.assertIn('chromeos-daisy4.cros', self.exp.remote)
108      self.assertNotIn('chromeos-daisy5.cros', self.exp.remote)
109
110      for l in self.exp.labels:
111        if l.name == 'image2':
112          self.assertNotIn('chromeos-daisy5.cros', l.remote)
113          self.assertIn('chromeos-daisy4.cros', l.remote)
114        elif l.name == 'image1':
115          self.assertNotIn('chromeos-daisy3.cros', l.remote)
116
117  @mock.patch('schedv2.BenchmarkRunCacheReader')
118  def test_BenchmarkRunCacheReader_1(self, reader):
119    """Test benchmarkrun set is split into 5 segments."""
120
121    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
122        kraken_iterations=9))
123    schedv2 = Schedv2(self.exp)
124    # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4,
125    # 4, 2 brs respectively.
126    # Assert that BenchmarkRunCacheReader() is called 5 times.
127    self.assertEquals(reader.call_count, 5)
128    # reader.call_args_list[n] - nth call.
129    # reader.call_args_list[n][0] - positioned args in nth call.
130    # reader.call_args_list[n][0][1] - the 2nd arg in nth call,
131    # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'.
132    self.assertEquals(len(reader.call_args_list[0][0][1]), 4)
133    self.assertEquals(len(reader.call_args_list[1][0][1]), 4)
134    self.assertEquals(len(reader.call_args_list[2][0][1]), 4)
135    self.assertEquals(len(reader.call_args_list[3][0][1]), 4)
136    self.assertEquals(len(reader.call_args_list[4][0][1]), 2)
137
138  @mock.patch('schedv2.BenchmarkRunCacheReader')
139  def test_BenchmarkRunCacheReader_2(self, reader):
140    """Test benchmarkrun set is split into 4 segments."""
141
142    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
143        kraken_iterations=8))
144    schedv2 = Schedv2(self.exp)
145    # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs.
146    self.assertEquals(reader.call_count, 4)
147    self.assertEquals(len(reader.call_args_list[0][0][1]), 4)
148    self.assertEquals(len(reader.call_args_list[1][0][1]), 4)
149    self.assertEquals(len(reader.call_args_list[2][0][1]), 4)
150    self.assertEquals(len(reader.call_args_list[3][0][1]), 4)
151
152  @mock.patch('schedv2.BenchmarkRunCacheReader')
153  def test_BenchmarkRunCacheReader_3(self, reader):
154    """Test benchmarkrun set is split into 2 segments."""
155
156    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
157        kraken_iterations=3))
158    schedv2 = Schedv2(self.exp)
159    # We have 3 * 2 == 6 brs, we use 2 threads.
160    self.assertEquals(reader.call_count, 2)
161    self.assertEquals(len(reader.call_args_list[0][0][1]), 3)
162    self.assertEquals(len(reader.call_args_list[1][0][1]), 3)
163
164  @mock.patch('schedv2.BenchmarkRunCacheReader')
165  def test_BenchmarkRunCacheReader_4(self, reader):
166    """Test benchmarkrun set is not splitted."""
167
168    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
169        kraken_iterations=1))
170    schedv2 = Schedv2(self.exp)
171    # We have 1 * 2 == 2 br, so only 1 instance.
172    self.assertEquals(reader.call_count, 1)
173    self.assertEquals(len(reader.call_args_list[0][0][1]), 2)
174
175  def test_cachehit(self):
176    """Test cache-hit and none-cache-hit brs are properly organized."""
177
178    def MockReadCache(br):
179      br.cache_hit = (br.label.name == 'image2')
180
181    with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
182                    new=MockReadCache) as f:
183      # We have 2 * 30 brs, half of which are put into _cached_br_list.
184      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
185          kraken_iterations=30))
186      schedv2 = Schedv2(self.exp)
187      self.assertEquals(len(schedv2._cached_br_list), 30)
188      # The non-cache-hit brs are put into Schedv2._label_brl_map.
189      self.assertEquals(
190          reduce(lambda a, x: a + len(x[1]), schedv2._label_brl_map.iteritems(),
191                 0), 30)
192
193  def test_nocachehit(self):
194    """Test no cache-hit."""
195
196    def MockReadCache(br):
197      br.cache_hit = False
198
199    with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
200                    new=MockReadCache) as f:
201      # We have 2 * 30 brs, none of which are put into _cached_br_list.
202      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
203          kraken_iterations=30))
204      schedv2 = Schedv2(self.exp)
205      self.assertEquals(len(schedv2._cached_br_list), 0)
206      # The non-cache-hit brs are put into Schedv2._label_brl_map.
207      self.assertEquals(
208          reduce(lambda a, x: a + len(x[1]), schedv2._label_brl_map.iteritems(),
209                 0), 60)
210
211
212if __name__ == '__main__':
213  test_flag.SetTestMode(True)
214  unittest.main()
215