schedv2_unittest.py revision ed69676d435b7b6983271ed8fab200627a0b966e 
1#!/usr/bin/env python2
2
3# Copyright 2015 Google Inc. All Rights Reserved.
4"""This contains the unit tests for the new Crosperf task scheduler."""
5
6from __future__ import print_function
7
8import mock
9import unittest
10import StringIO
11
12import benchmark_run
13import test_flag
14from experiment_factory import ExperimentFactory
15from experiment_file import ExperimentFile
16from cros_utils.command_executer import CommandExecuter
17from experiment_runner_unittest import FakeLogger
18from schedv2 import Schedv2
19
20EXPERIMENT_FILE_1 = """\
21board: daisy
22remote: chromeos-daisy1.cros chromeos-daisy2.cros
23
24benchmark: kraken {
25  suite: telemetry_Crosperf
26  iterations: 3
27}
28
29image1 {
30  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
31  remote: chromeos-daisy3.cros
32}
33
34image2 {
35  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
36  remote: chromeos-daisy4.cros chromeos-daisy5.cros
37}
38"""
39
40EXPERIMENT_FILE_WITH_FORMAT = """\
41board: daisy
42remote: chromeos-daisy1.cros chromeos-daisy2.cros
43
44benchmark: kraken {{
45  suite: telemetry_Crosperf
46  iterations: {kraken_iterations}
47}}
48
49image1 {{
50  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
51  remote: chromeos-daisy3.cros
52}}
53
54image2 {{
55  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
56  remote: chromeos-daisy4.cros chromeos-daisy5.cros
57}}
58"""
59
60
61class Schedv2Test(unittest.TestCase):
62  """Class for setting up and running the unit tests."""
63
64  def setUp(self):
65    self.exp = None
66
67  mock_logger = FakeLogger()
68  mock_cmd_exec = mock.Mock(spec=CommandExecuter)
69
70  @mock.patch('benchmark_run.BenchmarkRun', new=benchmark_run.MockBenchmarkRun)
71  def _make_fake_experiment(self, expstr):
72    """Create fake experiment from string.
73
74        Note - we mock out BenchmarkRun in this step.
75        """
76    experiment_file = ExperimentFile(StringIO.StringIO(expstr))
77    experiment = ExperimentFactory().GetExperiment(experiment_file,
78                                                   working_directory='',
79                                                   log_dir='')
80    return experiment
81
82  def test_remote(self):
83    """Test that remotes in labels are aggregated into experiment.remote."""
84
85    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
86    self.exp.log_level = 'verbose'
87    my_schedv2 = Schedv2(self.exp)
88    self.assertFalse(my_schedv2.is_complete())
89    self.assertIn('chromeos-daisy1.cros', self.exp.remote)
90    self.assertIn('chromeos-daisy2.cros', self.exp.remote)
91    self.assertIn('chromeos-daisy3.cros', self.exp.remote)
92    self.assertIn('chromeos-daisy4.cros', self.exp.remote)
93    self.assertIn('chromeos-daisy5.cros', self.exp.remote)
94
95  def test_unreachable_remote(self):
96    """Test unreachable remotes are removed from experiment and label."""
97
98    def MockIsReachable(cm):
99      return (cm.name != 'chromeos-daisy3.cros' and
100              cm.name != 'chromeos-daisy5.cros')
101
102    with mock.patch('machine_manager.MockCrosMachine.IsReachable',
103                    new=MockIsReachable):
104      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
105      self.assertIn('chromeos-daisy1.cros', self.exp.remote)
106      self.assertIn('chromeos-daisy2.cros', self.exp.remote)
107      self.assertNotIn('chromeos-daisy3.cros', self.exp.remote)
108      self.assertIn('chromeos-daisy4.cros', self.exp.remote)
109      self.assertNotIn('chromeos-daisy5.cros', self.exp.remote)
110
111      for l in self.exp.labels:
112        if l.name == 'image2':
113          self.assertNotIn('chromeos-daisy5.cros', l.remote)
114          self.assertIn('chromeos-daisy4.cros', l.remote)
115        elif l.name == 'image1':
116          self.assertNotIn('chromeos-daisy3.cros', l.remote)
117
118  @mock.patch('schedv2.BenchmarkRunCacheReader')
119  def test_BenchmarkRunCacheReader_1(self, reader):
120    """Test benchmarkrun set is split into 5 segments."""
121
122    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
123        kraken_iterations=9))
124    my_schedv2 = Schedv2(self.exp)
125    self.assertFalse(my_schedv2.is_complete())
126    # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4,
127    # 4, 2 brs respectively.
128    # Assert that BenchmarkRunCacheReader() is called 5 times.
129    self.assertEquals(reader.call_count, 5)
130    # reader.call_args_list[n] - nth call.
131    # reader.call_args_list[n][0] - positioned args in nth call.
132    # reader.call_args_list[n][0][1] - the 2nd arg in nth call,
133    # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'.
134    self.assertEquals(len(reader.call_args_list[0][0][1]), 4)
135    self.assertEquals(len(reader.call_args_list[1][0][1]), 4)
136    self.assertEquals(len(reader.call_args_list[2][0][1]), 4)
137    self.assertEquals(len(reader.call_args_list[3][0][1]), 4)
138    self.assertEquals(len(reader.call_args_list[4][0][1]), 2)
139
140  @mock.patch('schedv2.BenchmarkRunCacheReader')
141  def test_BenchmarkRunCacheReader_2(self, reader):
142    """Test benchmarkrun set is split into 4 segments."""
143
144    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
145        kraken_iterations=8))
146    my_schedv2 = Schedv2(self.exp)
147    self.assertFalse(my_schedv2.is_complete())
148    # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs.
149    self.assertEquals(reader.call_count, 4)
150    self.assertEquals(len(reader.call_args_list[0][0][1]), 4)
151    self.assertEquals(len(reader.call_args_list[1][0][1]), 4)
152    self.assertEquals(len(reader.call_args_list[2][0][1]), 4)
153    self.assertEquals(len(reader.call_args_list[3][0][1]), 4)
154
155  @mock.patch('schedv2.BenchmarkRunCacheReader')
156  def test_BenchmarkRunCacheReader_3(self, reader):
157    """Test benchmarkrun set is split into 2 segments."""
158
159    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
160        kraken_iterations=3))
161    my_schedv2 = Schedv2(self.exp)
162    self.assertFalse(my_schedv2.is_complete())
163    # We have 3 * 2 == 6 brs, we use 2 threads.
164    self.assertEquals(reader.call_count, 2)
165    self.assertEquals(len(reader.call_args_list[0][0][1]), 3)
166    self.assertEquals(len(reader.call_args_list[1][0][1]), 3)
167
168  @mock.patch('schedv2.BenchmarkRunCacheReader')
169  def test_BenchmarkRunCacheReader_4(self, reader):
170    """Test benchmarkrun set is not splitted."""
171
172    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
173        kraken_iterations=1))
174    my_schedv2 = Schedv2(self.exp)
175    self.assertFalse(my_schedv2.is_complete())
176    # We have 1 * 2 == 2 br, so only 1 instance.
177    self.assertEquals(reader.call_count, 1)
178    self.assertEquals(len(reader.call_args_list[0][0][1]), 2)
179
180  def test_cachehit(self):
181    """Test cache-hit and none-cache-hit brs are properly organized."""
182
183    def MockReadCache(br):
184      br.cache_hit = (br.label.name == 'image2')
185
186    with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
187                    new=MockReadCache):
188      # We have 2 * 30 brs, half of which are put into _cached_br_list.
189      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
190          kraken_iterations=30))
191      my_schedv2 = Schedv2(self.exp)
192      self.assertEquals(len(my_schedv2.get_cached_run_list()), 30)
193      # The non-cache-hit brs are put into Schedv2._label_brl_map.
194      self.assertEquals(
195          reduce(lambda a, x: a + len(x[1]),
196                 my_schedv2.get_label_map().iteritems(),
197                 0), 30)
198
199  def test_nocachehit(self):
200    """Test no cache-hit."""
201
202    def MockReadCache(br):
203      br.cache_hit = False
204
205    with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
206                    new=MockReadCache):
207      # We have 2 * 30 brs, none of which are put into _cached_br_list.
208      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format(
209          kraken_iterations=30))
210      my_schedv2 = Schedv2(self.exp)
211      self.assertEquals(len(my_schedv2.get_cached_run_list()), 0)
212      # The non-cache-hit brs are put into Schedv2._label_brl_map.
213      self.assertEquals(
214          reduce(lambda a, x: a + len(x[1]),
215                 my_schedv2.get_label_map().iteritems(),
216                 0), 60)
217
218
219if __name__ == '__main__':
220  test_flag.SetTestMode(True)
221  unittest.main()
222