benchmark_run_unittest.py revision e627fd61c2edba668eb2af8221892286b13f05a3 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Testing of benchmark_run."""
8
9import mock
10import unittest
11import inspect
12
13from cros_utils import logger
14
15import benchmark_run
16
17from suite_runner import MockSuiteRunner
18from suite_runner import SuiteRunner
19from label import MockLabel
20from benchmark import Benchmark
21from machine_manager import MockMachineManager
22from machine_manager import MachineManager
23from machine_manager import MockCrosMachine
24from results_cache import MockResultsCache
25from results_cache import CacheConditions
26from results_cache import Result
27from results_cache import ResultsCache
28
29
30class BenchmarkRunTest(unittest.TestCase):
31  """
32  Unit tests for the BenchmarkRun class and all of its methods.
33  """
34
35  def setUp(self):
36    self.test_benchmark = Benchmark("page_cycler.netsim.top_10",   # name
37                                    "page_cycler.netsim.top_10",   # test_name
38                                    "",                            # test_args
39                                    1,                             # iterations
40                                    False,                      # rm_chroot_tmp
41                                    "",                            # perf_args
42                                    suite="telemetry_Crosperf")    # suite
43
44    self.test_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run",
45                                "x86-alex", "chromeos2-row1-rack4-host9.cros",
46                                image_args="", cache_dir="", cache_only=False,
47                                log_level="average", compiler="gcc")
48
49
50    self.test_cache_conditions =  [CacheConditions.CACHE_FILE_EXISTS,
51                                   CacheConditions.CHECKSUMS_MATCH]
52
53    self.mock_logger = logger.GetLogger(log_dir="", mock=True)
54
55    self.mock_machine_manager = mock.Mock(spec=MachineManager)
56
57  def testDryRun(self):
58    my_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run",
59                         "x86-alex", "chromeos2-row1-rack4-host9.cros",
60                         image_args="", cache_dir="", cache_only=False,
61                         log_level="average", compiler="gcc")
62
63    logging_level = "average"
64    m = MockMachineManager("/tmp/chromeos_root", 0, logging_level)
65    m.AddMachine("chromeos2-row1-rack4-host9.cros")
66    bench = Benchmark("page_cycler.netsim.top_10",    # name
67                      "page_cycler.netsim.top_10",    # test_name
68                      "",             # test_args
69                      1,              # iterations
70                      False,          # rm_chroot_tmp
71                      "",             # perf_args
72                      suite="telemetry_Crosperf")     # suite
73    b = benchmark_run.MockBenchmarkRun("test run",
74                         bench,
75                         my_label,
76                         1,
77                         [],
78                         m,
79                         logger.GetLogger(),
80                         logging_level,
81                         "")
82    b.cache = MockResultsCache()
83    b.suite_runner = MockSuiteRunner()
84    b.start()
85
86    # Make sure the arguments to BenchmarkRun.__init__ have not changed
87    # since the last time this test was updated:
88    args_list = ['self', 'name', 'benchmark', 'label', 'iteration',
89                 'cache_conditions', 'machine_manager', 'logger_to_use',
90                 'log_level', 'share_cache']
91    arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
92    self.assertEqual(len(arg_spec.args), len(args_list))
93    self.assertEqual (arg_spec.args, args_list)
94
95
96  def test_init(self):
97    # Nothing really worth testing here; just field assignments.
98    pass
99
100  def test_read_cache(self):
101    # Nothing really worth testing here, either.
102    pass
103
104  def test_run(self):
105    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
106                                self.test_label, 1, self.test_cache_conditions,
107                                self.mock_machine_manager,
108                                self.mock_logger,
109                                "average", "")
110
111    def MockLogOutput(msg, print_to_console=False):
112      "Helper function for test_run."
113      self.log_output.append(msg)
114
115    def MockLogError(msg, print_to_console=False):
116      "Helper function for test_run."
117      self.log_error.append(msg)
118
119    def MockRecordStatus(msg):
120      "Helper function for test_run."
121      self.status.append(msg)
122
123    def FakeReadCache():
124      "Helper function for test_run."
125      br.cache = mock.Mock(spec=ResultsCache)
126      self.called_ReadCache = True
127      return 0
128
129    def FakeReadCacheSucceed():
130      "Helper function for test_run."
131      br.cache = mock.Mock(spec=ResultsCache)
132      br.result = mock.Mock(spec=Result)
133      br.result.out = "result.out stuff"
134      br.result.err = "result.err stuff"
135      br.result.retval = 0
136      self.called_ReadCache = True
137      return 0
138
139    def FakeReadCacheException():
140      "Helper function for test_run."
141      raise Exception("This is an exception test; it is supposed to happen")
142
143    def FakeAcquireMachine():
144      "Helper function for test_run."
145      mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros',
146                                      'chromeos', 'average')
147      return mock_machine
148
149    def FakeRunTest(_machine):
150      "Helper function for test_run."
151      mock_result = mock.Mock(spec=Result)
152      mock_result.retval = 0
153      return mock_result
154
155    def FakeRunTestFail(_machine):
156      "Helper function for test_run."
157      mock_result = mock.Mock(spec=Result)
158      mock_result.retval = 1
159      return mock_result
160
161    def ResetTestValues():
162      "Helper function for test_run."
163      self.log_output = []
164      self.log_error = []
165      self.status = []
166      br.result = None
167      self.called_ReadCache = False
168
169    # Assign all the fake functions to the appropriate objects.
170    br._logger.LogOutput = MockLogOutput
171    br._logger.LogError = MockLogError
172    br.timeline.Record = MockRecordStatus
173    br.ReadCache = FakeReadCache
174    br.RunTest = FakeRunTest
175    br.AcquireMachine = FakeAcquireMachine
176
177    # First test:  No cache hit, all goes well.
178    ResetTestValues()
179    br.run()
180    self.assertTrue (self.called_ReadCache)
181    self.assertEqual (self.log_output,
182                      ['test_run: No cache hit.',
183                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
184                       'Released machine: chromeos1-row3-rack5-host7.cros'])
185    self.assertEqual (len(self.log_error), 0)
186    self.assertEqual (self.status, ['WAITING', 'SUCCEEDED'])
187
188    # Second test: No cached result found; test run was "terminated" for some
189    # reason.
190    ResetTestValues()
191    br.terminated = True
192    br.run()
193    self.assertTrue (self.called_ReadCache)
194    self.assertEqual (self.log_output,
195                      ['test_run: No cache hit.',
196                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
197                       'Released machine: chromeos1-row3-rack5-host7.cros'])
198    self.assertEqual (len(self.log_error), 0)
199    self.assertEqual (self.status, ['WAITING'])
200
201    # Third test.  No cached result found; RunTest failed for some reason.
202    ResetTestValues()
203    br.terminated = False
204    br.RunTest = FakeRunTestFail
205    br.run()
206    self.assertTrue (self.called_ReadCache)
207    self.assertEqual (self.log_output,
208                      ['test_run: No cache hit.',
209                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
210                       'Released machine: chromeos1-row3-rack5-host7.cros'])
211    self.assertEqual (len(self.log_error), 0)
212    self.assertEqual (self.status, ['WAITING', 'FAILED'])
213
214    # Fourth test: ReadCache found a cached result.
215    ResetTestValues()
216    br.RunTest = FakeRunTest
217    br.ReadCache = FakeReadCacheSucceed
218    br.run()
219    self.assertTrue (self.called_ReadCache)
220    self.assertEqual (self.log_output,
221                      ['test_run: Cache hit.',
222                       'result.out stuff',
223                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
224                       'Released machine: chromeos1-row3-rack5-host7.cros'])
225    self.assertEqual (self.log_error, ['result.err stuff'])
226    self.assertEqual (self.status, ['SUCCEEDED'])
227
228    # Fifth test: ReadCache generates an exception; does the try/finally block
229    # work?
230    ResetTestValues()
231    br.ReadCache = FakeReadCacheException
232    br.machine = FakeAcquireMachine()
233    br.run()
234    self.assertEqual (self.log_error,
235                      ["Benchmark run: 'test_run' failed: This is an exception test; it is supposed to happen"])
236    self.assertEqual (self.status, ['FAILED'])
237
238
239  def test_terminate_pass(self):
240    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
241                                self.test_label, 1, self.test_cache_conditions,
242                                self.mock_machine_manager,
243                                self.mock_logger,
244                                "average", "")
245
246    def GetLastEventPassed():
247      "Helper function for test_terminate_pass"
248      return benchmark_run.STATUS_SUCCEEDED
249
250    def RecordStub(status):
251      "Helper function for test_terminate_pass"
252      self.status = status
253
254    self.status = benchmark_run.STATUS_SUCCEEDED
255    self.assertFalse (br.terminated)
256    self.assertFalse (br.suite_runner._ct.IsTerminated())
257
258    br.timeline.GetLastEvent = GetLastEventPassed
259    br.timeline.Record = RecordStub
260
261    br.Terminate()
262
263    self.assertTrue (br.terminated)
264    self.assertTrue (br.suite_runner._ct.IsTerminated())
265    self.assertEqual (self.status, benchmark_run.STATUS_FAILED)
266
267
268
269  def test_terminate_fail(self):
270    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
271                                self.test_label, 1, self.test_cache_conditions,
272                                self.mock_machine_manager,
273                                self.mock_logger,
274                                "average", "")
275
276    def GetLastEventFailed():
277      "Helper function for test_terminate_fail"
278      return benchmark_run.STATUS_FAILED
279
280    def RecordStub(status):
281      "Helper function for test_terminate_fail"
282      self.status = status
283
284    self.status = benchmark_run.STATUS_SUCCEEDED
285    self.assertFalse (br.terminated)
286    self.assertFalse (br.suite_runner._ct.IsTerminated())
287
288    br.timeline.GetLastEvent = GetLastEventFailed
289    br.timeline.Record = RecordStub
290
291    br.Terminate()
292
293    self.assertTrue (br.terminated)
294    self.assertTrue (br.suite_runner._ct.IsTerminated())
295    self.assertEqual (self.status, benchmark_run.STATUS_SUCCEEDED)
296
297
298  def test_acquire_machine(self):
299    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
300                                self.test_label, 1, self.test_cache_conditions,
301                                self.mock_machine_manager,
302                                self.mock_logger,
303                                "average", "")
304
305
306    br.terminated = True
307    self.assertRaises (Exception, br.AcquireMachine)
308
309    br.terminated = False
310    mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros',
311                                    'chromeos', 'average')
312    self.mock_machine_manager.AcquireMachine.return_value = mock_machine
313
314    machine = br.AcquireMachine()
315    self.assertEqual (machine.name, 'chromeos1-row3-rack5-host7.cros')
316
317
318  def test_get_extra_autotest_args(self):
319    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
320                                self.test_label, 1, self.test_cache_conditions,
321                                self.mock_machine_manager,
322                                self.mock_logger,
323                                "average", "")
324
325    def MockLogError(err_msg):
326      "Helper function for test_get_extra_autotest_args"
327      self.err_msg = err_msg
328
329    self.mock_logger.LogError = MockLogError
330
331    result = br._GetExtraAutotestArgs()
332    self.assertEqual(result, "")
333
334    self.test_benchmark.perf_args = "record -e cycles"
335    result = br._GetExtraAutotestArgs()
336    self.assertEqual(result,
337"--profiler=custom_perf --profiler_args='perf_options=\"record -a -e cycles\"'")
338
339    self.test_benchmark.suite = "telemetry"
340    result = br._GetExtraAutotestArgs()
341    self.assertEqual(result, "")
342    self.assertEqual(self.err_msg, "Telemetry does not support profiler.")
343
344    self.test_benchmark.perf_args = "record -e cycles"
345    self.test_benchmark.suite = "test_that"
346    result = br._GetExtraAutotestArgs()
347    self.assertEqual(result, "")
348    self.assertEqual(self.err_msg, "test_that does not support profiler.")
349
350    self.test_benchmark.perf_args = "junk args"
351    self.test_benchmark.suite = "telemetry_Crosperf"
352    self.assertRaises(Exception, br._GetExtraAutotestArgs)
353
354
355  @mock.patch.object(SuiteRunner, 'Run')
356  @mock.patch.object(Result, 'CreateFromRun')
357  def test_run_test(self, mock_result, mock_runner):
358    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
359                                self.test_label, 1, self.test_cache_conditions,
360                                self.mock_machine_manager,
361                                self.mock_logger,
362                                "average", "")
363
364    self.status = []
365    def MockRecord(status):
366      self.status.append(status)
367
368    br.timeline.Record = MockRecord
369    mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros',
370                                    'chromeos', 'average')
371    mock_runner.return_value = [0, "{'Score':100}", ""]
372
373    br.RunTest(mock_machine)
374
375    self.assertTrue(br.run_completed)
376    self.assertEqual (self.status, [ benchmark_run.STATUS_IMAGING,
377                                     benchmark_run.STATUS_RUNNING])
378
379    self.assertEqual (br.machine_manager.ImageMachine.call_count, 1)
380    br.machine_manager.ImageMachine.assert_called_with (mock_machine,
381                                                        self.test_label)
382    self.assertEqual (mock_runner.call_count, 1)
383    mock_runner.assert_called_with (mock_machine.name, br.label,
384                                    br.benchmark, "", br.profiler_args)
385
386    self.assertEqual (mock_result.call_count, 1)
387    mock_result.assert_called_with (self.mock_logger, 'average',
388                                    self.test_label,  None, "{'Score':100}",
389                                    "", 0, False, 'page_cycler.netsim.top_10',
390                                    'telemetry_Crosperf')
391
392
393
394  def test_set_cache_conditions(self):
395    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
396                                self.test_label, 1, self.test_cache_conditions,
397                                self.mock_machine_manager,
398                                self.mock_logger,
399                                "average", "")
400
401    phony_cache_conditions = [ 123, 456, True, False ]
402
403    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
404
405    br.SetCacheConditions (phony_cache_conditions)
406    self.assertEqual(br.cache_conditions, phony_cache_conditions)
407
408    br.SetCacheConditions(self.test_cache_conditions)
409    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
410
411
412if __name__ == "__main__":
413  unittest.main()
414