benchmark_run_unittest.py revision 1a224369afcbfd0276f4c7bdc625dec7f7b30d01 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Testing of benchmark_run."""
8
9import mock
10import unittest
11import inspect
12
13from utils import logger
14
15import benchmark_run
16
17from suite_runner import MockSuiteRunner
18from suite_runner import SuiteRunner
19from label import MockLabel
20from benchmark import Benchmark
21from machine_manager import MockMachineManager
22from machine_manager import MachineManager
23from machine_manager import MockCrosMachine
24from results_cache import MockResultsCache
25from results_cache import CacheConditions
26from results_cache import Result
27from results_cache import ResultsCache
28
29
30class BenchmarkRunTest(unittest.TestCase):
31  """
32  Unit tests for the BenchmarkRun class and all of its methods.
33  """
34
35  def setUp(self):
36    self.test_benchmark = Benchmark("page_cycler.netsim.top_10",   # name
37                                    "page_cycler.netsim.top_10",   # test_name
38                                    "",                            # test_args
39                                    1,                             # iterations
40                                    False,                      # rm_chroot_tmp
41                                    "",                            # perf_args
42                                    suite="telemetry_Crosperf")    # suite
43
44    self.test_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run",
45                                "x86-alex", "chromeos2-row1-rack4-host9.cros",
46                                image_args="",
47                                cache_dir="", cache_only=False)
48
49    self.test_cache_conditions =  [CacheConditions.CACHE_FILE_EXISTS,
50                                   CacheConditions.CHECKSUMS_MATCH]
51
52    self.mock_logger = logger.GetLogger(log_dir="", mock=True)
53
54    self.mock_machine_manager = mock.Mock(spec=MachineManager)
55
56  def testDryRun(self):
57    my_label = MockLabel("test1", "image1", "/tmp/test_benchmark_run",
58                         "x86-alex", "chromeos2-row1-rack4-host9.cros",
59                         image_args="",
60                         cache_dir="", cache_only=False)
61    logging_level = "average"
62    m = MockMachineManager("/tmp/chromeos_root", 0, logging_level)
63    m.AddMachine("chromeos2-row1-rack4-host9.cros")
64    bench = Benchmark("page_cycler.netsim.top_10",    # name
65                      "page_cycler.netsim.top_10",    # test_name
66                      "",             # test_args
67                      1,              # iteratins
68                      False,          # rm_chroot_tmp
69                      "",             # perf_args
70                      suite="telemetry_Crosperf")     # suite
71    b = benchmark_run.MockBenchmarkRun("test run",
72                         bench,
73                         my_label,
74                         1,
75                         [],
76                         m,
77                         logger.GetLogger(),
78                         logging_level,
79                         "")
80    b.cache = MockResultsCache()
81    b.suite_runner = MockSuiteRunner()
82    b.start()
83
84    # Make sure the arguments to BenchmarkRun.__init__ have not changed
85    # since the last time this test was updated:
86    args_list = ['self', 'name', 'benchmark', 'label', 'iteration',
87                 'cache_conditions', 'machine_manager', 'logger_to_use',
88                 'log_level', 'share_cache']
89    arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
90    self.assertEqual(len(arg_spec.args), len(args_list))
91    self.assertEqual (arg_spec.args, args_list)
92
93
94  def test_init(self):
95    # Nothing really worth testing here; just field assignments.
96    pass
97
98  def test_read_cache(self):
99    # Nothing really worth testing here, either.
100    pass
101
102  def test_run(self):
103    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
104                                self.test_label, 1, self.test_cache_conditions,
105                                self.mock_machine_manager,
106                                self.mock_logger,
107                                "average", "")
108
109    def MockLogOutput(msg, print_to_console=False):
110      "Helper function for test_run."
111      self.log_output.append(msg)
112
113    def MockLogError(msg, print_to_console=False):
114      "Helper function for test_run."
115      self.log_error.append(msg)
116
117    def MockRecordStatus(msg):
118      "Helper function for test_run."
119      self.status.append(msg)
120
121    def FakeReadCache():
122      "Helper function for test_run."
123      br.cache = mock.Mock(spec=ResultsCache)
124      self.called_ReadCache = True
125      return 0
126
127    def FakeReadCacheSucceed():
128      "Helper function for test_run."
129      br.cache = mock.Mock(spec=ResultsCache)
130      br.result = mock.Mock(spec=Result)
131      br.result.out = "result.out stuff"
132      br.result.err = "result.err stuff"
133      br.result.retval = 0
134      self.called_ReadCache = True
135      return 0
136
137    def FakeReadCacheException():
138      "Helper function for test_run."
139      raise Exception("This is an exception test; it is supposed to happen")
140
141    def FakeAcquireMachine():
142      "Helper function for test_run."
143      mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros',
144                                      'chromeos', 'average')
145      return mock_machine
146
147    def FakeRunTest(_machine):
148      "Helper function for test_run."
149      mock_result = mock.Mock(spec=Result)
150      mock_result.retval = 0
151      return mock_result
152
153    def FakeRunTestFail(_machine):
154      "Helper function for test_run."
155      mock_result = mock.Mock(spec=Result)
156      mock_result.retval = 1
157      return mock_result
158
159    def ResetTestValues():
160      "Helper function for test_run."
161      self.log_output = []
162      self.log_error = []
163      self.status = []
164      br.result = None
165      self.called_ReadCache = False
166
167    # Assign all the fake functions to the appropriate objects.
168    br._logger.LogOutput = MockLogOutput
169    br._logger.LogError = MockLogError
170    br.timeline.Record = MockRecordStatus
171    br.ReadCache = FakeReadCache
172    br.RunTest = FakeRunTest
173    br.AcquireMachine = FakeAcquireMachine
174
175    # First test:  No cache hit, all goes well.
176    ResetTestValues()
177    br.run()
178    self.assertTrue (self.called_ReadCache)
179    self.assertEqual (self.log_output,
180                      ['test_run: No cache hit.',
181                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
182                       'Released machine: chromeos1-row3-rack5-host7.cros'])
183    self.assertEqual (len(self.log_error), 0)
184    self.assertEqual (self.status, ['WAITING', 'SUCCEEDED'])
185
186    # Second test: No cached result found; test run was "terminated" for some
187    # reason.
188    ResetTestValues()
189    br.terminated = True
190    br.run()
191    self.assertTrue (self.called_ReadCache)
192    self.assertEqual (self.log_output,
193                      ['test_run: No cache hit.',
194                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
195                       'Released machine: chromeos1-row3-rack5-host7.cros'])
196    self.assertEqual (len(self.log_error), 0)
197    self.assertEqual (self.status, ['WAITING'])
198
199    # Third test.  No cached result found; RunTest failed for some reason.
200    ResetTestValues()
201    br.terminated = False
202    br.RunTest = FakeRunTestFail
203    br.run()
204    self.assertTrue (self.called_ReadCache)
205    self.assertEqual (self.log_output,
206                      ['test_run: No cache hit.',
207                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
208                       'Released machine: chromeos1-row3-rack5-host7.cros'])
209    self.assertEqual (len(self.log_error), 0)
210    self.assertEqual (self.status, ['WAITING', 'FAILED'])
211
212    # Fourth test: ReadCache found a cached result.
213    ResetTestValues()
214    br.RunTest = FakeRunTest
215    br.ReadCache = FakeReadCacheSucceed
216    br.run()
217    self.assertTrue (self.called_ReadCache)
218    self.assertEqual (self.log_output,
219                      ['test_run: Cache hit.',
220                       'result.out stuff',
221                       'Releasing machine: chromeos1-row3-rack5-host7.cros',
222                       'Released machine: chromeos1-row3-rack5-host7.cros'])
223    self.assertEqual (self.log_error, ['result.err stuff'])
224    self.assertEqual (self.status, ['SUCCEEDED'])
225
226    # Fifth test: ReadCache generates an exception; does the try/finally block
227    # work?
228    ResetTestValues()
229    br.ReadCache = FakeReadCacheException
230    br.machine = FakeAcquireMachine()
231    br.run()
232    self.assertEqual (self.log_output,
233                      ['Releasing machine: chromeos1-row3-rack5-host7.cros',
234                       'Released machine: chromeos1-row3-rack5-host7.cros'])
235    self.assertEqual (self.log_error,
236                      ["Benchmark run: 'test_run' failed: This is an exception test; it is supposed to happen"])
237    self.assertEqual (self.status, ['FAILED'])
238
239
240  def test_terminate_pass(self):
241    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
242                                self.test_label, 1, self.test_cache_conditions,
243                                self.mock_machine_manager,
244                                self.mock_logger,
245                                "average", "")
246
247    def GetLastEventPassed():
248      "Helper function for test_terminate_pass"
249      return benchmark_run.STATUS_SUCCEEDED
250
251    def RecordStub(status):
252      "Helper function for test_terminate_pass"
253      self.status = status
254
255    self.status = benchmark_run.STATUS_SUCCEEDED
256    self.assertFalse (br.terminated)
257    self.assertFalse (br.suite_runner._ct.IsTerminated())
258
259    br.timeline.GetLastEvent = GetLastEventPassed
260    br.timeline.Record = RecordStub
261
262    br.Terminate()
263
264    self.assertTrue (br.terminated)
265    self.assertTrue (br.suite_runner._ct.IsTerminated())
266    self.assertEqual (self.status, benchmark_run.STATUS_FAILED)
267
268
269
270  def test_terminate_fail(self):
271    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
272                                self.test_label, 1, self.test_cache_conditions,
273                                self.mock_machine_manager,
274                                self.mock_logger,
275                                "average", "")
276
277    def GetLastEventFailed():
278      "Helper function for test_terminate_fail"
279      return benchmark_run.STATUS_FAILED
280
281    def RecordStub(status):
282      "Helper function for test_terminate_fail"
283      self.status = status
284
285    self.status = benchmark_run.STATUS_SUCCEEDED
286    self.assertFalse (br.terminated)
287    self.assertFalse (br.suite_runner._ct.IsTerminated())
288
289    br.timeline.GetLastEvent = GetLastEventFailed
290    br.timeline.Record = RecordStub
291
292    br.Terminate()
293
294    self.assertTrue (br.terminated)
295    self.assertTrue (br.suite_runner._ct.IsTerminated())
296    self.assertEqual (self.status, benchmark_run.STATUS_SUCCEEDED)
297
298
299  def test_acquire_machine(self):
300    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
301                                self.test_label, 1, self.test_cache_conditions,
302                                self.mock_machine_manager,
303                                self.mock_logger,
304                                "average", "")
305
306
307    br.terminated = True
308    self.assertRaises (Exception, br.AcquireMachine)
309
310    br.terminated = False
311    mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros',
312                                    'chromeos', 'average')
313    self.mock_machine_manager.AcquireMachine.return_value = mock_machine
314
315    machine = br.AcquireMachine()
316    self.assertEqual (machine.name, 'chromeos1-row3-rack5-host7.cros')
317
318
319  def test_get_extra_autotest_args(self):
320    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
321                                self.test_label, 1, self.test_cache_conditions,
322                                self.mock_machine_manager,
323                                self.mock_logger,
324                                "average", "")
325
326    def MockLogError(err_msg):
327      "Helper function for test_get_extra_autotest_args"
328      self.err_msg = err_msg
329
330    self.mock_logger.LogError = MockLogError
331
332    result = br._GetExtraAutotestArgs()
333    self.assertEqual(result, "")
334
335    self.test_benchmark.perf_args = "record -e cycles"
336    result = br._GetExtraAutotestArgs()
337    self.assertEqual(result,
338"--profiler=custom_perf --profiler_args='perf_options=\"record -a -e cycles\"'")
339
340    self.test_benchmark.suite = "telemetry"
341    result = br._GetExtraAutotestArgs()
342    self.assertEqual(result, "")
343    self.assertEqual(self.err_msg, "Telemetry does not support profiler.")
344
345    self.test_benchmark.perf_args = "record -e cycles"
346    self.test_benchmark.suite = "test_that"
347    result = br._GetExtraAutotestArgs()
348    self.assertEqual(result, "")
349    self.assertEqual(self.err_msg, "test_that does not support profiler.")
350
351    self.test_benchmark.perf_args = "junk args"
352    self.test_benchmark.suite = "telemetry_Crosperf"
353    self.assertRaises(Exception, br._GetExtraAutotestArgs)
354
355
356  @mock.patch.object(SuiteRunner, 'Run')
357  @mock.patch.object(Result, 'CreateFromRun')
358  def test_run_test(self, mock_result, mock_runner):
359    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
360                                self.test_label, 1, self.test_cache_conditions,
361                                self.mock_machine_manager,
362                                self.mock_logger,
363                                "average", "")
364
365    self.status = []
366    def MockRecord(status):
367      self.status.append(status)
368
369    br.timeline.Record = MockRecord
370    mock_machine = MockCrosMachine ('chromeos1-row3-rack5-host7.cros',
371                                    'chromeos', 'average')
372    mock_runner.return_value = [0, "{'Score':100}", ""]
373
374    br.RunTest(mock_machine)
375
376    self.assertTrue(br.run_completed)
377    self.assertEqual (self.status, [ benchmark_run.STATUS_IMAGING,
378                                     benchmark_run.STATUS_RUNNING])
379
380    self.assertEqual (br.machine_manager.ImageMachine.call_count, 1)
381    br.machine_manager.ImageMachine.assert_called_with (mock_machine,
382                                                        self.test_label)
383    self.assertEqual (mock_runner.call_count, 1)
384    mock_runner.assert_called_with (mock_machine.name, br.label,
385                                    br.benchmark, "", br.profiler_args)
386
387    self.assertEqual (mock_result.call_count, 1)
388    mock_result.assert_called_with (self.mock_logger, 'average',
389                                    self.test_label,  "{'Score':100}",
390                                    "", 0, False, 'page_cycler.netsim.top_10',
391                                    'telemetry_Crosperf')
392
393
394
395  def test_set_cache_conditions(self):
396    br = benchmark_run.BenchmarkRun("test_run", self.test_benchmark,
397                                self.test_label, 1, self.test_cache_conditions,
398                                self.mock_machine_manager,
399                                self.mock_logger,
400                                "average", "")
401
402    phony_cache_conditions = [ 123, 456, True, False ]
403
404    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
405
406    br.SetCacheConditions (phony_cache_conditions)
407    self.assertEqual(br.cache_conditions, phony_cache_conditions)
408
409    br.SetCacheConditions(self.test_cache_conditions)
410    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
411
412
413if __name__ == "__main__":
414  unittest.main()
415