benchmark_run_unittest.py revision a93683485612c985c661eb4ebbb02a7ae1c4a623 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Testing of benchmark_run."""
7
8import mock
9import unittest
10import inspect
11
12from cros_utils import logger
13
14import benchmark_run
15
16from suite_runner import MockSuiteRunner
17from suite_runner import SuiteRunner
18from label import MockLabel
19from benchmark import Benchmark
20from machine_manager import MockMachineManager
21from machine_manager import MachineManager
22from machine_manager import MockCrosMachine
23from results_cache import MockResultsCache
24from results_cache import CacheConditions
25from results_cache import Result
26from results_cache import ResultsCache
27
28
29class BenchmarkRunTest(unittest.TestCase):
30  """Unit tests for the BenchmarkRun class and all of its methods.
31  """
32
33  def setUp(self):
34    self.test_benchmark = Benchmark('page_cycler.netsim.top_10',   # name
35                                    'page_cycler.netsim.top_10',   # test_name
36                                    '',                            # test_args
37                                    1,                             # iterations
38                                    False,                      # rm_chroot_tmp
39                                    '',                            # perf_args
40                                    suite='telemetry_Crosperf')    # suite
41
42    self.test_label = MockLabel('test1',
43                                'image1',
44                                '/tmp/test_benchmark_run',
45                                'x86-alex',
46                                'chromeos2-row1-rack4-host9.cros',
47                                image_args='',
48                                cache_dir='',
49                                cache_only=False,
50                                log_level='average',
51                                compiler='gcc')
52
53    self.test_cache_conditions = [CacheConditions.CACHE_FILE_EXISTS,
54                                  CacheConditions.CHECKSUMS_MATCH]
55
56    self.mock_logger = logger.GetLogger(log_dir='', mock=True)
57
58    self.mock_machine_manager = mock.Mock(spec=MachineManager)
59
60  def testDryRun(self):
61    my_label = MockLabel('test1',
62                         'image1',
63                         '/tmp/test_benchmark_run',
64                         'x86-alex',
65                         'chromeos2-row1-rack4-host9.cros',
66                         image_args='',
67                         cache_dir='',
68                         cache_only=False,
69                         log_level='average',
70                         compiler='gcc')
71
72    logging_level = 'average'
73    m = MockMachineManager('/tmp/chromeos_root', 0, logging_level, '')
74    m.AddMachine('chromeos2-row1-rack4-host9.cros')
75    bench = Benchmark('page_cycler.netsim.top_10',    # name
76                      'page_cycler.netsim.top_10',    # test_name
77                      '',             # test_args
78                      1,              # iterations
79                      False,          # rm_chroot_tmp
80                      '',             # perf_args
81                      suite='telemetry_Crosperf')     # suite
82    b = benchmark_run.MockBenchmarkRun('test run', bench, my_label, 1, [], m,
83                                       logger.GetLogger(), logging_level, '')
84    b.cache = MockResultsCache()
85    b.suite_runner = MockSuiteRunner()
86    b.start()
87
88    # Make sure the arguments to BenchmarkRun.__init__ have not changed
89    # since the last time this test was updated:
90    args_list = ['self', 'name', 'benchmark', 'label', 'iteration',
91                 'cache_conditions', 'machine_manager', 'logger_to_use',
92                 'log_level', 'share_cache']
93    arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
94    self.assertEqual(len(arg_spec.args), len(args_list))
95    self.assertEqual(arg_spec.args, args_list)
96
97  def test_init(self):
98    # Nothing really worth testing here; just field assignments.
99    pass
100
101  def test_read_cache(self):
102    # Nothing really worth testing here, either.
103    pass
104
105  def test_run(self):
106    br = benchmark_run.BenchmarkRun(
107        'test_run', self.test_benchmark, self.test_label, 1,
108        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
109        'average', '')
110
111    def MockLogOutput(msg, print_to_console=False):
112      'Helper function for test_run.'
113      self.log_output.append(msg)
114
115    def MockLogError(msg, print_to_console=False):
116      'Helper function for test_run.'
117      self.log_error.append(msg)
118
119    def MockRecordStatus(msg):
120      'Helper function for test_run.'
121      self.status.append(msg)
122
123    def FakeReadCache():
124      'Helper function for test_run.'
125      br.cache = mock.Mock(spec=ResultsCache)
126      self.called_ReadCache = True
127      return 0
128
129    def FakeReadCacheSucceed():
130      'Helper function for test_run.'
131      br.cache = mock.Mock(spec=ResultsCache)
132      br.result = mock.Mock(spec=Result)
133      br.result.out = 'result.out stuff'
134      br.result.err = 'result.err stuff'
135      br.result.retval = 0
136      self.called_ReadCache = True
137      return 0
138
139    def FakeReadCacheException():
140      'Helper function for test_run.'
141      raise Exception('This is an exception test; it is supposed to happen')
142
143    def FakeAcquireMachine():
144      'Helper function for test_run.'
145      mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
146                                     'chromeos', 'average')
147      return mock_machine
148
149    def FakeRunTest(_machine):
150      'Helper function for test_run.'
151      mock_result = mock.Mock(spec=Result)
152      mock_result.retval = 0
153      return mock_result
154
155    def FakeRunTestFail(_machine):
156      'Helper function for test_run.'
157      mock_result = mock.Mock(spec=Result)
158      mock_result.retval = 1
159      return mock_result
160
161    def ResetTestValues():
162      'Helper function for test_run.'
163      self.log_output = []
164      self.log_error = []
165      self.status = []
166      br.result = None
167      self.called_ReadCache = False
168
169    # Assign all the fake functions to the appropriate objects.
170    br._logger.LogOutput = MockLogOutput
171    br._logger.LogError = MockLogError
172    br.timeline.Record = MockRecordStatus
173    br.ReadCache = FakeReadCache
174    br.RunTest = FakeRunTest
175    br.AcquireMachine = FakeAcquireMachine
176
177    # First test:  No cache hit, all goes well.
178    ResetTestValues()
179    br.run()
180    self.assertTrue(self.called_ReadCache)
181    self.assertEqual(self.log_output,
182                     ['test_run: No cache hit.',
183                      'Releasing machine: chromeos1-row3-rack5-host7.cros',
184                      'Released machine: chromeos1-row3-rack5-host7.cros'])
185    self.assertEqual(len(self.log_error), 0)
186    self.assertEqual(self.status, ['WAITING', 'SUCCEEDED'])
187
188    # Second test: No cached result found; test run was "terminated" for some
189    # reason.
190    ResetTestValues()
191    br.terminated = True
192    br.run()
193    self.assertTrue(self.called_ReadCache)
194    self.assertEqual(self.log_output,
195                     ['test_run: No cache hit.',
196                      'Releasing machine: chromeos1-row3-rack5-host7.cros',
197                      'Released machine: chromeos1-row3-rack5-host7.cros'])
198    self.assertEqual(len(self.log_error), 0)
199    self.assertEqual(self.status, ['WAITING'])
200
201    # Third test.  No cached result found; RunTest failed for some reason.
202    ResetTestValues()
203    br.terminated = False
204    br.RunTest = FakeRunTestFail
205    br.run()
206    self.assertTrue(self.called_ReadCache)
207    self.assertEqual(self.log_output,
208                     ['test_run: No cache hit.',
209                      'Releasing machine: chromeos1-row3-rack5-host7.cros',
210                      'Released machine: chromeos1-row3-rack5-host7.cros'])
211    self.assertEqual(len(self.log_error), 0)
212    self.assertEqual(self.status, ['WAITING', 'FAILED'])
213
214    # Fourth test: ReadCache found a cached result.
215    ResetTestValues()
216    br.RunTest = FakeRunTest
217    br.ReadCache = FakeReadCacheSucceed
218    br.run()
219    self.assertTrue(self.called_ReadCache)
220    self.assertEqual(self.log_output,
221                     ['test_run: Cache hit.', 'result.out stuff',
222                      'Releasing machine: chromeos1-row3-rack5-host7.cros',
223                      'Released machine: chromeos1-row3-rack5-host7.cros'])
224    self.assertEqual(self.log_error, ['result.err stuff'])
225    self.assertEqual(self.status, ['SUCCEEDED'])
226
227    # Fifth test: ReadCache generates an exception; does the try/finally block
228    # work?
229    ResetTestValues()
230    br.ReadCache = FakeReadCacheException
231    br.machine = FakeAcquireMachine()
232    br.run()
233    self.assertEqual(self.log_error, [
234        "Benchmark run: 'test_run' failed: This is an exception test; it is "
235                     "supposed to happen"
236    ])
237    self.assertEqual(self.status, ['FAILED'])
238
239  def test_terminate_pass(self):
240    br = benchmark_run.BenchmarkRun(
241        'test_run', self.test_benchmark, self.test_label, 1,
242        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
243        'average', '')
244
245    def GetLastEventPassed():
246      'Helper function for test_terminate_pass'
247      return benchmark_run.STATUS_SUCCEEDED
248
249    def RecordStub(status):
250      'Helper function for test_terminate_pass'
251      self.status = status
252
253    self.status = benchmark_run.STATUS_SUCCEEDED
254    self.assertFalse(br.terminated)
255    self.assertFalse(br.suite_runner._ct.IsTerminated())
256
257    br.timeline.GetLastEvent = GetLastEventPassed
258    br.timeline.Record = RecordStub
259
260    br.Terminate()
261
262    self.assertTrue(br.terminated)
263    self.assertTrue(br.suite_runner._ct.IsTerminated())
264    self.assertEqual(self.status, benchmark_run.STATUS_FAILED)
265
266  def test_terminate_fail(self):
267    br = benchmark_run.BenchmarkRun(
268        'test_run', self.test_benchmark, self.test_label, 1,
269        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
270        'average', '')
271
272    def GetLastEventFailed():
273      'Helper function for test_terminate_fail'
274      return benchmark_run.STATUS_FAILED
275
276    def RecordStub(status):
277      'Helper function for test_terminate_fail'
278      self.status = status
279
280    self.status = benchmark_run.STATUS_SUCCEEDED
281    self.assertFalse(br.terminated)
282    self.assertFalse(br.suite_runner._ct.IsTerminated())
283
284    br.timeline.GetLastEvent = GetLastEventFailed
285    br.timeline.Record = RecordStub
286
287    br.Terminate()
288
289    self.assertTrue(br.terminated)
290    self.assertTrue(br.suite_runner._ct.IsTerminated())
291    self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED)
292
293  def test_acquire_machine(self):
294    br = benchmark_run.BenchmarkRun(
295        'test_run', self.test_benchmark, self.test_label, 1,
296        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
297        'average', '')
298
299    br.terminated = True
300    self.assertRaises(Exception, br.AcquireMachine)
301
302    br.terminated = False
303    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
304                                   'chromeos', 'average')
305    self.mock_machine_manager.AcquireMachine.return_value = mock_machine
306
307    machine = br.AcquireMachine()
308    self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros')
309
310  def test_get_extra_autotest_args(self):
311    br = benchmark_run.BenchmarkRun(
312        'test_run', self.test_benchmark, self.test_label, 1,
313        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
314        'average', '')
315
316    def MockLogError(err_msg):
317      'Helper function for test_get_extra_autotest_args'
318      self.err_msg = err_msg
319
320    self.mock_logger.LogError = MockLogError
321
322    result = br._GetExtraAutotestArgs()
323    self.assertEqual(result, '')
324
325    self.test_benchmark.perf_args = 'record -e cycles'
326    result = br._GetExtraAutotestArgs()
327    self.assertEqual(
328        result,
329        "--profiler=custom_perf --profiler_args='perf_options=\"record -a -e "
330        "cycles\"'")
331
332    self.test_benchmark.suite = 'telemetry'
333    result = br._GetExtraAutotestArgs()
334    self.assertEqual(result, '')
335    self.assertEqual(self.err_msg, 'Telemetry does not support profiler.')
336
337    self.test_benchmark.perf_args = 'record -e cycles'
338    self.test_benchmark.suite = 'test_that'
339    result = br._GetExtraAutotestArgs()
340    self.assertEqual(result, '')
341    self.assertEqual(self.err_msg, 'test_that does not support profiler.')
342
343    self.test_benchmark.perf_args = 'junk args'
344    self.test_benchmark.suite = 'telemetry_Crosperf'
345    self.assertRaises(Exception, br._GetExtraAutotestArgs)
346
347  @mock.patch.object(SuiteRunner, 'Run')
348  @mock.patch.object(Result, 'CreateFromRun')
349  def test_run_test(self, mock_result, mock_runner):
350    br = benchmark_run.BenchmarkRun(
351        'test_run', self.test_benchmark, self.test_label, 1,
352        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
353        'average', '')
354
355    self.status = []
356
357    def MockRecord(status):
358      self.status.append(status)
359
360    br.timeline.Record = MockRecord
361    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
362                                   'chromeos', 'average')
363    mock_runner.return_value = [0, "{'Score':100}", '']
364
365    br.RunTest(mock_machine)
366
367    self.assertTrue(br.run_completed)
368    self.assertEqual(self.status, [benchmark_run.STATUS_IMAGING,
369                                   benchmark_run.STATUS_RUNNING])
370
371    self.assertEqual(br.machine_manager.ImageMachine.call_count, 1)
372    br.machine_manager.ImageMachine.assert_called_with(mock_machine,
373                                                       self.test_label)
374    self.assertEqual(mock_runner.call_count, 1)
375    mock_runner.assert_called_with(mock_machine.name, br.label, br.benchmark,
376                                   '', br.profiler_args)
377
378    self.assertEqual(mock_result.call_count, 1)
379    mock_result.assert_called_with(
380        self.mock_logger, 'average', self.test_label, None, "{'Score':100}", '',
381        0, False, 'page_cycler.netsim.top_10', 'telemetry_Crosperf')
382
383  def test_set_cache_conditions(self):
384    br = benchmark_run.BenchmarkRun(
385        'test_run', self.test_benchmark, self.test_label, 1,
386        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
387        'average', '')
388
389    phony_cache_conditions = [123, 456, True, False]
390
391    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
392
393    br.SetCacheConditions(phony_cache_conditions)
394    self.assertEqual(br.cache_conditions, phony_cache_conditions)
395
396    br.SetCacheConditions(self.test_cache_conditions)
397    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
398
399
400if __name__ == '__main__':
401  unittest.main()
402