benchmark_run_unittest.py revision 4b133961b76c2cb8bc58f0ea2cded9e3438ffb6f 
1#!/usr/bin/env python2
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Testing of benchmark_run."""
7
8from __future__ import print_function
9
10import mock
11import unittest
12import inspect
13
14from cros_utils import logger
15
16import benchmark_run
17
18from suite_runner import MockSuiteRunner
19from suite_runner import SuiteRunner
20from label import MockLabel
21from benchmark import Benchmark
22from machine_manager import MockMachineManager
23from machine_manager import MachineManager
24from machine_manager import MockCrosMachine
25from results_cache import MockResultsCache
26from results_cache import CacheConditions
27from results_cache import Result
28from results_cache import ResultsCache
29
30
31class BenchmarkRunTest(unittest.TestCase):
32  """Unit tests for the BenchmarkRun class and all of its methods."""
33
34  def setUp(self):
35    self.status = []
36    self.called_ReadCache = None
37    self.log_error = []
38    self.log_output = []
39    self.err_msg = None
40    self.test_benchmark = Benchmark(
41        'page_cycler.netsim.top_10',  # name
42        'page_cycler.netsim.top_10',  # test_name
43        '',  # test_args
44        1,  # iterations
45        False,  # rm_chroot_tmp
46        '',  # perf_args
47        suite='telemetry_Crosperf')  # suite
48
49    self.test_label = MockLabel(
50        'test1',
51        'image1',
52        'autotest_dir',
53        '/tmp/test_benchmark_run',
54        'x86-alex',
55        'chromeos2-row1-rack4-host9.cros',
56        image_args='',
57        cache_dir='',
58        cache_only=False,
59        log_level='average',
60        compiler='gcc')
61
62    self.test_cache_conditions = [
63        CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH
64    ]
65
66    self.mock_logger = logger.GetLogger(log_dir='', mock=True)
67
68    self.mock_machine_manager = mock.Mock(spec=MachineManager)
69
70  def testDryRun(self):
71    my_label = MockLabel(
72        'test1',
73        'image1',
74        'autotest_dir',
75        '/tmp/test_benchmark_run',
76        'x86-alex',
77        'chromeos2-row1-rack4-host9.cros',
78        image_args='',
79        cache_dir='',
80        cache_only=False,
81        log_level='average',
82        compiler='gcc')
83
84    logging_level = 'average'
85    m = MockMachineManager('/tmp/chromeos_root', 0, logging_level, '')
86    m.AddMachine('chromeos2-row1-rack4-host9.cros')
87    bench = Benchmark(
88        'page_cycler.netsim.top_10',  # name
89        'page_cycler.netsim.top_10',  # test_name
90        '',  # test_args
91        1,  # iterations
92        False,  # rm_chroot_tmp
93        '',  # perf_args
94        suite='telemetry_Crosperf')  # suite
95    b = benchmark_run.MockBenchmarkRun('test run', bench, my_label, 1, [], m,
96                                       logger.GetLogger(), logging_level, '')
97    b.cache = MockResultsCache()
98    b.suite_runner = MockSuiteRunner()
99    b.start()
100
101    # Make sure the arguments to BenchmarkRun.__init__ have not changed
102    # since the last time this test was updated:
103    args_list = [
104        'self', 'name', 'benchmark', 'label', 'iteration', 'cache_conditions',
105        'machine_manager', 'logger_to_use', 'log_level', 'share_cache'
106    ]
107    arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
108    self.assertEqual(len(arg_spec.args), len(args_list))
109    self.assertEqual(arg_spec.args, args_list)
110
111  def test_init(self):
112    # Nothing really worth testing here; just field assignments.
113    pass
114
115  def test_read_cache(self):
116    # Nothing really worth testing here, either.
117    pass
118
119  def test_run(self):
120    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
121                                    self.test_label, 1,
122                                    self.test_cache_conditions,
123                                    self.mock_machine_manager, self.mock_logger,
124                                    'average', '')
125
126    def MockLogOutput(msg, print_to_console=False):
127      'Helper function for test_run.'
128      del print_to_console
129      self.log_output.append(msg)
130
131    def MockLogError(msg, print_to_console=False):
132      'Helper function for test_run.'
133      del print_to_console
134      self.log_error.append(msg)
135
136    def MockRecordStatus(msg):
137      'Helper function for test_run.'
138      self.status.append(msg)
139
140    def FakeReadCache():
141      'Helper function for test_run.'
142      br.cache = mock.Mock(spec=ResultsCache)
143      self.called_ReadCache = True
144      return 0
145
146    def FakeReadCacheSucceed():
147      'Helper function for test_run.'
148      br.cache = mock.Mock(spec=ResultsCache)
149      br.result = mock.Mock(spec=Result)
150      br.result.out = 'result.out stuff'
151      br.result.err = 'result.err stuff'
152      br.result.retval = 0
153      self.called_ReadCache = True
154      return 0
155
156    def FakeReadCacheException():
157      'Helper function for test_run.'
158      raise RuntimeError('This is an exception test; it is supposed to happen')
159
160    def FakeAcquireMachine():
161      'Helper function for test_run.'
162      mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
163                                     'chromeos', 'average')
164      return mock_machine
165
166    def FakeRunTest(_machine):
167      'Helper function for test_run.'
168      mock_result = mock.Mock(spec=Result)
169      mock_result.retval = 0
170      return mock_result
171
172    def FakeRunTestFail(_machine):
173      'Helper function for test_run.'
174      mock_result = mock.Mock(spec=Result)
175      mock_result.retval = 1
176      return mock_result
177
178    def ResetTestValues():
179      'Helper function for test_run.'
180      self.log_output = []
181      self.log_error = []
182      self.status = []
183      br.result = None
184      self.called_ReadCache = False
185
186    # Assign all the fake functions to the appropriate objects.
187    br.logger().LogOutput = MockLogOutput
188    br.logger().LogError = MockLogError
189    br.timeline.Record = MockRecordStatus
190    br.ReadCache = FakeReadCache
191    br.RunTest = FakeRunTest
192    br.AcquireMachine = FakeAcquireMachine
193
194    # First test:  No cache hit, all goes well.
195    ResetTestValues()
196    br.run()
197    self.assertTrue(self.called_ReadCache)
198    self.assertEqual(self.log_output, [
199        'test_run: No cache hit.',
200        'Releasing machine: chromeos1-row3-rack5-host7.cros',
201        'Released machine: chromeos1-row3-rack5-host7.cros'
202    ])
203    self.assertEqual(len(self.log_error), 0)
204    self.assertEqual(self.status, ['WAITING', 'SUCCEEDED'])
205
206    # Second test: No cached result found; test run was "terminated" for some
207    # reason.
208    ResetTestValues()
209    br.terminated = True
210    br.run()
211    self.assertTrue(self.called_ReadCache)
212    self.assertEqual(self.log_output, [
213        'test_run: No cache hit.',
214        'Releasing machine: chromeos1-row3-rack5-host7.cros',
215        'Released machine: chromeos1-row3-rack5-host7.cros'
216    ])
217    self.assertEqual(len(self.log_error), 0)
218    self.assertEqual(self.status, ['WAITING'])
219
220    # Third test.  No cached result found; RunTest failed for some reason.
221    ResetTestValues()
222    br.terminated = False
223    br.RunTest = FakeRunTestFail
224    br.run()
225    self.assertTrue(self.called_ReadCache)
226    self.assertEqual(self.log_output, [
227        'test_run: No cache hit.',
228        'Releasing machine: chromeos1-row3-rack5-host7.cros',
229        'Released machine: chromeos1-row3-rack5-host7.cros'
230    ])
231    self.assertEqual(len(self.log_error), 0)
232    self.assertEqual(self.status, ['WAITING', 'FAILED'])
233
234    # Fourth test: ReadCache found a cached result.
235    ResetTestValues()
236    br.RunTest = FakeRunTest
237    br.ReadCache = FakeReadCacheSucceed
238    br.run()
239    self.assertTrue(self.called_ReadCache)
240    self.assertEqual(self.log_output, [
241        'test_run: Cache hit.', 'result.out stuff',
242        'Releasing machine: chromeos1-row3-rack5-host7.cros',
243        'Released machine: chromeos1-row3-rack5-host7.cros'
244    ])
245    self.assertEqual(self.log_error, ['result.err stuff'])
246    self.assertEqual(self.status, ['SUCCEEDED'])
247
248    # Fifth test: ReadCache generates an exception; does the try/finally block
249    # work?
250    ResetTestValues()
251    br.ReadCache = FakeReadCacheException
252    br.machine = FakeAcquireMachine()
253    br.run()
254    self.assertEqual(self.log_error, [
255        "Benchmark run: 'test_run' failed: This is an exception test; it is "
256        'supposed to happen'
257    ])
258    self.assertEqual(self.status, ['FAILED'])
259
260  def test_terminate_pass(self):
261    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
262                                    self.test_label, 1,
263                                    self.test_cache_conditions,
264                                    self.mock_machine_manager, self.mock_logger,
265                                    'average', '')
266
267    def GetLastEventPassed():
268      'Helper function for test_terminate_pass'
269      return benchmark_run.STATUS_SUCCEEDED
270
271    def RecordStub(status):
272      'Helper function for test_terminate_pass'
273      self.status = status
274
275    self.status = benchmark_run.STATUS_SUCCEEDED
276    self.assertFalse(br.terminated)
277    self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
278
279    br.timeline.GetLastEvent = GetLastEventPassed
280    br.timeline.Record = RecordStub
281
282    br.Terminate()
283
284    self.assertTrue(br.terminated)
285    self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
286    self.assertEqual(self.status, benchmark_run.STATUS_FAILED)
287
288  def test_terminate_fail(self):
289    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
290                                    self.test_label, 1,
291                                    self.test_cache_conditions,
292                                    self.mock_machine_manager, self.mock_logger,
293                                    'average', '')
294
295    def GetLastEventFailed():
296      'Helper function for test_terminate_fail'
297      return benchmark_run.STATUS_FAILED
298
299    def RecordStub(status):
300      'Helper function for test_terminate_fail'
301      self.status = status
302
303    self.status = benchmark_run.STATUS_SUCCEEDED
304    self.assertFalse(br.terminated)
305    self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
306
307    br.timeline.GetLastEvent = GetLastEventFailed
308    br.timeline.Record = RecordStub
309
310    br.Terminate()
311
312    self.assertTrue(br.terminated)
313    self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
314    self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED)
315
316  def test_acquire_machine(self):
317    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
318                                    self.test_label, 1,
319                                    self.test_cache_conditions,
320                                    self.mock_machine_manager, self.mock_logger,
321                                    'average', '')
322
323    br.terminated = True
324    self.assertRaises(Exception, br.AcquireMachine)
325
326    br.terminated = False
327    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
328                                   'chromeos', 'average')
329    self.mock_machine_manager.AcquireMachine.return_value = mock_machine
330
331    machine = br.AcquireMachine()
332    self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros')
333
334  def test_get_extra_autotest_args(self):
335    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
336                                    self.test_label, 1,
337                                    self.test_cache_conditions,
338                                    self.mock_machine_manager, self.mock_logger,
339                                    'average', '')
340
341    def MockLogError(err_msg):
342      'Helper function for test_get_extra_autotest_args'
343      self.err_msg = err_msg
344
345    self.mock_logger.LogError = MockLogError
346
347    result = br.GetExtraAutotestArgs()
348    self.assertEqual(result, '')
349
350    self.test_benchmark.perf_args = 'record -e cycles'
351    result = br.GetExtraAutotestArgs()
352    self.assertEqual(
353        result,
354        "--profiler=custom_perf --profiler_args='perf_options=\"record -a -e "
355        "cycles\"'")
356
357    self.test_benchmark.suite = 'telemetry'
358    result = br.GetExtraAutotestArgs()
359    self.assertEqual(result, '')
360    self.assertEqual(self.err_msg, 'Telemetry does not support profiler.')
361
362    self.test_benchmark.perf_args = 'record -e cycles'
363    self.test_benchmark.suite = 'test_that'
364    result = br.GetExtraAutotestArgs()
365    self.assertEqual(result, '')
366    self.assertEqual(self.err_msg, 'test_that does not support profiler.')
367
368    self.test_benchmark.perf_args = 'junk args'
369    self.test_benchmark.suite = 'telemetry_Crosperf'
370    self.assertRaises(Exception, br.GetExtraAutotestArgs)
371
372  @mock.patch.object(SuiteRunner, 'Run')
373  @mock.patch.object(Result, 'CreateFromRun')
374  def test_run_test(self, mock_result, mock_runner):
375    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
376                                    self.test_label, 1,
377                                    self.test_cache_conditions,
378                                    self.mock_machine_manager, self.mock_logger,
379                                    'average', '')
380
381    self.status = []
382
383    def MockRecord(status):
384      self.status.append(status)
385
386    br.timeline.Record = MockRecord
387    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
388                                   'chromeos', 'average')
389    mock_runner.return_value = [0, "{'Score':100}", '']
390
391    br.RunTest(mock_machine)
392
393    self.assertTrue(br.run_completed)
394    self.assertEqual(
395        self.status,
396        [benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING])
397
398    self.assertEqual(br.machine_manager.ImageMachine.call_count, 1)
399    br.machine_manager.ImageMachine.assert_called_with(mock_machine,
400                                                       self.test_label)
401    self.assertEqual(mock_runner.call_count, 1)
402    mock_runner.assert_called_with(mock_machine.name, br.label, br.benchmark,
403                                   '', br.profiler_args)
404
405    self.assertEqual(mock_result.call_count, 1)
406    mock_result.assert_called_with(self.mock_logger, 'average', self.test_label,
407                                   None, "{'Score':100}", '', 0,
408                                   'page_cycler.netsim.top_10',
409                                   'telemetry_Crosperf')
410
411  def test_set_cache_conditions(self):
412    br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
413                                    self.test_label, 1,
414                                    self.test_cache_conditions,
415                                    self.mock_machine_manager, self.mock_logger,
416                                    'average', '')
417
418    phony_cache_conditions = [123, 456, True, False]
419
420    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
421
422    br.SetCacheConditions(phony_cache_conditions)
423    self.assertEqual(br.cache_conditions, phony_cache_conditions)
424
425    br.SetCacheConditions(self.test_cache_conditions)
426    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
427
428
429if __name__ == '__main__':
430  unittest.main()
431