experiment_runner_unittest.py revision 7658aecb942b63472453be82a24bc2036121383d 
1#!/usr/bin/python
2#
3# Copyright 2014 Google Inc. All Rights Reserved
4import StringIO
5import getpass
6import os
7import time
8
9import mock
10import unittest
11
12import experiment_runner
13import experiment_status
14import machine_manager
15import config
16from experiment_file import ExperimentFile
17from experiment_factory import ExperimentFactory
18from results_report import TextResultsReport
19from results_report import HTMLResultsReport
20from results_cache import Result
21
22from utils import logger
23from utils import command_executer
24from utils.email_sender import EmailSender
25from utils.file_utils import FileUtils
26
27EXPERIMENT_FILE_1 = """
28  board: parrot
29  remote: chromeos-parrot1.cros chromreos-parrot2.cros
30
31  benchmark: kraken {
32    suite: telemetry_Crosperf
33    iterations: 3
34  }
35
36  image1 {
37    chromeos_root: /usr/local/google/chromeos
38    chromeos_image: /usr/local/google/chromeos/src/build/images/parrot/latest/cros_image1.bin
39  }
40
41  image2 {
42    chromeos_image: /usr/local/google/chromeos/src/build/imaages/parrot/latest/cros_image2.bin
43  }
44  """
45
46class FakeLogger(object):
47
48  def __init__(self):
49    self.LogOutputCount = 0
50    self.LogErrorCount = 0
51    self.output_msgs = []
52    self.error_msgs = []
53    self.dot_count = 0
54    self.LogStartDotsCount = 0
55    self.LogEndDotsCount = 0
56    self.LogAppendDotCount = 0
57
58  def LogOutput(self, msg):
59    self.LogOutputCount += 1
60    self.output_msgs.append(msg)
61
62  def LogError(self, msg):
63    self.LogErrorCount += 1
64    self.error_msgs.append(msg)
65
66  def LogStartDots(self):
67    self.LogStartDotsCount += 1
68    self.dot_count += 1
69
70  def LogAppendDot(self):
71    self.LogAppendDotCount += 1
72    self.dot_count += 1
73
74  def LogEndDots(self):
75    self.LogEndDotsCount += 1
76
77  def Reset(self):
78    self.LogOutputCount = 0
79    self.LogErrorCount = 0
80    self.output_msgs = []
81    self.error_msgs = []
82    self.dot_count = 0
83    self.LogStartDotsCount = 0
84    self.LogEndDotsCount = 0
85    self.LogAppendDotCount = 0
86
87class ExperimentRunnerTest(unittest.TestCase):
88
89  run_counter = 0
90  mock_logger = FakeLogger()
91  mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
92
93  def make_fake_experiment(self):
94    experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1))
95    experiment = ExperimentFactory().GetExperiment(experiment_file,
96                                                   working_directory="",
97                                                   log_dir="")
98    return experiment
99
100  @mock.patch.object (machine_manager.MachineManager, 'AddMachine')
101  @mock.patch.object (os.path, 'isfile')
102  def setUp(self, mock_isfile, mock_addmachine):
103    mock_isfile.return_value = True
104    self.exp = self.make_fake_experiment()
105
106
107  def test_init(self):
108    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
109                                            self.mock_cmd_exec)
110    self.assertFalse (er._terminated)
111    self.assertEqual (er.STATUS_TIME_DELAY, 10)
112
113    self.exp.log_level = "verbose"
114    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
115                                            self.mock_cmd_exec)
116    self.assertEqual (er.STATUS_TIME_DELAY, 30)
117
118
119
120  @mock.patch.object(experiment_status.ExperimentStatus, 'GetStatusString')
121  @mock.patch.object(experiment_status.ExperimentStatus, 'GetProgressString')
122  def test_run(self, mock_progress_string, mock_status_string):
123
124    self.run_count = 0
125    self.is_complete_count = 0
126
127    def reset():
128      self.run_count = 0
129      self.is_complete_count = 0
130
131    def FakeRun():
132      self.run_count += 1
133      return 0
134
135    def FakeIsComplete():
136      self.is_complete_count += 1
137      if self.is_complete_count < 3:
138        return False
139      else:
140        return True
141
142    self.mock_logger.Reset()
143    self.exp.Run = FakeRun
144    self.exp.IsComplete = FakeIsComplete
145
146    # Test 1: log_level == "quiet"
147    self.exp.log_level = "quiet"
148    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
149                                            self.mock_cmd_exec)
150    er.STATUS_TIME_DELAY = 2
151    mock_status_string.return_value = "Fake status string"
152    er._Run(self.exp)
153    self.assertEqual(self.run_count, 1)
154    self.assertTrue(self.is_complete_count > 0)
155    self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
156    self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
157    self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
158    self.assertEqual(self.mock_logger.dot_count, 2)
159    self.assertEqual(mock_progress_string.call_count, 0)
160    self.assertEqual(mock_status_string.call_count, 2)
161    self.assertEqual(self.mock_logger.output_msgs,
162                     ['==============================', 'Fake status string',
163                      '=============================='])
164    self.assertEqual(len(self.mock_logger.error_msgs), 0)
165
166    # Test 2: log_level == "average"
167    self.mock_logger.Reset()
168    reset()
169    self.exp.log_level = "average"
170    mock_status_string.call_count = 0
171    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
172                                            self.mock_cmd_exec)
173    er.STATUS_TIME_DELAY = 2
174    mock_status_string.return_value = "Fake status string"
175    er._Run(self.exp)
176    self.assertEqual(self.run_count, 1)
177    self.assertTrue(self.is_complete_count > 0)
178    self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
179    self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
180    self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
181    self.assertEqual(self.mock_logger.dot_count, 2)
182    self.assertEqual(mock_progress_string.call_count, 0)
183    self.assertEqual(mock_status_string.call_count, 2)
184    self.assertEqual(self.mock_logger.output_msgs,
185                     ['==============================', 'Fake status string',
186                      '=============================='])
187    self.assertEqual(len(self.mock_logger.error_msgs), 0)
188
189
190    # Test 3: log_level == "verbose"
191    self.mock_logger.Reset()
192    reset()
193    self.exp.log_level = "verbose"
194    mock_status_string.call_count = 0
195    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
196                                            self.mock_cmd_exec)
197    er.STATUS_TIME_DELAY = 2
198    mock_status_string.return_value = "Fake status string"
199    mock_progress_string.return_value = "Fake progress string"
200    er._Run(self.exp)
201    self.assertEqual(self.run_count, 1)
202    self.assertTrue(self.is_complete_count > 0)
203    self.assertEqual(self.mock_logger.LogStartDotsCount, 0)
204    self.assertEqual(self.mock_logger.LogAppendDotCount, 0)
205    self.assertEqual(self.mock_logger.LogEndDotsCount, 0)
206    self.assertEqual(self.mock_logger.dot_count, 0)
207    self.assertEqual(mock_progress_string.call_count, 2)
208    self.assertEqual(mock_status_string.call_count, 2)
209    self.assertEqual(self.mock_logger.output_msgs,
210                     ['==============================',
211                      'Fake progress string', 'Fake status string',
212                      '==============================',
213                      '==============================',
214                      'Fake progress string', 'Fake status string',
215                      '=============================='])
216    self.assertEqual(len(self.mock_logger.error_msgs), 0)
217
218
219  @mock.patch.object(TextResultsReport, 'GetReport')
220  def test_print_table(self, mock_report):
221    self.mock_logger.Reset()
222    mock_report.return_value = "This is a fake experiment report."
223    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
224                                            self.mock_cmd_exec)
225    er._PrintTable(self.exp)
226    self.assertEqual(mock_report.call_count, 1)
227    self.assertEqual(self.mock_logger.output_msgs,
228                     [ 'This is a fake experiment report.' ])
229
230
231  @mock.patch.object(HTMLResultsReport, 'GetReport')
232  @mock.patch.object(TextResultsReport, 'GetReport')
233  @mock.patch.object(EmailSender, 'Attachment')
234  @mock.patch.object(EmailSender, 'SendEmail')
235  @mock.patch.object(getpass, 'getuser')
236  def test_email(self, mock_getuser, mock_emailer, mock_attachment,
237                 mock_text_report, mock_html_report):
238
239    mock_getuser.return_value = "john.smith@google.com"
240    mock_text_report.return_value = "This is a fake text report."
241    mock_html_report.return_value = "This is a fake html report."
242
243    self.mock_logger.Reset()
244    config.AddConfig("no_email", True)
245    self.exp.email_to = ["jane.doe@google.com"]
246    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
247                                            self.mock_cmd_exec)
248    # Test 1. Config:no_email; exp.email_to set ==> no email sent
249    er._Email(self.exp)
250    self.assertEqual(mock_getuser.call_count, 0)
251    self.assertEqual(mock_emailer.call_count, 0)
252    self.assertEqual(mock_attachment.call_count, 0)
253    self.assertEqual(mock_text_report.call_count, 0)
254    self.assertEqual(mock_html_report.call_count, 0)
255
256    # Test 2. Config: email. exp.email_to set; cache hit.  => send email
257    self.mock_logger.Reset()
258    config.AddConfig("no_email", False)
259    for r in self.exp.benchmark_runs:
260      r.cache_hit = True
261    er._Email(self.exp)
262    self.assertEqual(mock_getuser.call_count, 1)
263    self.assertEqual(mock_emailer.call_count, 1)
264    self.assertEqual(mock_attachment.call_count, 1)
265    self.assertEqual(mock_text_report.call_count, 1)
266    self.assertEqual(mock_html_report.call_count, 1)
267    self.assertEqual(len(mock_emailer.call_args), 2)
268    self.assertEqual(mock_emailer.call_args[0],
269                     (['john.smith@google.com', 'jane.doe@google.com'],
270                      ': image1 vs. image2',
271                      "<pre style='font-size: 13px'>This is a fake text "
272                      "report.\nResults are stored in _results.\n</pre>"))
273    self.assertTrue(type(mock_emailer.call_args[1]) is dict)
274    self.assertEqual(len(mock_emailer.call_args[1]), 2)
275    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
276    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
277
278    mock_attachment.assert_called_with('report.html',
279                                       'This is a fake html report.')
280
281    # Test 3. Config: email; exp.mail_to set; no cache hit.  => send email
282    self.mock_logger.Reset()
283    mock_getuser.reset_mock()
284    mock_emailer.reset_mock()
285    mock_attachment.reset_mock()
286    mock_text_report.reset_mock()
287    mock_html_report.reset_mock()
288    config.AddConfig("no_email", False)
289    for r in self.exp.benchmark_runs:
290      r.cache_hit = False
291    er._Email(self.exp)
292    self.assertEqual(mock_getuser.call_count, 1)
293    self.assertEqual(mock_emailer.call_count, 1)
294    self.assertEqual(mock_attachment.call_count, 1)
295    self.assertEqual(mock_text_report.call_count, 1)
296    self.assertEqual(mock_html_report.call_count, 1)
297    self.assertEqual(len(mock_emailer.call_args), 2)
298    self.assertEqual(mock_emailer.call_args[0],
299                     (['john.smith@google.com', 'jane.doe@google.com'],
300                      ': image1 vs. image2',
301                      "<pre style='font-size: 13px'>This is a fake text "
302                      "report.\nResults are stored in _results.\n</pre>"))
303    self.assertTrue(type(mock_emailer.call_args[1]) is dict)
304    self.assertEqual(len(mock_emailer.call_args[1]), 2)
305    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
306    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
307
308    mock_attachment.assert_called_with('report.html',
309                                       'This is a fake html report.')
310
311    # Test 4. Config: email; exp.mail_to = None; no cache hit. => send email
312    self.mock_logger.Reset()
313    mock_getuser.reset_mock()
314    mock_emailer.reset_mock()
315    mock_attachment.reset_mock()
316    mock_text_report.reset_mock()
317    mock_html_report.reset_mock()
318    self.exp.email_to = []
319    er._Email(self.exp)
320    self.assertEqual(mock_getuser.call_count, 1)
321    self.assertEqual(mock_emailer.call_count, 1)
322    self.assertEqual(mock_attachment.call_count, 1)
323    self.assertEqual(mock_text_report.call_count, 1)
324    self.assertEqual(mock_html_report.call_count, 1)
325    self.assertEqual(len(mock_emailer.call_args), 2)
326    self.assertEqual(mock_emailer.call_args[0],
327                     (['john.smith@google.com'],
328                      ': image1 vs. image2',
329                      "<pre style='font-size: 13px'>This is a fake text "
330                      "report.\nResults are stored in _results.\n</pre>"))
331    self.assertTrue(type(mock_emailer.call_args[1]) is dict)
332    self.assertEqual(len(mock_emailer.call_args[1]), 2)
333    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
334    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
335
336    mock_attachment.assert_called_with('report.html',
337                                       'This is a fake html report.')
338
339    # Test 5. Config: email; exp.mail_to = None; cache hit => no email sent
340    self.mock_logger.Reset()
341    mock_getuser.reset_mock()
342    mock_emailer.reset_mock()
343    mock_attachment.reset_mock()
344    mock_text_report.reset_mock()
345    mock_html_report.reset_mock()
346    for r in self.exp.benchmark_runs:
347      r.cache_hit = True
348    er._Email(self.exp)
349    self.assertEqual(mock_getuser.call_count, 0)
350    self.assertEqual(mock_emailer.call_count, 0)
351    self.assertEqual(mock_attachment.call_count, 0)
352    self.assertEqual(mock_text_report.call_count, 0)
353    self.assertEqual(mock_html_report.call_count, 0)
354
355  @mock.patch.object(FileUtils, 'RmDir')
356  @mock.patch.object(FileUtils, 'MkDirP')
357  @mock.patch.object(FileUtils, 'WriteFile')
358  @mock.patch.object(HTMLResultsReport, 'GetReport')
359  @mock.patch.object(Result, 'CopyResultsTo')
360  @mock.patch.object(Result, 'CleanUp')
361  def test_store_results(self, mock_cleanup, mock_copy, mock_report,
362                         mock_writefile, mock_mkdir, mock_rmdir):
363
364    self.mock_logger.Reset()
365    self.exp.results_directory='/usr/local/crosperf-results'
366    bench_run = self.exp.benchmark_runs[5]
367    bench_path = '/usr/local/crosperf-results/' + filter(str.isalnum,
368                                                         bench_run.name)
369    self.assertEqual (len(self.exp.benchmark_runs), 6)
370
371    er = experiment_runner.ExperimentRunner(self.exp, self.mock_logger,
372                                            self.mock_cmd_exec)
373
374    # Test 1. Make sure nothing is done if _terminated is true.
375    er._terminated = True
376    er._StoreResults(self.exp)
377    self.assertEqual(mock_cleanup.call_count, 0)
378    self.assertEqual(mock_copy.call_count, 0)
379    self.assertEqual(mock_report.call_count, 0)
380    self.assertEqual(mock_writefile.call_count, 0)
381    self.assertEqual(mock_mkdir.call_count, 0)
382    self.assertEqual(mock_rmdir.call_count, 0)
383    self.assertEqual(self.mock_logger.LogOutputCount, 0)
384
385    # Test 2. _terminated is false; everything works properly.
386    fake_result = Result(self.mock_logger, self.exp.labels[0], "average")
387    for r in self.exp.benchmark_runs:
388      r.result = fake_result
389    er._terminated = False
390    er._StoreResults(self.exp)
391    self.assertEqual(mock_cleanup.call_count, 6)
392    mock_cleanup.called_with(bench_run.benchmark.rm_chroot_tmp)
393    self.assertEqual(mock_copy.call_count, 6)
394    mock_copy.called_with(bench_path)
395    self.assertEqual(mock_writefile.call_count, 3)
396    self.assertEqual (len(mock_writefile.call_args_list), 3)
397    first_args = mock_writefile.call_args_list[0]
398    second_args = mock_writefile.call_args_list[1]
399    self.assertEqual(first_args[0][0],
400                     '/usr/local/crosperf-results/experiment.exp')
401    self.assertEqual(second_args[0][0],
402                     '/usr/local/crosperf-results/results.html')
403    self.assertEqual(mock_mkdir.call_count, 1)
404    mock_mkdir.called_with('/usr/local/crosperf-results')
405    self.assertEqual(mock_rmdir.call_count, 1)
406    mock_rmdir.called_with('/usr/local/crosperf-results')
407    self.assertEqual(self.mock_logger.LogOutputCount, 4)
408    self.assertEqual(self.mock_logger.output_msgs,
409                     [ 'Storing experiment file in /usr/local/crosperf-results.',
410                       'Storing results report in /usr/local/crosperf-results.',
411                       'Storing email message body in /usr/local/crosperf-results.',
412                       'Storing results of each benchmark run.' ])
413
414
415
416if __name__ == '__main__':
417  unittest.main()
418