crb_driver.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4
5import datetime
6import optparse
7import os
8import smtplib
9import sys
10import time
11from email.mime.text import MIMEText
12
13from autotest_gatherer import AutotestGatherer as AutotestGatherer
14from autotest_run import AutotestRun as AutotestRun
15from machine_manager_singleton import MachineManagerSingleton as MachineManagerSingleton
16from utils import logger
17from utils.file_utils import FileUtils
18
19
20def CanonicalizeChromeOSRoot(chromeos_root):
21  chromeos_root = os.path.expanduser(chromeos_root)
22  if os.path.isfile(os.path.join(chromeos_root, 'src/scripts/enter_chroot.sh')):
23    return chromeos_root
24  else:
25    return None
26
27
28class Autotest(object):
29
30  def __init__(self, autotest_string):
31    self.name = None
32    self.iterations = None
33    self.args = None
34    fields = autotest_string.split(',', 1)
35    self.name = fields[0]
36    if len(fields) > 1:
37      autotest_string = fields[1]
38      fields = autotest_string.split(',', 1)
39    else:
40      return
41    self.iterations = int(fields[0])
42    if len(fields) > 1:
43      self.args = fields[1]
44    else:
45      return
46
47  def __str__(self):
48    return '\n'.join([self.name, self.iterations, self.args])
49
50
51def CreateAutotestListFromString(autotest_strings, default_iterations=None):
52  autotest_list = []
53  for autotest_string in autotest_strings.split(':'):
54    autotest = Autotest(autotest_string)
55    if default_iterations and not autotest.iterations:
56      autotest.iterations = default_iterations
57
58    autotest_list.append(autotest)
59  return autotest_list
60
61
62def CreateAutotestRuns(images,
63                       autotests,
64                       remote,
65                       board,
66                       exact_remote,
67                       rerun,
68                       rerun_if_failed,
69                       main_chromeos_root=None):
70  autotest_runs = []
71  for image in images:
72    logger.GetLogger().LogOutput('Computing md5sum of: %s' % image)
73    image_checksum = FileUtils().Md5File(image)
74    logger.GetLogger().LogOutput('md5sum %s: %s' % (image, image_checksum))
75    ###    image_checksum = "abcdefghi"
76
77    chromeos_root = main_chromeos_root
78    if not main_chromeos_root:
79      image_chromeos_root = os.path.join(
80          os.path.dirname(image), '../../../../..')
81      chromeos_root = CanonicalizeChromeOSRoot(image_chromeos_root)
82      assert chromeos_root, 'chromeos_root: %s invalid' % image_chromeos_root
83    else:
84      chromeos_root = CanonicalizeChromeOSRoot(main_chromeos_root)
85      assert chromeos_root, 'chromeos_root: %s invalid' % main_chromeos_root
86
87    # We just need a single ChromeOS root in the MachineManagerSingleton. It is
88    # needed because we can save re-image time by checking the image checksum at
89    # the beginning and assigning autotests to machines appropriately.
90    if not MachineManagerSingleton().chromeos_root:
91      MachineManagerSingleton().chromeos_root = chromeos_root
92
93    for autotest in autotests:
94      for iteration in range(autotest.iterations):
95        autotest_run = AutotestRun(autotest,
96                                   chromeos_root=chromeos_root,
97                                   chromeos_image=image,
98                                   board=board,
99                                   remote=remote,
100                                   iteration=iteration,
101                                   image_checksum=image_checksum,
102                                   exact_remote=exact_remote,
103                                   rerun=rerun,
104                                   rerun_if_failed=rerun_if_failed)
105        autotest_runs.append(autotest_run)
106  return autotest_runs
107
108
109def GetNamesAndIterations(autotest_runs):
110  strings = []
111  for autotest_run in autotest_runs:
112    strings.append('%s:%s' % (autotest_run.autotest.name,
113                              autotest_run.iteration))
114  return ' %s (%s)' % (len(strings), ' '.join(strings))
115
116
117def GetStatusString(autotest_runs):
118  status_bins = {}
119  for autotest_run in autotest_runs:
120    if autotest_run.status not in status_bins:
121      status_bins[autotest_run.status] = []
122    status_bins[autotest_run.status].append(autotest_run)
123
124  status_strings = []
125  for key, val in status_bins.items():
126    status_strings.append('%s: %s' % (key, GetNamesAndIterations(val)))
127  return 'Thread Status:\n%s' % '\n'.join(status_strings)
128
129
130def GetProgressBar(num_done, num_total):
131  ret = 'Done: %s%%' % int(100.0 * num_done / num_total)
132  bar_length = 50
133  done_char = '>'
134  undone_char = ' '
135  num_done_chars = bar_length * num_done / num_total
136  num_undone_chars = bar_length - num_done_chars
137  ret += ' [%s%s]' % (num_done_chars * done_char,
138                      num_undone_chars * undone_char)
139  return ret
140
141
142def GetProgressString(start_time, num_remain, num_total):
143  current_time = time.time()
144  elapsed_time = current_time - start_time
145  try:
146    eta_seconds = float(num_remain) * elapsed_time / (num_total - num_remain)
147    eta_seconds = int(eta_seconds)
148    eta = datetime.timedelta(seconds=eta_seconds)
149  except ZeroDivisionError:
150    eta = 'Unknown'
151  strings = []
152  strings.append('Current time: %s Elapsed: %s ETA: %s' %
153                 (datetime.datetime.now(),
154                  datetime.timedelta(seconds=int(elapsed_time)), eta))
155  strings.append(GetProgressBar(num_total - num_remain, num_total))
156  return '\n'.join(strings)
157
158
159def RunAutotestRunsInParallel(autotest_runs):
160  start_time = time.time()
161  active_threads = []
162  for autotest_run in autotest_runs:
163    # Set threads to daemon so program exits when ctrl-c is pressed.
164    autotest_run.daemon = True
165    autotest_run.start()
166    active_threads.append(autotest_run)
167
168  print_interval = 30
169  last_printed_time = time.time()
170  while active_threads:
171    try:
172      active_threads = [t for t in active_threads
173                        if t is not None and t.isAlive()]
174      for t in active_threads:
175        t.join(1)
176      if time.time() - last_printed_time > print_interval:
177        border = '=============================='
178        logger.GetLogger().LogOutput(border)
179        logger.GetLogger().LogOutput(GetProgressString(start_time, len(
180            [t for t in autotest_runs if t.status not in ['SUCCEEDED', 'FAILED']
181            ]), len(autotest_runs)))
182        logger.GetLogger().LogOutput(GetStatusString(autotest_runs))
183        logger.GetLogger().LogOutput('%s\n' %
184                                     MachineManagerSingleton().AsString())
185        logger.GetLogger().LogOutput(border)
186        last_printed_time = time.time()
187    except KeyboardInterrupt:
188      print 'C-c received... cleaning up threads.'
189      for t in active_threads:
190        t.terminate = True
191      return 1
192  return 0
193
194
195def RunAutotestRunsSerially(autotest_runs):
196  for autotest_run in autotest_runs:
197    retval = autotest_run.Run()
198    if retval:
199      return retval
200
201
202def ProduceTables(autotest_runs, full_table, fit_string):
203  l = logger.GetLogger()
204  ags_dict = {}
205  for autotest_run in autotest_runs:
206    name = autotest_run.full_name
207    if name not in ags_dict:
208      ags_dict[name] = AutotestGatherer()
209    ags_dict[name].runs.append(autotest_run)
210    output = ''
211  for b, ag in ags_dict.items():
212    output += 'Benchmark: %s\n' % b
213    output += ag.GetFormattedMainTable(percents_only=not full_table,
214                                       fit_string=fit_string)
215    output += '\n'
216
217  summary = ''
218  for b, ag in ags_dict.items():
219    summary += 'Benchmark Summary Table: %s\n' % b
220    summary += ag.GetFormattedSummaryTable(percents_only=not full_table,
221                                           fit_string=fit_string)
222    summary += '\n'
223
224  output += summary
225  output += ('Number of re-images performed: %s' %
226             MachineManagerSingleton().num_reimages)
227  l.LogOutput(output)
228
229  if autotest_runs:
230    board = autotest_runs[0].board
231  else:
232    board = ''
233
234  subject = '%s: %s' % (board, ', '.join(ags_dict.keys()))
235
236  if any(autotest_run.run_completed for autotest_run in autotest_runs):
237    SendEmailToUser(subject, summary)
238
239
240def SendEmailToUser(subject, text_to_send):
241  # Email summary to the current user.
242  msg = MIMEText(text_to_send)
243
244  # me == the sender's email address
245  # you == the recipient's email address
246  me = os.path.basename(__file__)
247  you = os.getlogin()
248  msg['Subject'] = '[%s] %s' % (os.path.basename(__file__), subject)
249  msg['From'] = me
250  msg['To'] = you
251
252  # Send the message via our own SMTP server, but don't include the
253  # envelope header.
254  s = smtplib.SMTP('localhost')
255  s.sendmail(me, [you], msg.as_string())
256  s.quit()
257
258
259def Main(argv):
260  """The main function."""
261  # Common initializations
262  ###  command_executer.InitCommandExecuter(True)
263  l = logger.GetLogger()
264
265  parser = optparse.OptionParser()
266  parser.add_option('-t',
267                    '--tests',
268                    dest='tests',
269                    help=('Tests to compare.'
270                          'Optionally specify per-test iterations by:'
271                          '<test>,<iter>:<args>'))
272  parser.add_option('-c',
273                    '--chromeos_root',
274                    dest='chromeos_root',
275                    help='A *single* chromeos_root where scripts can be found.')
276  parser.add_option('-n',
277                    '--iterations',
278                    dest='iterations',
279                    help='Iterations to run per benchmark.',
280                    default=1)
281  parser.add_option('-r',
282                    '--remote',
283                    dest='remote',
284                    help='The remote chromeos machine.')
285  parser.add_option('-b', '--board', dest='board', help='The remote board.')
286  parser.add_option('--full_table',
287                    dest='full_table',
288                    help='Print full tables.',
289                    action='store_true',
290                    default=True)
291  parser.add_option('--exact_remote',
292                    dest='exact_remote',
293                    help='Run tests on the exact remote.',
294                    action='store_true',
295                    default=False)
296  parser.add_option('--fit_string',
297                    dest='fit_string',
298                    help='Fit strings to fixed sizes.',
299                    action='store_true',
300                    default=False)
301  parser.add_option('--rerun',
302                    dest='rerun',
303                    help='Re-run regardless of cache hit.',
304                    action='store_true',
305                    default=False)
306  parser.add_option('--rerun_if_failed',
307                    dest='rerun_if_failed',
308                    help='Re-run if previous run was a failure.',
309                    action='store_true',
310                    default=False)
311  parser.add_option('--no_lock',
312                    dest='no_lock',
313                    help='Do not lock the machine before running the tests.',
314                    action='store_true',
315                    default=False)
316  l.LogOutput(' '.join(argv))
317  [options, args] = parser.parse_args(argv)
318
319  if options.remote is None:
320    l.LogError('No remote machine specified.')
321    parser.print_help()
322    return 1
323
324  if not options.board:
325    l.LogError('No board specified.')
326    parser.print_help()
327    return 1
328
329  remote = options.remote
330  tests = options.tests
331  board = options.board
332  exact_remote = options.exact_remote
333  iterations = int(options.iterations)
334
335  autotests = CreateAutotestListFromString(tests, iterations)
336
337  main_chromeos_root = options.chromeos_root
338  images = args[1:]
339  fit_string = options.fit_string
340  full_table = options.full_table
341  rerun = options.rerun
342  rerun_if_failed = options.rerun_if_failed
343
344  MachineManagerSingleton().no_lock = options.no_lock
345
346  # Now try creating all the Autotests
347  autotest_runs = CreateAutotestRuns(images, autotests, remote, board,
348                                     exact_remote, rerun, rerun_if_failed,
349                                     main_chromeos_root)
350
351  try:
352    # At this point we have all the autotest runs.
353    for machine in remote.split(','):
354      MachineManagerSingleton().AddMachine(machine)
355
356    retval = RunAutotestRunsInParallel(autotest_runs)
357    if retval:
358      return retval
359
360    # Now print tables
361    ProduceTables(autotest_runs, full_table, fit_string)
362  finally:
363    # not sure why this isn't called at the end normally...
364    MachineManagerSingleton().__del__()
365
366  return 0
367
368
369if __name__ == '__main__':
370  sys.exit(Main(sys.argv))
371