crb_driver.py revision c7f1593f9af3ea1b9264b37628c36f3a70e1749a 
1#!/usr/bin/python
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4
5import datetime
6import optparse
7import os
8import smtplib
9import sys
10import time
11from email.mime.text import MIMEText
12
13from autotest_gatherer import AutotestGatherer as AutotestGatherer
14from autotest_run import AutotestRun as AutotestRun
15from machine_manager_singleton import MachineManagerSingleton as MachineManagerSingleton
16from utils import logger
17from utils.file_utils import FileUtils
18
19
20def CanonicalizeChromeOSRoot(chromeos_root):
21  chromeos_root = os.path.expanduser(chromeos_root)
22  if os.path.isfile(os.path.join(chromeos_root,
23                                 "src/scripts/enter_chroot.sh")):
24    return chromeos_root
25  else:
26    return None
27
28
29class Autotest(object):
30  def __init__(self, autotest_string):
31    self.name = None
32    self.iterations = None
33    self.args = None
34    fields = autotest_string.split(",", 1)
35    self.name = fields[0]
36    if len(fields) > 1:
37      autotest_string = fields[1]
38      fields = autotest_string.split(",", 1)
39    else: return
40    self.iterations = int(fields[0])
41    if len(fields) > 1:
42      self.args = fields[1]
43    else: return
44
45  def __str__(self):
46    return "\n".join([self.name, self.iterations, self.args])
47
48
49def CreateAutotestListFromString(autotest_strings, default_iterations=None):
50  autotest_list = []
51  for autotest_string in autotest_strings.split(":"):
52    autotest = Autotest(autotest_string)
53    if default_iterations and not autotest.iterations:
54      autotest.iterations = default_iterations
55
56    autotest_list.append(autotest)
57  return autotest_list
58
59
60def CreateAutotestRuns(images, autotests, remote, board, exact_remote,
61                       rerun, rerun_if_failed, main_chromeos_root=None):
62  autotest_runs = []
63  for image in images:
64    logger.GetLogger().LogOutput("Computing md5sum of: %s" % image)
65    image_checksum = FileUtils().Md5File(image)
66    logger.GetLogger().LogOutput("md5sum %s: %s" % (image, image_checksum))
67###    image_checksum = "abcdefghi"
68
69    chromeos_root = main_chromeos_root
70    if not main_chromeos_root:
71      image_chromeos_root = os.path.join(os.path.dirname(image),
72                                         "../../../../..")
73      chromeos_root = CanonicalizeChromeOSRoot(image_chromeos_root)
74      assert chromeos_root, "chromeos_root: %s invalid" % image_chromeos_root
75    else:
76      chromeos_root = CanonicalizeChromeOSRoot(main_chromeos_root)
77      assert chromeos_root, "chromeos_root: %s invalid" % main_chromeos_root
78
79    # We just need a single ChromeOS root in the MachineManagerSingleton. It is
80    # needed because we can save re-image time by checking the image checksum at
81    # the beginning and assigning autotests to machines appropriately.
82    if not MachineManagerSingleton().chromeos_root:
83      MachineManagerSingleton().chromeos_root = chromeos_root
84
85    for autotest in autotests:
86      for iteration in range(autotest.iterations):
87        autotest_run = AutotestRun(autotest,
88                                   chromeos_root=chromeos_root,
89                                   chromeos_image=image,
90                                   board=board,
91                                   remote=remote,
92                                   iteration=iteration,
93                                   image_checksum=image_checksum,
94                                   exact_remote=exact_remote,
95                                   rerun=rerun,
96                                   rerun_if_failed=rerun_if_failed)
97        autotest_runs.append(autotest_run)
98  return autotest_runs
99
100
101def GetNamesAndIterations(autotest_runs):
102  strings = []
103  for autotest_run in autotest_runs:
104    strings.append("%s:%s" % (autotest_run.autotest.name,
105                              autotest_run.iteration))
106  return " %s (%s)" % (len(strings), " ".join(strings))
107
108
109def GetStatusString(autotest_runs):
110  status_bins = {}
111  for autotest_run in autotest_runs:
112    if autotest_run.status not in status_bins:
113      status_bins[autotest_run.status] = []
114    status_bins[autotest_run.status].append(autotest_run)
115
116  status_strings = []
117  for key, val in status_bins.items():
118    status_strings.append("%s: %s" % (key, GetNamesAndIterations(val)))
119  return "Thread Status:\n%s" % "\n".join(status_strings)
120
121
122def GetProgressBar(num_done, num_total):
123  ret = "Done: %s%%" % int(100.0 * num_done / num_total)
124  bar_length = 50
125  done_char = ">"
126  undone_char = " "
127  num_done_chars = bar_length * num_done / num_total
128  num_undone_chars = bar_length - num_done_chars
129  ret += " [%s%s]" % (num_done_chars * done_char, num_undone_chars *
130                      undone_char)
131  return ret
132
133
134def GetProgressString(start_time, num_remain, num_total):
135  current_time = time.time()
136  elapsed_time = current_time - start_time
137  try:
138    eta_seconds = float(num_remain) * elapsed_time / (num_total - num_remain)
139    eta_seconds = int(eta_seconds)
140    eta = datetime.timedelta(seconds=eta_seconds)
141  except ZeroDivisionError:
142    eta = "Unknown"
143  strings = []
144  strings.append("Current time: %s Elapsed: %s ETA: %s" %
145          (datetime.datetime.now(),
146           datetime.timedelta(seconds=int(elapsed_time)),
147           eta))
148  strings.append(GetProgressBar(num_total - num_remain, num_total))
149  return "\n".join(strings)
150
151
152def RunAutotestRunsInParallel(autotest_runs):
153  start_time = time.time()
154  active_threads = []
155  for autotest_run in autotest_runs:
156    # Set threads to daemon so program exits when ctrl-c is pressed.
157    autotest_run.daemon = True
158    autotest_run.start()
159    active_threads.append(autotest_run)
160
161  print_interval = 30
162  last_printed_time = time.time()
163  while active_threads:
164    try:
165      active_threads = [t for t in active_threads if t is not None
166                        and t.isAlive()]
167      for t in active_threads:
168        t.join(1)
169      if time.time() - last_printed_time > print_interval:
170        border = "=============================="
171        logger.GetLogger().LogOutput(border)
172        logger.GetLogger().LogOutput(GetProgressString(
173            start_time,
174            len([t for t in autotest_runs if t.status not in ["SUCCEEDED",
175                                                              "FAILED"]]),
176            len(autotest_runs)))
177        logger.GetLogger().LogOutput(GetStatusString(autotest_runs))
178        logger.GetLogger().LogOutput("%s\n" %
179                                     MachineManagerSingleton().AsString())
180        logger.GetLogger().LogOutput(border)
181        last_printed_time = time.time()
182    except KeyboardInterrupt:
183      print "C-c received... cleaning up threads."
184      for t in active_threads:
185        t.terminate = True
186      return 1
187  return 0
188
189
190def RunAutotestRunsSerially(autotest_runs):
191  for autotest_run in autotest_runs:
192    retval = autotest_run.Run()
193    if retval: return retval
194
195
196def ProduceTables(autotest_runs, full_table, fit_string):
197  l = logger.GetLogger()
198  ags_dict = {}
199  for autotest_run in autotest_runs:
200    name = autotest_run.full_name
201    if name not in ags_dict:
202      ags_dict[name] = AutotestGatherer()
203    ags_dict[name].runs.append(autotest_run)
204    output = ""
205  for b, ag in ags_dict.items():
206    output += "Benchmark: %s\n" % b
207    output += ag.GetFormattedMainTable(percents_only=not full_table,
208                                       fit_string=fit_string)
209    output += "\n"
210
211  summary = ""
212  for b, ag in ags_dict.items():
213    summary += "Benchmark Summary Table: %s\n" % b
214    summary += ag.GetFormattedSummaryTable(percents_only=not full_table,
215                                           fit_string=fit_string)
216    summary += "\n"
217
218  output += summary
219  output += ("Number of re-images performed: %s" %
220             MachineManagerSingleton().num_reimages)
221  l.LogOutput(output)
222
223  if autotest_runs:
224    board = autotest_runs[0].board
225  else:
226    board = ""
227
228  subject = "%s: %s" % (board, ", ".join(ags_dict.keys()))
229
230  if any(autotest_run.run_completed for autotest_run in autotest_runs):
231    SendEmailToUser(subject, summary)
232
233
234def SendEmailToUser(subject, text_to_send):
235  # Email summary to the current user.
236  msg = MIMEText(text_to_send)
237
238  # me == the sender's email address
239  # you == the recipient's email address
240  me = os.path.basename(__file__)
241  you = os.getlogin()
242  msg["Subject"] = "[%s] %s" % (os.path.basename(__file__), subject)
243  msg["From"] = me
244  msg["To"] = you
245
246  # Send the message via our own SMTP server, but don't include the
247  # envelope header.
248  s = smtplib.SMTP("localhost")
249  s.sendmail(me, [you], msg.as_string())
250  s.quit()
251
252
253def Main(argv):
254  """The main function."""
255  # Common initializations
256###  command_executer.InitCommandExecuter(True)
257  l = logger.GetLogger()
258
259  parser = optparse.OptionParser()
260  parser.add_option("-t", "--tests", dest="tests",
261                    help=("Tests to compare."
262                          "Optionally specify per-test iterations by:"
263                          "<test>,<iter>:<args>"))
264  parser.add_option("-c", "--chromeos_root", dest="chromeos_root",
265                    help="A *single* chromeos_root where scripts can be found.")
266  parser.add_option("-n", "--iterations", dest="iterations",
267                    help="Iterations to run per benchmark.",
268                    default=1)
269  parser.add_option("-r", "--remote", dest="remote",
270                    help="The remote chromeos machine.")
271  parser.add_option("-b", "--board", dest="board",
272                    help="The remote board.")
273  parser.add_option("--full_table", dest="full_table",
274                    help="Print full tables.",
275                    action="store_true",
276                    default=True)
277  parser.add_option("--exact_remote",
278                    dest="exact_remote",
279                    help="Run tests on the exact remote.",
280                    action="store_true",
281                    default=False)
282  parser.add_option("--fit_string", dest="fit_string",
283                    help="Fit strings to fixed sizes.",
284                    action="store_true",
285                    default=False)
286  parser.add_option("--rerun",
287                    dest="rerun",
288                    help="Re-run regardless of cache hit.",
289                    action="store_true",
290                    default=False)
291  parser.add_option("--rerun_if_failed",
292                    dest="rerun_if_failed",
293                    help="Re-run if previous run was a failure.",
294                    action="store_true",
295                    default=False)
296  parser.add_option("--no_lock",
297                    dest="no_lock",
298                    help="Do not lock the machine before running the tests.",
299                    action="store_true",
300                    default=False)
301  l.LogOutput(" ".join(argv))
302  [options, args] = parser.parse_args(argv)
303
304  if options.remote is None:
305    l.LogError("No remote machine specified.")
306    parser.print_help()
307    return 1
308
309  if not options.board:
310    l.LogError("No board specified.")
311    parser.print_help()
312    return 1
313
314  remote = options.remote
315  tests = options.tests
316  board = options.board
317  exact_remote = options.exact_remote
318  iterations = int(options.iterations)
319
320  autotests = CreateAutotestListFromString(tests, iterations)
321
322  main_chromeos_root = options.chromeos_root
323  images = args[1:]
324  fit_string = options.fit_string
325  full_table = options.full_table
326  rerun = options.rerun
327  rerun_if_failed = options.rerun_if_failed
328
329  MachineManagerSingleton().no_lock = options.no_lock
330
331  # Now try creating all the Autotests
332  autotest_runs = CreateAutotestRuns(images, autotests, remote, board,
333                                     exact_remote, rerun, rerun_if_failed,
334                                     main_chromeos_root)
335
336  try:
337    # At this point we have all the autotest runs.
338    for machine in remote.split(","):
339      MachineManagerSingleton().AddMachine(machine)
340
341    retval = RunAutotestRunsInParallel(autotest_runs)
342    if retval: return retval
343
344    # Now print tables
345    ProduceTables(autotest_runs, full_table, fit_string)
346  finally:
347    # not sure why this isn't called at the end normally...
348    MachineManagerSingleton().__del__()
349
350  return 0
351
352if __name__ == "__main__":
353  sys.exit(Main(sys.argv))
354