buildbot_test_toolchains.py revision ddde50532281f7f796dd7dc44b562b29d25ab381 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH = "230260"
27
28# CL that uses LLVM to build the peppy image.
29USE_LLVM_PATCH = "295217"
30
31# The boards on which we run weekly reports
32WEEKLY_REPORT_BOARDS = ["lumpy"]
33
34CROSTC_ROOT = "/usr/local/google/crostc"
35ROLE_ACCOUNT = "mobiletc-prebuild"
36TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
37MAIL_PROGRAM = "~/var/bin/mail-sheriff"
38WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data")
39PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives")
40NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports")
41
42class ToolchainComparator():
43  """
44  Class for doing the nightly tests work.
45  """
46
47  def __init__(self, board, remotes, chromeos_root, weekday, patches, noschedv2=False):
48    self._board = board
49    self._remotes = remotes
50    self._chromeos_root = chromeos_root
51    self._base_dir = os.getcwd()
52    self._ce = command_executer.GetCommandExecuter()
53    self._l = logger.GetLogger()
54    self._build = "%s-release" % board
55    self._patches = patches.split(',')
56    self._patches_string = '_'.join(str(p) for p in self._patches)
57    self._noschedv2 = noschedv2
58
59    if not weekday:
60      self._weekday = time.strftime("%a")
61    else:
62      self._weekday = weekday
63    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
64                                           "%Y-%m-%d_%H:%M:%S")
65    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
66        "%s.%s" % (timestamp, board),
67        )
68
69  def _ParseVanillaImage(self, trybot_image):
70    """
71    Parse a trybot artifact name to get corresponding vanilla image.
72
73    This function takes an artifact name, such as
74    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
75    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
76    """
77    start_pos = trybot_image.find(self._build)
78    end_pos = trybot_image.rfind("-b")
79    vanilla_image = trybot_image[start_pos:end_pos]
80    return vanilla_image
81
82  def _FinishSetup(self):
83    """
84    Make sure testing_rsa file is properly set up.
85    """
86    # Fix protections on ssh key
87    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
88               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
89               "/testing_rsa")
90    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
91    if ret_val != 0:
92      raise RuntimeError("chmod for testing_rsa failed")
93
94  def _TestImages(self, trybot_image, vanilla_image):
95    """
96    Create crosperf experiment file.
97
98    Given the names of the trybot and vanilla images, create the
99    appropriate crosperf experiment file and launch crosperf on it.
100    """
101    experiment_file_dir = os.path.join (self._chromeos_root, "..",
102                                        self._weekday)
103    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
104
105    compiler_string = "gcc"
106    if self._patches_string == USE_LLVM_PATCH:
107      experiment_file_name = "%s_llvm_experiment.txt" % self._board
108      compiler_string = "llvm"
109
110    experiment_file = os.path.join (experiment_file_dir,
111                                    experiment_file_name)
112    experiment_header = """
113    board: %s
114    remote: %s
115    retries: 1
116    """ % (self._board, self._remotes)
117    experiment_tests = """
118    benchmark: all_toolchain_perf {
119      suite: telemetry_Crosperf
120      iterations: 3
121    }
122    """
123    with open(experiment_file, "w") as f:
124      print >> f, experiment_header
125      print >> f, experiment_tests
126
127      # Now add vanilla to test file.
128      official_image = """
129          vanilla_image {
130            chromeos_root: %s
131            build: %s
132            compiler: gcc
133          }
134          """ % (self._chromeos_root, vanilla_image)
135      print >> f, official_image
136
137      experiment_image = """
138          test_image {
139            chromeos_root: %s
140            build: %s
141            compiler: %s
142          }
143          """ % (self._chromeos_root, trybot_image, compiler_string)
144      print >> f, experiment_image
145
146    crosperf = os.path.join(TOOLCHAIN_DIR,
147                            "crosperf",
148                            "crosperf")
149    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
150    command = ("{crosperf} --no_email=True --results_dir={r_dir} "
151               "--json_report=True {noschedv2_opts} {exp_file}").format(
152                crosperf=crosperf,
153                r_dir=self._reports_dir,
154                noschedv2_opts=noschedv2_opts,
155                exp_file=experiment_file)
156
157    ret = self._ce.RunCommand(command)
158    if ret != 0:
159      raise RuntimeError("Couldn't run crosperf!")
160    else:
161      # Copy json report to pending archives directory.
162      command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR)
163      ret = self._ce.RunCommand(command)
164    return
165
166  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
167    """
168    Put files in place for running seven-day reports.
169
170    Create tar files of the custom and official images and copy them
171    to the weekly reports directory, so they exist when the weekly report
172    gets generated.  IMPORTANT NOTE: This function must run *after*
173    crosperf has been run; otherwise the vanilla images will not be there.
174    """
175
176    dry_run = False
177    if (os.getlogin() != ROLE_ACCOUNT):
178      self._l.LogOutput("Running this from non-role account; not copying "
179                        "tar files for weekly reports.")
180      dry_run = True
181
182    images_path = os.path.join(os.path.realpath(self._chromeos_root),
183                               "chroot/tmp")
184
185    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
186    dest_dir = os.path.join (data_dir, self._weekday)
187    if not os.path.exists(dest_dir):
188      os.makedirs(dest_dir)
189
190    # Make sure dest_dir is empty (clean out last week's data).
191    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
192    if dry_run:
193      print "CMD: %s" % cmd
194    else:
195      self._ce.RunCommand(cmd)
196
197    # Now create new tar files and copy them over.
198    labels = [ "test", "vanilla" ]
199    for label_name in labels:
200      if label_name == "test":
201        test_path = trybot_image
202      else:
203        test_path = vanilla_image
204      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
205      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
206             "cp %s %s/.") % (images_path,
207                              tar_file_name,
208                              test_path,
209                              tar_file_name,
210                              dest_dir)
211      if dry_run:
212        print "CMD: %s" % cmd
213        tar_ret = 0
214      else:
215        tar_ret = self._ce.RunCommand(cmd)
216      if tar_ret:
217        self._l.LogOutput("Error while creating/copying test tar file(%s)."
218                          % tar_file_name)
219
220  def _SendEmail(self):
221    """Find email message generated by crosperf and send it."""
222    filename = os.path.join(self._reports_dir,
223                            "msg_body.html")
224    if (os.path.exists(filename) and
225        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
226      email_title = "buildbot test results"
227      if self._patches_string == USE_LLVM_PATCH:
228        email_title = "buildbot llvm test results"
229      command = ('cat %s | %s -s "%s, %s" -team -html'
230                 % (filename, MAIL_PROGRAM, email_title, self._board))
231      self._ce.RunCommand(command)
232
233  def DoAll(self):
234    """
235    Main function inside ToolchainComparator class.
236
237    Launch trybot, get image names, create crosperf experiment file, run
238    crosperf, and copy images into seven-day report directories.
239    """
240    date_str = datetime.date.today()
241    description = "master_%s_%s_%s" % (self._patches_string,
242                                       self._build,
243                                       date_str)
244    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
245                                                 self._build,
246                                                 self._patches,
247                                                 description,
248                                                 build_toolchain=True)
249
250    vanilla_image = self._ParseVanillaImage(trybot_image)
251
252    print ("trybot_image: %s" % trybot_image)
253    print ("vanilla_image: %s" % vanilla_image)
254    if len(trybot_image) == 0:
255        self._l.LogError("Unable to find trybot_image for %s!" % description)
256        return 1
257    if len(vanilla_image) == 0:
258        self._l.LogError("Unable to find vanilla image for %s!" % description)
259        return 1
260    if os.getlogin() == ROLE_ACCOUNT:
261      self._FinishSetup()
262
263    self._TestImages(trybot_image, vanilla_image)
264    self._SendEmail()
265    if (self._patches_string == USE_NEXT_GCC_PATCH and
266        self._board in WEEKLY_REPORT_BOARDS):
267        # Only try to copy the image files if the test runs ran successfully.
268        self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
269    return 0
270
271
272def Main(argv):
273  """The main function."""
274
275  # Common initializations
276  command_executer.InitCommandExecuter()
277  parser = optparse.OptionParser()
278  parser.add_option("--remote",
279                    dest="remote",
280                    help="Remote machines to run tests on.")
281  parser.add_option("--board",
282                    dest="board",
283                    default="x86-zgb",
284                    help="The target board.")
285  parser.add_option("--chromeos_root",
286                    dest="chromeos_root",
287                    help="The chromeos root from which to run tests.")
288  parser.add_option("--weekday", default="",
289                    dest="weekday",
290                    help="The day of the week for which to run tests.")
291  parser.add_option("--patch",
292                    dest="patches",
293                    help="The patches to use for the testing, "
294                    "seprate the patch numbers with ',' "
295                    "for more than one patches.")
296  parser.add_option("--noschedv2",
297                    dest="noschedv2",
298                    action="store_true",
299                    default=False,
300                    help="Pass --noschedv2 to crosperf.")
301
302  options, _ = parser.parse_args(argv)
303  if not options.board:
304    print "Please give a board."
305    return 1
306  if not options.remote:
307    print "Please give at least one remote machine."
308    return 1
309  if not options.chromeos_root:
310    print "Please specify the ChromeOS root directory."
311    return 1
312  if options.patches:
313    patches = options.patches
314  else:
315    patches = USE_NEXT_GCC_PATCH
316
317  fc = ToolchainComparator(options.board, options.remote,
318                           options.chromeos_root, options.weekday, patches,
319                           options.noschedv2)
320  return fc.DoAll()
321
322
323if __name__ == "__main__":
324  retval = Main(sys.argv)
325  sys.exit(retval)
326