buildbot_test_toolchains.py revision 43494297a1745c084d8a2f3beb52ff1c70a2007c 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18import urllib2
19
20from utils import command_executer
21from utils import logger
22
23from utils import buildbot_utils
24
25# CL that updated GCC ebuilds to use 'next_gcc'.
26USE_NEXT_GCC_PATCH = "230260"
27
28# CL that uses LLVM to build the peppy image.
29USE_LLVM_PATCH = "295217"
30
31# The boards on which we run weekly reports
32WEEKLY_REPORT_BOARDS = ["lumpy"]
33
34CROSTC_ROOT = "/usr/local/google/crostc"
35ROLE_ACCOUNT = "mobiletc-prebuild"
36TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
37MAIL_PROGRAM = "~/var/bin/mail-sheriff"
38WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data")
39PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives")
40NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports")
41
42class ToolchainComparator():
43  """
44  Class for doing the nightly tests work.
45  """
46
47  def __init__(self, board, remotes, chromeos_root, weekday, patches, noschedv2=False):
48    self._board = board
49    self._remotes = remotes
50    self._chromeos_root = chromeos_root
51    self._base_dir = os.getcwd()
52    self._ce = command_executer.GetCommandExecuter()
53    self._l = logger.GetLogger()
54    self._build = "%s-release" % board
55    self._patches = patches.split(',')
56    self._patches_string = '_'.join(str(p) for p in self._patches)
57    self._noschedv2 = noschedv2
58
59    if not weekday:
60      self._weekday = time.strftime("%a")
61    else:
62      self._weekday = weekday
63    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
64                                           "%Y-%m-%d_%H:%M:%S")
65    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
66        "%s.%s" % (timestamp, board),
67        )
68
69  def _ParseVanillaImage(self, trybot_image):
70    """
71    Parse a trybot artifact name to get corresponding vanilla image.
72
73    This function takes an artifact name, such as
74    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
75    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
76    """
77    start_pos = trybot_image.find(self._build)
78    end_pos = trybot_image.rfind("-b")
79    vanilla_image = trybot_image[start_pos:end_pos]
80    return vanilla_image
81
82  def _FinishSetup(self):
83    """
84    Make sure testing_rsa file is properly set up.
85    """
86    # Fix protections on ssh key
87    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
88               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
89               "/testing_rsa")
90    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
91    if ret_val != 0:
92      raise RuntimeError("chmod for testing_rsa failed")
93
94  def _TestImages(self, trybot_image, vanilla_image):
95    """
96    Create crosperf experiment file.
97
98    Given the names of the trybot and vanilla images, create the
99    appropriate crosperf experiment file and launch crosperf on it.
100    """
101    experiment_file_dir = os.path.join (self._chromeos_root, "..",
102                                        self._weekday)
103    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
104
105    if self._patches_string == USE_LLVM_PATCH:
106      experiment_file_name = "%s_llvm_experiment.txt" % self._board
107
108    experiment_file = os.path.join (experiment_file_dir,
109                                    experiment_file_name)
110    experiment_header = """
111    board: %s
112    remote: %s
113    retries: 1
114    """ % (self._board, self._remotes)
115    experiment_tests = """
116    benchmark: all_toolchain_perf {
117      suite: telemetry_Crosperf
118      iterations: 3
119    }
120    """
121    with open(experiment_file, "w") as f:
122      print >> f, experiment_header
123      print >> f, experiment_tests
124
125      # Now add vanilla to test file.
126      official_image = """
127          vanilla_image {
128            chromeos_root: %s
129            build: %s
130          }
131          """ % (self._chromeos_root, vanilla_image)
132      print >> f, official_image
133
134      experiment_image = """
135          test_image {
136            chromeos_root: %s
137            build: %s
138          }
139          """ % (self._chromeos_root, trybot_image)
140      print >> f, experiment_image
141
142    crosperf = os.path.join(TOOLCHAIN_DIR,
143                            "crosperf",
144                            "crosperf")
145    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
146    command = ("{crosperf} --no_email=True --results_dir={r_dir} "
147               "--json_report=True {noschedv2_opts} {exp_file}").format(
148                crosperf=crosperf,
149                r_dir=self._reports_dir,
150                noschedv2_opts=noschedv2_opts,
151                exp_file=experiment_file)
152
153    ret = self._ce.RunCommand(command)
154    if ret != 0:
155      raise RuntimeError("Couldn't run crosperf!")
156    else:
157      # Copy json report to pending archives directory.
158      command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR)
159      ret = self._ce.RunCommand(command)
160    return
161
162  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
163    """
164    Put files in place for running seven-day reports.
165
166    Create tar files of the custom and official images and copy them
167    to the weekly reports directory, so they exist when the weekly report
168    gets generated.  IMPORTANT NOTE: This function must run *after*
169    crosperf has been run; otherwise the vanilla images will not be there.
170    """
171
172    dry_run = False
173    if (os.getlogin() != ROLE_ACCOUNT):
174      self._l.LogOutput("Running this from non-role account; not copying "
175                        "tar files for weekly reports.")
176      dry_run = True
177
178    images_path = os.path.join(os.path.realpath(self._chromeos_root),
179                               "chroot/tmp")
180
181    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
182    dest_dir = os.path.join (data_dir, self._weekday)
183    if not os.path.exists(dest_dir):
184      os.makedirs(dest_dir)
185
186    # Make sure dest_dir is empty (clean out last week's data).
187    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
188    if dry_run:
189      print "CMD: %s" % cmd
190    else:
191      self._ce.RunCommand(cmd)
192
193    # Now create new tar files and copy them over.
194    labels = [ "test", "vanilla" ]
195    for label_name in labels:
196      if label_name == "test":
197        test_path = trybot_image
198      else:
199        test_path = vanilla_image
200      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
201      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
202             "cp %s %s/.") % (images_path,
203                              tar_file_name,
204                              test_path,
205                              tar_file_name,
206                              dest_dir)
207      if dry_run:
208        print "CMD: %s" % cmd
209        tar_ret = 0
210      else:
211        tar_ret = self._ce.RunCommand(cmd)
212      if tar_ret:
213        self._l.LogOutput("Error while creating/copying test tar file(%s)."
214                          % tar_file_name)
215
216  def _SendEmail(self):
217    """Find email message generated by crosperf and send it."""
218    filename = os.path.join(self._reports_dir,
219                            "msg_body.html")
220    if (os.path.exists(filename) and
221        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
222      email_title = "buildbot test results"
223      if self._patches_string == USE_LLVM_PATCH:
224        email_title = "buildbot llvm test results"
225      command = ('cat %s | %s -s "%s, %s" -team -html'
226                 % (filename, MAIL_PROGRAM, email_title, self._board))
227      self._ce.RunCommand(command)
228
229  def DoAll(self):
230    """
231    Main function inside ToolchainComparator class.
232
233    Launch trybot, get image names, create crosperf experiment file, run
234    crosperf, and copy images into seven-day report directories.
235    """
236    date_str = datetime.date.today()
237    description = "master_%s_%s_%s" % (self._patches_string,
238                                       self._build,
239                                       date_str)
240    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
241                                                 self._build,
242                                                 self._patches,
243                                                 description,
244                                                 build_toolchain=True)
245
246    vanilla_image = self._ParseVanillaImage(trybot_image)
247
248    print ("trybot_image: %s" % trybot_image)
249    print ("vanilla_image: %s" % vanilla_image)
250    if len(trybot_image) == 0:
251        self._l.LogError("Unable to find trybot_image for %s!" % description)
252        return 1
253    if len(vanilla_image) == 0:
254        self._l.LogError("Unable to find vanilla image for %s!" % description)
255        return 1
256    if os.getlogin() == ROLE_ACCOUNT:
257      self._FinishSetup()
258
259    self._TestImages(trybot_image, vanilla_image)
260    self._SendEmail()
261    if (self._patches_string == USE_NEXT_GCC_PATCH and
262        self._board in WEEKLY_REPORT_BOARDS):
263        # Only try to copy the image files if the test runs ran successfully.
264        self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
265    return 0
266
267
268def Main(argv):
269  """The main function."""
270
271  # Common initializations
272  command_executer.InitCommandExecuter()
273  parser = optparse.OptionParser()
274  parser.add_option("--remote",
275                    dest="remote",
276                    help="Remote machines to run tests on.")
277  parser.add_option("--board",
278                    dest="board",
279                    default="x86-zgb",
280                    help="The target board.")
281  parser.add_option("--chromeos_root",
282                    dest="chromeos_root",
283                    help="The chromeos root from which to run tests.")
284  parser.add_option("--weekday", default="",
285                    dest="weekday",
286                    help="The day of the week for which to run tests.")
287  parser.add_option("--patch",
288                    dest="patches",
289                    help="The patches to use for the testing, "
290                    "seprate the patch numbers with ',' "
291                    "for more than one patches.")
292  parser.add_option("--noschedv2",
293                    dest="noschedv2",
294                    action="store_true",
295                    default=False,
296                    help="Pass --noschedv2 to crosperf.")
297
298  options, _ = parser.parse_args(argv)
299  if not options.board:
300    print "Please give a board."
301    return 1
302  if not options.remote:
303    print "Please give at least one remote machine."
304    return 1
305  if not options.chromeos_root:
306    print "Please specify the ChromeOS root directory."
307    return 1
308  if options.patches:
309    patches = options.patches
310  else:
311    patches = USE_NEXT_GCC_PATCH
312
313  fc = ToolchainComparator(options.board, options.remote,
314                           options.chromeos_root, options.weekday, patches,
315                           options.noschedv2)
316  return fc.DoAll()
317
318
319if __name__ == "__main__":
320  retval = Main(sys.argv)
321  sys.exit(retval)
322