buildbot_test_toolchains.py revision 7f20acb9c9036015a665a14f27633c639e9b9d91 
1#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two.  It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import datetime
14import optparse
15import os
16import sys
17import time
18
19from utils import command_executer
20from utils import logger
21
22from utils import buildbot_utils
23
24# CL that updated GCC ebuilds to use 'next_gcc'.
25USE_NEXT_GCC_PATCH = "230260"
26
27# CL that uses LLVM to build the peppy image.
28USE_LLVM_PATCH = "295217"
29
30# The boards on which we run weekly reports
31WEEKLY_REPORT_BOARDS = ["lumpy"]
32
33CROSTC_ROOT = "/usr/local/google/crostc"
34ROLE_ACCOUNT = "mobiletc-prebuild"
35TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
36MAIL_PROGRAM = "~/var/bin/mail-sheriff"
37WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data")
38PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives")
39NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports")
40
41class ToolchainComparator():
42  """
43  Class for doing the nightly tests work.
44  """
45
46  def __init__(self, board, remotes, chromeos_root, weekday, patches, noschedv2=False):
47    self._board = board
48    self._remotes = remotes
49    self._chromeos_root = chromeos_root
50    self._base_dir = os.getcwd()
51    self._ce = command_executer.GetCommandExecuter()
52    self._l = logger.GetLogger()
53    self._build = "%s-release" % board
54    self._patches = patches.split(',')
55    self._patches_string = '_'.join(str(p) for p in self._patches)
56    self._noschedv2 = noschedv2
57
58    if not weekday:
59      self._weekday = time.strftime("%a")
60    else:
61      self._weekday = weekday
62    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
63                                           "%Y-%m-%d_%H:%M:%S")
64    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
65        "%s.%s" % (timestamp, board),
66        )
67
68  def _ParseVanillaImage(self, trybot_image):
69    """
70    Parse a trybot artifact name to get corresponding vanilla image.
71
72    This function takes an artifact name, such as
73    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
74    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
75    """
76    start_pos = trybot_image.find(self._build)
77    end_pos = trybot_image.rfind("-b")
78    vanilla_image = trybot_image[start_pos:end_pos]
79    return vanilla_image
80
81  def _FinishSetup(self):
82    """
83    Make sure testing_rsa file is properly set up.
84    """
85    # Fix protections on ssh key
86    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
87               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
88               "/testing_rsa")
89    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
90    if ret_val != 0:
91      raise RuntimeError("chmod for testing_rsa failed")
92
93  def _TestImages(self, trybot_image, vanilla_image):
94    """
95    Create crosperf experiment file.
96
97    Given the names of the trybot and vanilla images, create the
98    appropriate crosperf experiment file and launch crosperf on it.
99    """
100    experiment_file_dir = os.path.join (self._chromeos_root, "..",
101                                        self._weekday)
102    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
103
104    compiler_string = "gcc"
105    if self._patches_string == USE_LLVM_PATCH:
106      experiment_file_name = "%s_llvm_experiment.txt" % self._board
107      compiler_string = "llvm"
108
109    experiment_file = os.path.join (experiment_file_dir,
110                                    experiment_file_name)
111    experiment_header = """
112    board: %s
113    remote: %s
114    retries: 1
115    """ % (self._board, self._remotes)
116    experiment_tests = """
117    benchmark: all_toolchain_perf_pure_telemetry {
118      suite: telemetry_Crosperf
119      iterations: 3
120    }
121    """
122    with open(experiment_file, "w") as f:
123      print >> f, experiment_header
124      print >> f, experiment_tests
125
126      # Now add vanilla to test file.
127      official_image = """
128          vanilla_image {
129            chromeos_root: %s
130            build: %s
131            compiler: gcc
132          }
133          """ % (self._chromeos_root, vanilla_image)
134      print >> f, official_image
135
136      label_string = "%s_trybot_image" % compiler_string
137      if USE_NEXT_GCC_PATCH in self._patches:
138        label_string = "gcc_next_trybot_image"
139
140      experiment_image = """
141          %s {
142            chromeos_root: %s
143            build: %s
144            compiler: %s
145          }
146          """ % (label_string, self._chromeos_root, trybot_image,
147                 compiler_string)
148      print >> f, experiment_image
149
150    crosperf = os.path.join(TOOLCHAIN_DIR,
151                            "crosperf",
152                            "crosperf")
153    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
154    command = ("{crosperf} --no_email=True --results_dir={r_dir} "
155               "--json_report=True {noschedv2_opts} {exp_file}").format(
156                crosperf=crosperf,
157                r_dir=self._reports_dir,
158                noschedv2_opts=noschedv2_opts,
159                exp_file=experiment_file)
160
161    ret = self._ce.RunCommand(command)
162    if ret != 0:
163      raise RuntimeError("Couldn't run crosperf!")
164    else:
165      # Copy json report to pending archives directory.
166      command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR)
167      ret = self._ce.RunCommand(command)
168    return
169
170  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
171    """
172    Put files in place for running seven-day reports.
173
174    Create tar files of the custom and official images and copy them
175    to the weekly reports directory, so they exist when the weekly report
176    gets generated.  IMPORTANT NOTE: This function must run *after*
177    crosperf has been run; otherwise the vanilla images will not be there.
178    """
179
180    dry_run = False
181    if (os.getlogin() != ROLE_ACCOUNT):
182      self._l.LogOutput("Running this from non-role account; not copying "
183                        "tar files for weekly reports.")
184      dry_run = True
185
186    images_path = os.path.join(os.path.realpath(self._chromeos_root),
187                               "chroot/tmp")
188
189    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
190    dest_dir = os.path.join (data_dir, self._weekday)
191    if not os.path.exists(dest_dir):
192      os.makedirs(dest_dir)
193
194    # Make sure dest_dir is empty (clean out last week's data).
195    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
196    if dry_run:
197      print "CMD: %s" % cmd
198    else:
199      self._ce.RunCommand(cmd)
200
201    # Now create new tar files and copy them over.
202    labels = [ "test", "vanilla" ]
203    for label_name in labels:
204      if label_name == "test":
205        test_path = trybot_image
206      else:
207        test_path = vanilla_image
208      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
209      cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
210             "cp %s %s/.") % (images_path,
211                              tar_file_name,
212                              test_path,
213                              tar_file_name,
214                              dest_dir)
215      if dry_run:
216        print "CMD: %s" % cmd
217        tar_ret = 0
218      else:
219        tar_ret = self._ce.RunCommand(cmd)
220      if tar_ret:
221        self._l.LogOutput("Error while creating/copying test tar file(%s)."
222                          % tar_file_name)
223
224  def _SendEmail(self):
225    """Find email message generated by crosperf and send it."""
226    filename = os.path.join(self._reports_dir,
227                            "msg_body.html")
228    if (os.path.exists(filename) and
229        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
230      email_title = "buildbot test results"
231      if self._patches_string == USE_LLVM_PATCH:
232        email_title = "buildbot llvm test results"
233      command = ('cat %s | %s -s "%s, %s" -team -html'
234                 % (filename, MAIL_PROGRAM, email_title, self._board))
235      self._ce.RunCommand(command)
236
237  def DoAll(self):
238    """
239    Main function inside ToolchainComparator class.
240
241    Launch trybot, get image names, create crosperf experiment file, run
242    crosperf, and copy images into seven-day report directories.
243    """
244    date_str = datetime.date.today()
245    description = "master_%s_%s_%s" % (self._patches_string,
246                                       self._build,
247                                       date_str)
248    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
249                                                 self._build,
250                                                 self._patches,
251                                                 description,
252                                                 build_toolchain=True)
253
254    vanilla_image = self._ParseVanillaImage(trybot_image)
255
256    print ("trybot_image: %s" % trybot_image)
257    print ("vanilla_image: %s" % vanilla_image)
258    if len(trybot_image) == 0:
259      self._l.LogError("Unable to find trybot_image for %s!" % description)
260      return 1
261    if len(vanilla_image) == 0:
262      self._l.LogError("Unable to find vanilla image for %s!" % description)
263      return 1
264    if os.getlogin() == ROLE_ACCOUNT:
265      self._FinishSetup()
266
267    self._TestImages(trybot_image, vanilla_image)
268    self._SendEmail()
269    if (self._patches_string == USE_NEXT_GCC_PATCH and
270        self._board in WEEKLY_REPORT_BOARDS):
271      # Only try to copy the image files if the test runs ran successfully.
272      self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
273    return 0
274
275
276def Main(argv):
277  """The main function."""
278
279  # Common initializations
280  command_executer.InitCommandExecuter()
281  parser = optparse.OptionParser()
282  parser.add_option("--remote",
283                    dest="remote",
284                    help="Remote machines to run tests on.")
285  parser.add_option("--board",
286                    dest="board",
287                    default="x86-zgb",
288                    help="The target board.")
289  parser.add_option("--chromeos_root",
290                    dest="chromeos_root",
291                    help="The chromeos root from which to run tests.")
292  parser.add_option("--weekday", default="",
293                    dest="weekday",
294                    help="The day of the week for which to run tests.")
295  parser.add_option("--patch",
296                    dest="patches",
297                    help="The patches to use for the testing, "
298                    "seprate the patch numbers with ',' "
299                    "for more than one patches.")
300  parser.add_option("--noschedv2",
301                    dest="noschedv2",
302                    action="store_true",
303                    default=False,
304                    help="Pass --noschedv2 to crosperf.")
305
306  options, _ = parser.parse_args(argv)
307  if not options.board:
308    print "Please give a board."
309    return 1
310  if not options.remote:
311    print "Please give at least one remote machine."
312    return 1
313  if not options.chromeos_root:
314    print "Please specify the ChromeOS root directory."
315    return 1
316  if options.patches:
317    patches = options.patches
318  else:
319    patches = USE_NEXT_GCC_PATCH
320
321  fc = ToolchainComparator(options.board, options.remote,
322                           options.chromeos_root, options.weekday, patches,
323                           options.noschedv2)
324  return fc.DoAll()
325
326
327if __name__ == "__main__":
328  retval = Main(sys.argv)
329  sys.exit(retval)
330