test_toolchains.py revision aa700b0a78b10b709b990bcb535bc5588ebb2bb4 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import datetime
5import optparse
6import os
7import sys
8import build_chromeos
9import setup_chromeos
10import time
11from utils import command_executer
12from utils import misc
13from utils import logger
14
15
16WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
17AFDO_BOARDS = ["butterfly", "lumpy", "stumpy", "stout", "parrot", "parrot_ivb"]
18MAIL_PROGRAM = "~/var/bin/mail-sheriff"
19
20class GCCConfig(object):
21  def __init__(self, githash):
22    self.githash = githash
23
24
25class ToolchainConfig:
26  def __init__(self, gcc_config=None, binutils_config=None):
27    self.gcc_config = gcc_config
28
29
30class ChromeOSCheckout(object):
31  def __init__(self, board, chromeos_root):
32    self._board = board
33    self._chromeos_root = chromeos_root
34    self._ce = command_executer.GetCommandExecuter()
35    self._l = logger.GetLogger()
36    self._build_num = None
37
38  def _DeleteChroot(self):
39    command = "cd %s; cros_sdk --delete" % self._chromeos_root
40    return self._ce.RunCommand(command)
41
42  def _DeleteCcahe(self):
43    # crosbug.com/34956
44    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
45    return self._ce.RunCommand(command)
46
47  def _GetBuildNumber(self):
48    """ This function assumes a ChromeOS image has been built in the chroot.
49    It translates the 'latest' symlink in the
50    <chroot>/src/build/images/<board> directory, to find the actual
51    ChromeOS build number for the image that was built.  For example, if
52    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
53    This function would parse it out and assign 'R37-5982' to self._build_num.
54    This is used to determine the official, vanilla build to use for
55    comparison tests.
56    """
57    # Get the path to 'latest'
58    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
59                                              self._board),
60                             "latest")
61    # Translate the symlink to its 'real' path.
62    real_path = os.path.realpath(sym_path)
63    # Break up the path and get the last piece
64    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
65    path_pieces = real_path.split("/")
66    last_piece = path_pieces[-1]
67    # Break this piece into the image number + other pieces, and get the
68    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
69    image_parts = last_piece.split(".")
70    self._build_num = image_parts[0]
71
72  def _BuildAndImage(self, label=""):
73    if (not label or
74        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
75      build_chromeos_args = [build_chromeos.__file__,
76                             "--chromeos_root=%s" % self._chromeos_root,
77                             "--board=%s" % self._board,
78                             "--rebuild"]
79      if self._public:
80        build_chromeos_args.append("--env=USE=-chrome_internal")
81
82      if self._board in AFDO_BOARDS:
83        build_chromeos_args.append("--env=USE=afdo_use")
84
85      ret = build_chromeos.Main(build_chromeos_args)
86      if ret != 0:
87        raise RuntimeError("Couldn't build ChromeOS!")
88
89      if not  self._build_num:
90        self._GetBuildNumber()
91      # Check to see if we need to create the symbolic link for the vanilla
92      # image, and do so if appropriate.
93      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
94        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
95        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
96                                          'chroot/tmp', build_name)
97        misc.LabelLatestImage(self._chromeos_root, self._board, label,
98                              full_vanilla_path)
99      else:
100        misc.LabelLatestImage(self._chromeos_root, self._board, label)
101    return label
102
103  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
104    env_string = misc.GetEnvStringFromDict(env_dict)
105    command = ("%s %s" %
106               (env_string,
107                misc.GetSetupBoardCommand(self._board,
108                                          usepkg=usepkg_flag,
109                                          force=clobber_flag)))
110    ret = self._ce.ChrootRunCommand(self._chromeos_root,
111                                    command)
112    error_str = "Could not setup board: '%s'" % command
113    assert ret == 0, error_str
114
115  def _UnInstallToolchain(self):
116    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
117               misc.GetCtargetFromBoard(self._board,
118                                   self._chromeos_root))
119    ret = self._ce.ChrootRunCommand(self._chromeos_root,
120                                    command)
121    if ret != 0:
122      raise RuntimeError("Couldn't uninstall the toolchain!")
123
124  def _CheckoutChromeOS(self):
125    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
126    if not os.path.exists(self._chromeos_root):
127      setup_chromeos_args = [setup_chromeos.__file__,
128                             "--dir=%s" % self._chromeos_root]
129      if self._public:
130        setup_chromeos_args.append("--public")
131      ret = setup_chromeos.Main(setup_chromeos_args)
132      if ret != 0:
133        raise RuntimeError("Couldn't run setup_chromeos!")
134
135
136  def _BuildToolchain(self, config):
137    # Call setup_board for basic, vanilla setup.
138    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
139    # Now uninstall the vanilla compiler and setup/build our custom
140    # compiler.
141    self._UnInstallToolchain()
142    envdict = {"USE": "git_gcc",
143               "GCC_GITHASH": config.gcc_config.githash,
144               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
145    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
146
147
148class ToolchainComparator(ChromeOSCheckout):
149  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
150    self._board = board
151    self._remotes = remotes
152    self._chromeos_root = "chromeos"
153    self._configs = configs
154    self._clean = clean
155    self._public = public
156    self._force_mismatch = force_mismatch
157    self._ce = command_executer.GetCommandExecuter()
158    self._l = logger.GetLogger()
159    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
160                                           "%Y-%m-%d_%H:%M:%S")
161    self._reports_dir = os.path.join(
162        os.path.expanduser("~/nightly_test_reports"),
163        "%s.%s" % (timestamp, board),
164        )
165    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
166
167
168  def _FinishSetup(self):
169    # Get correct .boto file
170    current_dir = os.getcwd()
171    src = "/home/mobiletc-prebuild/.boto"
172    dest = os.path.join(current_dir, self._chromeos_root,
173                        "src/private-overlays/chromeos-overlay/"
174                        "googlestorage_account.boto")
175    # Copy the file to the correct place
176    copy_cmd = "cp %s %s" % (src, dest)
177    retval = self._ce.RunCommand(copy_cmd)
178    if retval != 0:
179      raise RuntimeError("Couldn't copy .boto file for google storage.")
180
181    # Fix protections on ssh key
182    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
183               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
184               "/testing_rsa")
185    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
186    if retval != 0:
187      raise RuntimeError("chmod for testing_rsa failed")
188
189  def _TestLabels(self, labels):
190    experiment_file = "toolchain_experiment.txt"
191    image_args = ""
192    if self._force_mismatch:
193      image_args = "--force-mismatch"
194    experiment_header = """
195    board: %s
196    remote: %s
197    """ % (self._board, self._remotes)
198    experiment_tests = """
199    benchmark: all_perfv2 {
200      suite: telemetry_Crosperf
201      iterations: 3
202    }
203    """
204    with open(experiment_file, "w") as f:
205      print >>f, experiment_header
206      print >>f, experiment_tests
207      for label in labels:
208        # TODO(asharif): Fix crosperf so it accepts labels with symbols
209        crosperf_label = label
210        crosperf_label = crosperf_label.replace("-", "minus")
211        crosperf_label = crosperf_label.replace("+", "plus")
212        crosperf_label = crosperf_label.replace(".", "")
213
214        # Use the official build instead of building vanilla ourselves.
215        if label == "vanilla":
216          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
217
218          # Now add 'official build' to test file.
219          official_image = """
220          official_image {
221            chromeos_root: %s
222            build: %s
223          }
224          """ % (self._chromeos_root, build_name)
225          print >>f, official_image
226
227        else:
228          experiment_image = """
229          %s {
230            chromeos_image: %s
231            image_args: %s
232          }
233          """ % (crosperf_label,
234                 os.path.join(misc.GetImageDir(self._chromeos_root,
235                                               self._board),
236                              label, "chromiumos_test_image.bin"),
237                 image_args)
238          print >>f, experiment_image
239
240    crosperf = os.path.join(os.path.dirname(__file__),
241                            "crosperf",
242                            "crosperf")
243    command = ("%s --no_email=True --use_file_locks=True --results_dir=%s %s" %
244               (crosperf, self._reports_dir, experiment_file)
245
246    ret = self._ce.RunCommand(command)
247    if ret != 0:
248      raise RuntimeError("Couldn't run crosperf!")
249    return
250
251
252  def _CopyWeeklyReportFiles(self, labels):
253    """Create tar files of the custom and official images and copy them
254    to the weekly reports directory, so they exist when the weekly report
255    gets generated.  IMPORTANT NOTE: This function must run *after*
256    crosperf has been run; otherwise the vanilla images will not be there.
257    """
258    images_path = os.path.join(os.path.realpath(self._chromeos_root),
259                               "src/build/images", self._board)
260    weekday = time.strftime("%a")
261    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
262    dest_dir = os.path.join (data_dir, weekday)
263    if not os.path.exists(dest_dir):
264      os.makedirs(dest_dir)
265    # Make sure dest_dir is empty (clean out last week's data).
266    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
267    self._ce.RunCommand(cmd)
268    # Now create new tar files and copy them over.
269    for l in labels:
270      test_path = os.path.join(images_path, l)
271      if os.path.exists(test_path):
272        if l != "vanilla":
273          label_name = "test"
274        else:
275          label_name = "vanilla"
276        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
277        cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
278               "cp %s %s/.") % (images_path,
279                                tar_file_name,
280                                l, tar_file_name,
281                                dest_dir)
282        tar_ret = self._ce.RunCommand(cmd)
283        if tar_ret != 0:
284          self._l.LogOutput("Error while creating/copying test tar file(%s)."
285                            % tar_file_name)
286
287  def _SendEmail(self):
288    """Find email msesage generated by crosperf and send it."""
289    filename = os.path.join(self._reports_dir, "msg_body.html")
290    if (os.path.exists(filename) and
291        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
292      command = ('cat %s | %s -s "Nightly test results, %s" -team -html'
293                 % (filename, MAIL_PROGRAM, self._board))
294      self._ce.RunCommand(command)
295
296  def DoAll(self):
297    self._CheckoutChromeOS()
298    labels = []
299    labels.append("vanilla")
300    for config in self._configs:
301      label = misc.GetFilenameFromString(config.gcc_config.githash)
302      if (not misc.DoesLabelExist(self._chromeos_root,
303                                  self._board,
304                                  label)):
305        self._BuildToolchain(config)
306        label = self._BuildAndImage(label)
307      labels.append(label)
308    self._FinishSetup()
309    self._TestLabels(labels)
310    self._SendEmail()
311    # Only try to copy the image files if the test runs ran successfully.
312    self._CopyWeeklyReportFiles(labels)
313    if self._clean:
314      ret = self._DeleteChroot()
315      if ret != 0:
316        return ret
317      ret = self._DeleteCcahe()
318      if ret != 0:
319        return ret
320    return 0
321
322
323def Main(argv):
324  """The main function."""
325  # Common initializations
326###  command_executer.InitCommandExecuter(True)
327  command_executer.InitCommandExecuter()
328  parser = optparse.OptionParser()
329  parser.add_option("--remote",
330                    dest="remote",
331                    help="Remote machines to run tests on.")
332  parser.add_option("--board",
333                    dest="board",
334                    default="x86-zgb",
335                    help="The target board.")
336  parser.add_option("--githashes",
337                    dest="githashes",
338                    default="master",
339                    help="The gcc githashes to test.")
340  parser.add_option("--clean",
341                    dest="clean",
342                    default=False,
343                    action="store_true",
344                    help="Clean the chroot after testing.")
345  parser.add_option("--public",
346                    dest="public",
347                    default=False,
348                    action="store_true",
349                    help="Use the public checkout/build.")
350  parser.add_option("--force-mismatch",
351                    dest="force_mismatch",
352                    default="",
353                    help="Force the image regardless of board mismatch")
354  options, _ = parser.parse_args(argv)
355  if not options.board:
356    print "Please give a board."
357    return 1
358  if not options.remote:
359    print "Please give at least one remote machine."
360    return 1
361  toolchain_configs = []
362  for githash in options.githashes.split(","):
363    gcc_config = GCCConfig(githash=githash)
364    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
365    toolchain_configs.append(toolchain_config)
366  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
367                           options.clean, options.public,
368                           options.force_mismatch)
369  return fc.DoAll()
370
371
372if __name__ == "__main__":
373  retval = Main(sys.argv)
374  sys.exit(retval)
375