test_toolchains.py revision d1f03b83a267caa1e92d8d04c483c5ec0a8c5355 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import datetime
5import optparse
6import os
7import sys
8import build_chromeos
9import setup_chromeos
10import time
11from utils import command_executer
12from utils import misc
13from utils import logger
14
15
16WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
17AFDO_BOARDS = ["butterfly", "lumpy", "stumpy", "stout", "parrot", "parrot_ivb"]
18MAIL_PROGRAM = "~/var/bin/mail-sheriff"
19
20class GCCConfig(object):
21  def __init__(self, githash):
22    self.githash = githash
23
24
25class ToolchainConfig:
26  def __init__(self, gcc_config=None, binutils_config=None):
27    self.gcc_config = gcc_config
28
29
30class ChromeOSCheckout(object):
31  def __init__(self, board, chromeos_root):
32    self._board = board
33    self._chromeos_root = chromeos_root
34    self._ce = command_executer.GetCommandExecuter()
35    self._l = logger.GetLogger()
36    self._build_num = None
37
38  def _DeleteChroot(self):
39    command = "cd %s; cros_sdk --delete" % self._chromeos_root
40    return self._ce.RunCommand(command)
41
42  def _DeleteCcahe(self):
43    # crosbug.com/34956
44    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
45    return self._ce.RunCommand(command)
46
47  def _GetBuildNumber(self):
48    """ This function assumes a ChromeOS image has been built in the chroot.
49    It translates the 'latest' symlink in the
50    <chroot>/src/build/images/<board> directory, to find the actual
51    ChromeOS build number for the image that was built.  For example, if
52    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
53    This function would parse it out and assign 'R37-5982' to self._build_num.
54    This is used to determine the official, vanilla build to use for
55    comparison tests.
56    """
57    # Get the path to 'latest'
58    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
59                                              self._board),
60                             "latest")
61    # Translate the symlink to its 'real' path.
62    real_path = os.path.realpath(sym_path)
63    # Break up the path and get the last piece
64    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
65    path_pieces = real_path.split("/")
66    last_piece = path_pieces[-1]
67    # Break this piece into the image number + other pieces, and get the
68    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
69    image_parts = last_piece.split(".")
70    self._build_num = image_parts[0]
71
72  def _BuildAndImage(self, label=""):
73    if (not label or
74        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
75      build_chromeos_args = [build_chromeos.__file__,
76                             "--chromeos_root=%s" % self._chromeos_root,
77                             "--board=%s" % self._board,
78                             "--rebuild"]
79      if self._public:
80        build_chromeos_args.append("--env=USE=-chrome_internal")
81
82      if self._board in AFDO_BOARDS:
83        build_chromeos_args.append("--env=USE=afdo_use")
84
85      ret = build_chromeos.Main(build_chromeos_args)
86      if ret != 0:
87        raise RuntimeError("Couldn't build ChromeOS!")
88
89      if not  self._build_num:
90        self._GetBuildNumber()
91      # Check to see if we need to create the symbolic link for the vanilla
92      # image, and do so if appropriate.
93      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
94        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
95        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
96                                          'chroot/tmp', build_name)
97        misc.LabelLatestImage(self._chromeos_root, self._board, label,
98                              full_vanilla_path)
99      else:
100        misc.LabelLatestImage(self._chromeos_root, self._board, label)
101    return label
102
103  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
104    env_string = misc.GetEnvStringFromDict(env_dict)
105    command = ("%s %s" %
106               (env_string,
107                misc.GetSetupBoardCommand(self._board,
108                                          usepkg=usepkg_flag,
109                                          force=clobber_flag)))
110    ret = self._ce.ChrootRunCommand(self._chromeos_root,
111                                    command)
112    error_str = "Could not setup board: '%s'" % command
113    assert ret == 0, error_str
114
115  def _UnInstallToolchain(self):
116    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
117               misc.GetCtargetFromBoard(self._board,
118                                   self._chromeos_root))
119    ret = self._ce.ChrootRunCommand(self._chromeos_root,
120                                    command)
121    if ret != 0:
122      raise RuntimeError("Couldn't uninstall the toolchain!")
123
124  def _CheckoutChromeOS(self):
125    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
126    if not os.path.exists(self._chromeos_root):
127      setup_chromeos_args = [setup_chromeos.__file__,
128                             "--dir=%s" % self._chromeos_root]
129      if self._public:
130        setup_chromeos_args.append("--public")
131      ret = setup_chromeos.Main(setup_chromeos_args)
132      if ret != 0:
133        raise RuntimeError("Couldn't run setup_chromeos!")
134
135
136  def _BuildToolchain(self, config):
137    # Call setup_board for basic, vanilla setup.
138    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
139    # Now uninstall the vanilla compiler and setup/build our custom
140    # compiler.
141    self._UnInstallToolchain()
142    envdict = {"USE": "git_gcc",
143               "GCC_GITHASH": config.gcc_config.githash,
144               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
145    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
146
147
148class ToolchainComparator(ChromeOSCheckout):
149  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
150    self._board = board
151    self._remotes = remotes
152    self._chromeos_root = "chromeos"
153    self._configs = configs
154    self._clean = clean
155    self._public = public
156    self._force_mismatch = force_mismatch
157    self._ce = command_executer.GetCommandExecuter()
158    self._l = logger.GetLogger()
159    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
160                                           "%Y-%m-%d_%H:%M:%S")
161    self._reports_dir = os.path.join(
162        os.path.expanduser("~/nightly_test_reports"),
163        "%s.%s" % (timestamp, board),
164        )
165    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
166
167
168  def _FinishSetup(self):
169    # Get correct .boto file
170    current_dir = os.getcwd()
171    src = "/home/mobiletc-prebuild/.boto"
172    dest = os.path.join(current_dir, self._chromeos_root,
173                        "src/private-overlays/chromeos-overlay/"
174                        "googlestorage_account.boto")
175    # Copy the file to the correct place
176    copy_cmd = "cp %s %s" % (src, dest)
177    retval = self._ce.RunCommand(copy_cmd)
178    if retval != 0:
179      raise RuntimeError("Couldn't copy .boto file for google storage.")
180
181    # Fix protections on ssh key
182    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
183               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
184               "/testing_rsa")
185    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
186    if retval != 0:
187      raise RuntimeError("chmod for testing_rsa failed")
188
189  def _TestLabels(self, labels):
190    experiment_file = "toolchain_experiment.txt"
191    image_args = ""
192    if self._force_mismatch:
193      image_args = "--force-mismatch"
194    experiment_header = """
195    board: %s
196    remote: %s
197    retries: 1
198    """ % (self._board, self._remotes)
199    experiment_tests = """
200    benchmark: all_toolchain_perf {
201      suite: telemetry_Crosperf
202      iterations: 3
203    }
204    """
205    with open(experiment_file, "w") as f:
206      print >>f, experiment_header
207      print >>f, experiment_tests
208      for label in labels:
209        # TODO(asharif): Fix crosperf so it accepts labels with symbols
210        crosperf_label = label
211        crosperf_label = crosperf_label.replace("-", "minus")
212        crosperf_label = crosperf_label.replace("+", "plus")
213        crosperf_label = crosperf_label.replace(".", "")
214
215        # Use the official build instead of building vanilla ourselves.
216        if label == "vanilla":
217          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
218
219          # Now add 'official build' to test file.
220          official_image = """
221          official_image {
222            chromeos_root: %s
223            build: %s
224          }
225          """ % (self._chromeos_root, build_name)
226          print >>f, official_image
227
228        else:
229          experiment_image = """
230          %s {
231            chromeos_image: %s
232            image_args: %s
233          }
234          """ % (crosperf_label,
235                 os.path.join(misc.GetImageDir(self._chromeos_root,
236                                               self._board),
237                              label, "chromiumos_test_image.bin"),
238                 image_args)
239          print >>f, experiment_image
240
241    crosperf = os.path.join(os.path.dirname(__file__),
242                            "crosperf",
243                            "crosperf")
244    command = ("%s --no_email=True --results_dir=%s %s" %
245               (crosperf, self._reports_dir, experiment_file))
246
247    ret = self._ce.RunCommand(command)
248    if ret != 0:
249      raise RuntimeError("Couldn't run crosperf!")
250    return
251
252
253  def _CopyWeeklyReportFiles(self, labels):
254    """Create tar files of the custom and official images and copy them
255    to the weekly reports directory, so they exist when the weekly report
256    gets generated.  IMPORTANT NOTE: This function must run *after*
257    crosperf has been run; otherwise the vanilla images will not be there.
258    """
259    images_path = os.path.join(os.path.realpath(self._chromeos_root),
260                               "src/build/images", self._board)
261    weekday = time.strftime("%a")
262    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
263    dest_dir = os.path.join (data_dir, weekday)
264    if not os.path.exists(dest_dir):
265      os.makedirs(dest_dir)
266    # Make sure dest_dir is empty (clean out last week's data).
267    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
268    self._ce.RunCommand(cmd)
269    # Now create new tar files and copy them over.
270    for l in labels:
271      test_path = os.path.join(images_path, l)
272      if os.path.exists(test_path):
273        if l != "vanilla":
274          label_name = "test"
275        else:
276          label_name = "vanilla"
277        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
278        cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
279               "cp %s %s/.") % (images_path,
280                                tar_file_name,
281                                l, tar_file_name,
282                                dest_dir)
283        tar_ret = self._ce.RunCommand(cmd)
284        if tar_ret != 0:
285          self._l.LogOutput("Error while creating/copying test tar file(%s)."
286                            % tar_file_name)
287
288  def _SendEmail(self):
289    """Find email msesage generated by crosperf and send it."""
290    filename = os.path.join(self._reports_dir, "msg_body.html")
291    if (os.path.exists(filename) and
292        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
293      command = ('cat %s | %s -s "Nightly test results, %s" -team -html'
294                 % (filename, MAIL_PROGRAM, self._board))
295      self._ce.RunCommand(command)
296
297  def DoAll(self):
298    self._CheckoutChromeOS()
299    labels = []
300    labels.append("vanilla")
301    for config in self._configs:
302      label = misc.GetFilenameFromString(config.gcc_config.githash)
303      if (not misc.DoesLabelExist(self._chromeos_root,
304                                  self._board,
305                                  label)):
306        self._BuildToolchain(config)
307        label = self._BuildAndImage(label)
308      labels.append(label)
309    self._FinishSetup()
310    self._TestLabels(labels)
311    self._SendEmail()
312    # Only try to copy the image files if the test runs ran successfully.
313    self._CopyWeeklyReportFiles(labels)
314    if self._clean:
315      ret = self._DeleteChroot()
316      if ret != 0:
317        return ret
318      ret = self._DeleteCcahe()
319      if ret != 0:
320        return ret
321    return 0
322
323
324def Main(argv):
325  """The main function."""
326  # Common initializations
327###  command_executer.InitCommandExecuter(True)
328  command_executer.InitCommandExecuter()
329  parser = optparse.OptionParser()
330  parser.add_option("--remote",
331                    dest="remote",
332                    help="Remote machines to run tests on.")
333  parser.add_option("--board",
334                    dest="board",
335                    default="x86-zgb",
336                    help="The target board.")
337  parser.add_option("--githashes",
338                    dest="githashes",
339                    default="master",
340                    help="The gcc githashes to test.")
341  parser.add_option("--clean",
342                    dest="clean",
343                    default=False,
344                    action="store_true",
345                    help="Clean the chroot after testing.")
346  parser.add_option("--public",
347                    dest="public",
348                    default=False,
349                    action="store_true",
350                    help="Use the public checkout/build.")
351  parser.add_option("--force-mismatch",
352                    dest="force_mismatch",
353                    default="",
354                    help="Force the image regardless of board mismatch")
355  options, _ = parser.parse_args(argv)
356  if not options.board:
357    print "Please give a board."
358    return 1
359  if not options.remote:
360    print "Please give at least one remote machine."
361    return 1
362  toolchain_configs = []
363  for githash in options.githashes.split(","):
364    gcc_config = GCCConfig(githash=githash)
365    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
366    toolchain_configs.append(toolchain_config)
367  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
368                           options.clean, options.public,
369                           options.force_mismatch)
370  return fc.DoAll()
371
372
373if __name__ == "__main__":
374  retval = Main(sys.argv)
375  sys.exit(retval)
376