test_toolchains.py revision 7f3190b3fb89522ea85b87200241962634369e9a 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import optparse
5import os
6import sys
7import build_chromeos
8import setup_chromeos
9import time
10from utils import command_executer
11from utils import misc
12from utils import logger
13
14
15WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
16AFDO_BOARDS = ["butterfly", "lumpy", "stumpy", "stout", "parrot", "parrot_ivb"]
17MAIL_PROGRAM = "~/var/bin/mail-sheriff"
18
19class GCCConfig(object):
20  def __init__(self, githash):
21    self.githash = githash
22
23
24class ToolchainConfig:
25  def __init__(self, gcc_config=None, binutils_config=None):
26    self.gcc_config = gcc_config
27
28
29class ChromeOSCheckout(object):
30  def __init__(self, board, chromeos_root):
31    self._board = board
32    self._chromeos_root = chromeos_root
33    self._ce = command_executer.GetCommandExecuter()
34    self._l = logger.GetLogger()
35    self._build_num = None
36
37  def _DeleteChroot(self):
38    command = "cd %s; cros_sdk --delete" % self._chromeos_root
39    return self._ce.RunCommand(command)
40
41  def _DeleteCcahe(self):
42    # crosbug.com/34956
43    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
44    return self._ce.RunCommand(command)
45
46  def _GetBuildNumber(self):
47    """ This function assumes a ChromeOS image has been built in the chroot.
48    It translates the 'latest' symlink in the
49    <chroot>/src/build/images/<board> directory, to find the actual
50    ChromeOS build number for the image that was built.  For example, if
51    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
52    This function would parse it out and assign 'R37-5982' to self._build_num.
53    This is used to determine the official, vanilla build to use for
54    comparison tests.
55    """
56    # Get the path to 'latest'
57    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
58                                              self._board),
59                             "latest")
60    # Translate the symlink to its 'real' path.
61    real_path = os.path.realpath(sym_path)
62    # Break up the path and get the last piece
63    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
64    path_pieces = real_path.split("/")
65    last_piece = path_pieces[-1]
66    # Break this piece into the image number + other pieces, and get the
67    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
68    image_parts = last_piece.split(".")
69    self._build_num = image_parts[0]
70
71  def _BuildAndImage(self, label=""):
72    if (not label or
73        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
74      build_chromeos_args = [build_chromeos.__file__,
75                             "--chromeos_root=%s" % self._chromeos_root,
76                             "--board=%s" % self._board,
77                             "--rebuild"]
78      if self._public:
79        build_chromeos_args.append("--env=USE=-chrome_internal")
80
81      if self._board in AFDO_BOARDS:
82        build_chromeos_args.append("--env=USE=afdo_use")
83
84      ret = build_chromeos.Main(build_chromeos_args)
85      if ret != 0:
86        raise RuntimeError("Couldn't build ChromeOS!")
87
88      if not  self._build_num:
89        self._GetBuildNumber()
90      # Check to see if we need to create the symbolic link for the vanilla
91      # image, and do so if appropriate.
92      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
93        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
94        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
95                                          'chroot/tmp', build_name)
96        misc.LabelLatestImage(self._chromeos_root, self._board, label,
97                              full_vanilla_path)
98      else:
99        misc.LabelLatestImage(self._chromeos_root, self._board, label)
100    return label
101
102  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
103    env_string = misc.GetEnvStringFromDict(env_dict)
104    command = ("%s %s" %
105               (env_string,
106                misc.GetSetupBoardCommand(self._board,
107                                          usepkg=usepkg_flag,
108                                          force=clobber_flag)))
109    ret = self._ce.ChrootRunCommand(self._chromeos_root,
110                                    command)
111    error_str = "Could not setup board: '%s'" % command
112    assert ret == 0, error_str
113
114  def _UnInstallToolchain(self):
115    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
116               misc.GetCtargetFromBoard(self._board,
117                                   self._chromeos_root))
118    ret = self._ce.ChrootRunCommand(self._chromeos_root,
119                                    command)
120    if ret != 0:
121      raise RuntimeError("Couldn't uninstall the toolchain!")
122
123  def _CheckoutChromeOS(self):
124    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
125    if not os.path.exists(self._chromeos_root):
126      setup_chromeos_args = [setup_chromeos.__file__,
127                             "--dir=%s" % self._chromeos_root]
128      if self._public:
129        setup_chromeos_args.append("--public")
130      ret = setup_chromeos.Main(setup_chromeos_args)
131      if ret != 0:
132        raise RuntimeError("Couldn't run setup_chromeos!")
133
134
135  def _BuildToolchain(self, config):
136    # Call setup_board for basic, vanilla setup.
137    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
138    # Now uninstall the vanilla compiler and setup/build our custom
139    # compiler.
140    self._UnInstallToolchain()
141    envdict = {"USE": "git_gcc",
142               "GCC_GITHASH": config.gcc_config.githash,
143               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
144    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
145
146
147class ToolchainComparator(ChromeOSCheckout):
148  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
149    self._board = board
150    self._remotes = remotes
151    self._chromeos_root = "chromeos"
152    self._configs = configs
153    self._clean = clean
154    self._public = public
155    self._force_mismatch = force_mismatch
156    self._ce = command_executer.GetCommandExecuter()
157    self._l = logger.GetLogger()
158    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
159                                           "%Y-%m-%d_%H:%M:%S")
160    self._reports_dir = os.path.join(
161        os.path.expanduser("~/nightly_test_reports"),
162        "%s.%s" % (timestamp, board),
163        )
164    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
165
166
167  def _FinishSetup(self):
168    # Get correct .boto file
169    current_dir = os.getcwd()
170    src = "/home/mobiletc-prebuild/.boto"
171    dest = os.path.join(current_dir, self._chromeos_root,
172                        "src/private-overlays/chromeos-overlay/"
173                        "googlestorage_account.boto")
174    # Copy the file to the correct place
175    copy_cmd = "cp %s %s" % (src, dest)
176    retval = self._ce.RunCommand(copy_cmd)
177    if retval != 0:
178      raise RuntimeError("Couldn't copy .boto file for google storage.")
179
180    # Fix protections on ssh key
181    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
182               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
183               "/testing_rsa")
184    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
185    if retval != 0:
186      raise RuntimeError("chmod for testing_rsa failed")
187
188  def _TestLabels(self, labels):
189    experiment_file = "toolchain_experiment.txt"
190    image_args = ""
191    if self._force_mismatch:
192      image_args = "--force-mismatch"
193    experiment_header = """
194    board: %s
195    remote: %s
196    """ % (self._board, self._remotes)
197    experiment_tests = """
198    benchmark: all_perfv2 {
199      suite: telemetry_Crosperf
200      iterations: 3
201    }
202    """
203    with open(experiment_file, "w") as f:
204      print >>f, experiment_header
205      print >>f, experiment_tests
206      for label in labels:
207        # TODO(asharif): Fix crosperf so it accepts labels with symbols
208        crosperf_label = label
209        crosperf_label = crosperf_label.replace("-", "minus")
210        crosperf_label = crosperf_label.replace("+", "plus")
211        crosperf_label = crosperf_label.replace(".", "")
212
213        # Use the official build instead of building vanilla ourselves.
214        if label == "vanilla":
215          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
216
217          # Now add 'official build' to test file.
218          official_image = """
219          official_image {
220            chromeos_root: %s
221            build: %s
222          }
223          """ % (self._chromeos_root, build_name)
224          print >>f, official_image
225
226        else:
227          experiment_image = """
228          %s {
229            chromeos_image: %s
230            image_args: %s
231          }
232          """ % (crosperf_label,
233                 os.path.join(misc.GetImageDir(self._chromeos_root,
234                                               self._board),
235                              label, "chromiumos_test_image.bin"),
236                 image_args)
237          print >>f, experiment_image
238
239    crosperf = os.path.join(os.path.dirname(__file__),
240                            "crosperf",
241                            "crosperf")
242    command = "%s --no_email=True --results_dir=%s %s" % (crosperf, self._reports_dir,
243                                                          experiment_file)
244    ret = self._ce.RunCommand(command)
245    if ret != 0:
246      raise RuntimeError("Couldn't run crosperf!")
247    return
248
249
250  def _CopyWeeklyReportFiles(self, labels):
251    """Create tar files of the custom and official images and copy them
252    to the weekly reports directory, so they exist when the weekly report
253    gets generated.  IMPORTANT NOTE: This function must run *after*
254    crosperf has been run; otherwise the vanilla images will not be there.
255    """
256    images_path = os.path.join(os.path.realpath(self._chromeos_root),
257                               "src/build/images", self._board)
258    weekday = time.strftime("%a")
259    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
260    dest_dir = os.path.join (data_dir, weekday)
261    if not os.path.exists(dest_dir):
262      os.makedirs(dest_dir)
263    # Make sure dest_dir is empty (clean out last week's data).
264    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
265    self._ce.RunCommand(cmd)
266    # Now create new tar files and copy them over.
267    for l in labels:
268      test_path = os.path.join(images_path, l)
269      if os.path.exists(test_path):
270        if l != "vanilla":
271          label_name = "test"
272        else:
273          label_name = "vanilla"
274        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
275        cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
276               "cp %s %s/.") % (images_path,
277                                tar_file_name,
278                                l, tar_file_name,
279                                dest_dir)
280        tar_ret = self._ce.RunCommand(cmd)
281        if tar_ret != 0:
282          self._l.LogOutput("Error while creating/copying test tar file(%s)."
283                            % tar_file_name)
284
285  def _SendEmail(self):
286    """Find email msesage generated by crosperf and send it."""
287    filename = os.path.join(self._reports_dir, "msg_body.html")
288    if (os.path.exists(filename) and
289        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
290      commmand = ('cat %s | %s -s "Nightly test results, %s" -team -html'
291                  % (filename, MAIL_PROGRAM, self._board))
292      self._ce.RunCommand(command)
293
294  def DoAll(self):
295    self._CheckoutChromeOS()
296    labels = []
297    labels.append("vanilla")
298    for config in self._configs:
299      label = misc.GetFilenameFromString(config.gcc_config.githash)
300      if (not misc.DoesLabelExist(self._chromeos_root,
301                                  self._board,
302                                  label)):
303        self._BuildToolchain(config)
304        label = self._BuildAndImage(label)
305      labels.append(label)
306    self._FinishSetup()
307    self._TestLabels(labels):
308    self._SendEmail()
309    # Only try to copy the image files if the test runs ran successfully.
310    self._CopyWeeklyReportFiles(labels)
311    if self._clean:
312      ret = self._DeleteChroot()
313      if ret != 0:
314        return ret
315      ret = self._DeleteCcahe()
316      if ret != 0:
317        return ret
318    return 0
319
320
321def Main(argv):
322  """The main function."""
323  # Common initializations
324###  command_executer.InitCommandExecuter(True)
325  command_executer.InitCommandExecuter()
326  parser = optparse.OptionParser()
327  parser.add_option("--remote",
328                    dest="remote",
329                    help="Remote machines to run tests on.")
330  parser.add_option("--board",
331                    dest="board",
332                    default="x86-zgb",
333                    help="The target board.")
334  parser.add_option("--githashes",
335                    dest="githashes",
336                    default="master",
337                    help="The gcc githashes to test.")
338  parser.add_option("--clean",
339                    dest="clean",
340                    default=False,
341                    action="store_true",
342                    help="Clean the chroot after testing.")
343  parser.add_option("--public",
344                    dest="public",
345                    default=False,
346                    action="store_true",
347                    help="Use the public checkout/build.")
348  parser.add_option("--force-mismatch",
349                    dest="force_mismatch",
350                    default="",
351                    help="Force the image regardless of board mismatch")
352  options, _ = parser.parse_args(argv)
353  if not options.board:
354    print "Please give a board."
355    return 1
356  if not options.remote:
357    print "Please give at least one remote machine."
358    return 1
359  toolchain_configs = []
360  for githash in options.githashes.split(","):
361    gcc_config = GCCConfig(githash=githash)
362    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
363    toolchain_configs.append(toolchain_config)
364  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
365                           options.clean, options.public,
366                           options.force_mismatch)
367  return fc.DoAll()
368
369
370if __name__ == "__main__":
371  retval = Main(sys.argv)
372  sys.exit(retval)
373