test_toolchains.py revision 364131220999a99b408c5ef848da17844028c604 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import datetime
5import optparse
6import os
7import sys
8import build_chromeos
9import setup_chromeos
10import time
11from utils import command_executer
12from utils import misc
13from utils import logger
14
15
16WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
17AFDO_BOARDS = ["butterfly", "lumpy", "stumpy", "stout", "parrot", "parrot_ivb"]
18MAIL_PROGRAM = "~/var/bin/mail-sheriff"
19
20class GCCConfig(object):
21  def __init__(self, githash):
22    self.githash = githash
23
24
25class ToolchainConfig:
26  def __init__(self, gcc_config=None, binutils_config=None):
27    self.gcc_config = gcc_config
28
29
30class ChromeOSCheckout(object):
31  def __init__(self, board, chromeos_root):
32    self._board = board
33    self._chromeos_root = chromeos_root
34    self._ce = command_executer.GetCommandExecuter()
35    self._l = logger.GetLogger()
36    self._build_num = None
37
38  def _DeleteChroot(self):
39    command = "cd %s; cros_sdk --delete" % self._chromeos_root
40    return self._ce.RunCommand(command)
41
42  def _DeleteCcahe(self):
43    # crosbug.com/34956
44    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
45    return self._ce.RunCommand(command)
46
47  def _GetBuildNumber(self):
48    """ This function assumes a ChromeOS image has been built in the chroot.
49    It translates the 'latest' symlink in the
50    <chroot>/src/build/images/<board> directory, to find the actual
51    ChromeOS build number for the image that was built.  For example, if
52    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
53    This function would parse it out and assign 'R37-5982' to self._build_num.
54    This is used to determine the official, vanilla build to use for
55    comparison tests.
56    """
57    # Get the path to 'latest'
58    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
59                                              self._board),
60                             "latest")
61    # Translate the symlink to its 'real' path.
62    real_path = os.path.realpath(sym_path)
63    # Break up the path and get the last piece
64    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
65    path_pieces = real_path.split("/")
66    last_piece = path_pieces[-1]
67    # Break this piece into the image number + other pieces, and get the
68    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
69    image_parts = last_piece.split(".")
70    self._build_num = image_parts[0]
71
72  def _BuildAndImage(self, label=""):
73    if (not label or
74        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
75      build_chromeos_args = [build_chromeos.__file__,
76                             "--chromeos_root=%s" % self._chromeos_root,
77                             "--board=%s" % self._board,
78                             "--rebuild"]
79      if self._public:
80        build_chromeos_args.append("--env=USE=-chrome_internal")
81
82      if self._board in AFDO_BOARDS:
83        build_chromeos_args.append("--env=USE=afdo_use")
84
85      ret = build_chromeos.Main(build_chromeos_args)
86      if ret != 0:
87        raise RuntimeError("Couldn't build ChromeOS!")
88
89      if not  self._build_num:
90        self._GetBuildNumber()
91      # Check to see if we need to create the symbolic link for the vanilla
92      # image, and do so if appropriate.
93      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
94        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
95        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
96                                          'chroot/tmp', build_name)
97        misc.LabelLatestImage(self._chromeos_root, self._board, label,
98                              full_vanilla_path)
99      else:
100        misc.LabelLatestImage(self._chromeos_root, self._board, label)
101    return label
102
103  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
104    env_string = misc.GetEnvStringFromDict(env_dict)
105    command = ("%s %s" %
106               (env_string,
107                misc.GetSetupBoardCommand(self._board,
108                                          usepkg=usepkg_flag,
109                                          force=clobber_flag)))
110    ret = self._ce.ChrootRunCommand(self._chromeos_root,
111                                    command)
112    error_str = "Could not setup board: '%s'" % command
113    assert ret == 0, error_str
114
115  def _UnInstallToolchain(self):
116    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
117               misc.GetCtargetFromBoard(self._board,
118                                   self._chromeos_root))
119    ret = self._ce.ChrootRunCommand(self._chromeos_root,
120                                    command)
121    if ret != 0:
122      raise RuntimeError("Couldn't uninstall the toolchain!")
123
124  def _CheckoutChromeOS(self):
125    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
126    if not os.path.exists(self._chromeos_root):
127      setup_chromeos_args = [setup_chromeos.__file__,
128                             "--dir=%s" % self._chromeos_root]
129      if self._public:
130        setup_chromeos_args.append("--public")
131      ret = setup_chromeos.Main(setup_chromeos_args)
132      if ret != 0:
133        raise RuntimeError("Couldn't run setup_chromeos!")
134
135
136  def _BuildToolchain(self, config):
137    # Call setup_board for basic, vanilla setup.
138    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
139    # Now uninstall the vanilla compiler and setup/build our custom
140    # compiler.
141    self._UnInstallToolchain()
142    envdict = {"USE": "git_gcc",
143               "GCC_GITHASH": config.gcc_config.githash,
144               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
145    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
146
147
148class ToolchainComparator(ChromeOSCheckout):
149  def __init__(self, board, remotes, configs, clean,
150               public, force_mismatch, schedv2=False):
151    self._board = board
152    self._remotes = remotes
153    self._chromeos_root = "chromeos"
154    self._configs = configs
155    self._clean = clean
156    self._public = public
157    self._force_mismatch = force_mismatch
158    self._ce = command_executer.GetCommandExecuter()
159    self._l = logger.GetLogger()
160    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
161                                           "%Y-%m-%d_%H:%M:%S")
162    self._reports_dir = os.path.join(
163        os.path.expanduser("~/nightly_test_reports"),
164        "%s.%s" % (timestamp, board),
165        )
166    self._schedv2 = schedv2
167    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
168
169
170  def _FinishSetup(self):
171    # Get correct .boto file
172    current_dir = os.getcwd()
173    src = "/home/mobiletc-prebuild/.boto"
174    dest = os.path.join(current_dir, self._chromeos_root,
175                        "src/private-overlays/chromeos-overlay/"
176                        "googlestorage_account.boto")
177    # Copy the file to the correct place
178    copy_cmd = "cp %s %s" % (src, dest)
179    retval = self._ce.RunCommand(copy_cmd)
180    if retval != 0:
181      raise RuntimeError("Couldn't copy .boto file for google storage.")
182
183    # Fix protections on ssh key
184    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
185               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
186               "/testing_rsa")
187    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
188    if retval != 0:
189      raise RuntimeError("chmod for testing_rsa failed")
190
191  def _TestLabels(self, labels):
192    experiment_file = "toolchain_experiment.txt"
193    image_args = ""
194    if self._force_mismatch:
195      image_args = "--force-mismatch"
196    experiment_header = """
197    board: %s
198    remote: %s
199    retries: 1
200    """ % (self._board, self._remotes)
201    experiment_tests = """
202    benchmark: all_toolchain_perf {
203      suite: telemetry_Crosperf
204      iterations: 3
205    }
206    """
207    with open(experiment_file, "w") as f:
208      print >>f, experiment_header
209      print >>f, experiment_tests
210      for label in labels:
211        # TODO(asharif): Fix crosperf so it accepts labels with symbols
212        crosperf_label = label
213        crosperf_label = crosperf_label.replace("-", "minus")
214        crosperf_label = crosperf_label.replace("+", "plus")
215        crosperf_label = crosperf_label.replace(".", "")
216
217        # Use the official build instead of building vanilla ourselves.
218        if label == "vanilla":
219          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
220
221          # Now add 'official build' to test file.
222          official_image = """
223          official_image {
224            chromeos_root: %s
225            build: %s
226          }
227          """ % (self._chromeos_root, build_name)
228          print >>f, official_image
229
230        else:
231          experiment_image = """
232          %s {
233            chromeos_image: %s
234            image_args: %s
235          }
236          """ % (crosperf_label,
237                 os.path.join(misc.GetImageDir(self._chromeos_root,
238                                               self._board),
239                              label, "chromiumos_test_image.bin"),
240                 image_args)
241          print >>f, experiment_image
242
243    crosperf = os.path.join(os.path.dirname(__file__),
244                            "crosperf",
245                            "crosperf")
246    schedv2_opts = '--schedv2 --logging_level=verbose' if self._schedv2 else ''
247    command = ("{crosperf} --no_email=True --results_dir={r_dir} "
248               "{schedv2_opts} {exp_file}").format(
249                crosperf=crosperf,
250                r_dir=self._reports_dir,
251                schedv2_opts=schedv2_opts,
252                exp_file=experiment_file)
253
254    ret = self._ce.RunCommand(command)
255    if ret != 0:
256      raise RuntimeError("Couldn't run crosperf!")
257    return
258
259
260  def _CopyWeeklyReportFiles(self, labels):
261    """Create tar files of the custom and official images and copy them
262    to the weekly reports directory, so they exist when the weekly report
263    gets generated.  IMPORTANT NOTE: This function must run *after*
264    crosperf has been run; otherwise the vanilla images will not be there.
265    """
266    images_path = os.path.join(os.path.realpath(self._chromeos_root),
267                               "src/build/images", self._board)
268    weekday = time.strftime("%a")
269    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
270    dest_dir = os.path.join (data_dir, weekday)
271    if not os.path.exists(dest_dir):
272      os.makedirs(dest_dir)
273    # Make sure dest_dir is empty (clean out last week's data).
274    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
275    self._ce.RunCommand(cmd)
276    # Now create new tar files and copy them over.
277    for l in labels:
278      test_path = os.path.join(images_path, l)
279      if os.path.exists(test_path):
280        if l != "vanilla":
281          label_name = "test"
282        else:
283          label_name = "vanilla"
284        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
285        cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
286               "cp %s %s/.") % (images_path,
287                                tar_file_name,
288                                l, tar_file_name,
289                                dest_dir)
290        tar_ret = self._ce.RunCommand(cmd)
291        if tar_ret != 0:
292          self._l.LogOutput("Error while creating/copying test tar file(%s)."
293                            % tar_file_name)
294
295  def _SendEmail(self):
296    """Find email msesage generated by crosperf and send it."""
297    filename = os.path.join(self._reports_dir, "msg_body.html")
298    if (os.path.exists(filename) and
299        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
300      command = ('cat %s | %s -s "Nightly test results, %s" -team -html'
301                 % (filename, MAIL_PROGRAM, self._board))
302      self._ce.RunCommand(command)
303
304  def DoAll(self):
305    self._CheckoutChromeOS()
306    labels = []
307    labels.append("vanilla")
308    for config in self._configs:
309      label = misc.GetFilenameFromString(config.gcc_config.githash)
310      if (not misc.DoesLabelExist(self._chromeos_root,
311                                  self._board,
312                                  label)):
313        self._BuildToolchain(config)
314        label = self._BuildAndImage(label)
315      labels.append(label)
316    self._FinishSetup()
317    self._TestLabels(labels)
318    self._SendEmail()
319    # Only try to copy the image files if the test runs ran successfully.
320    self._CopyWeeklyReportFiles(labels)
321    if self._clean:
322      ret = self._DeleteChroot()
323      if ret != 0:
324        return ret
325      ret = self._DeleteCcahe()
326      if ret != 0:
327        return ret
328    return 0
329
330
331def Main(argv):
332  """The main function."""
333  # Common initializations
334###  command_executer.InitCommandExecuter(True)
335  command_executer.InitCommandExecuter()
336  parser = optparse.OptionParser()
337  parser.add_option("--remote",
338                    dest="remote",
339                    help="Remote machines to run tests on.")
340  parser.add_option("--board",
341                    dest="board",
342                    default="x86-zgb",
343                    help="The target board.")
344  parser.add_option("--githashes",
345                    dest="githashes",
346                    default="master",
347                    help="The gcc githashes to test.")
348  parser.add_option("--clean",
349                    dest="clean",
350                    default=False,
351                    action="store_true",
352                    help="Clean the chroot after testing.")
353  parser.add_option("--public",
354                    dest="public",
355                    default=False,
356                    action="store_true",
357                    help="Use the public checkout/build.")
358  parser.add_option("--force-mismatch",
359                    dest="force_mismatch",
360                    default="",
361                    help="Force the image regardless of board mismatch")
362  parser.add_option("--schedv2",
363                    dest="schedv2",
364                    action="store_true",
365                    default=False,
366                    help="Pass --schedv2 to crosperf.")
367  options, _ = parser.parse_args(argv)
368  if not options.board:
369    print "Please give a board."
370    return 1
371  if not options.remote:
372    print "Please give at least one remote machine."
373    return 1
374  toolchain_configs = []
375  for githash in options.githashes.split(","):
376    gcc_config = GCCConfig(githash=githash)
377    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
378    toolchain_configs.append(toolchain_config)
379  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
380                           options.clean, options.public,
381                           options.force_mismatch,
382                           options.schedv2)
383  return fc.DoAll()
384
385
386if __name__ == "__main__":
387  retval = Main(sys.argv)
388  sys.exit(retval)
389