test_toolchains.py revision 80eab98c8820e503118ba0962cf9f0f74a51e28d 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import datetime
5import optparse
6import os
7import string
8import sys
9import build_chromeos
10import setup_chromeos
11import time
12from utils import command_executer
13from utils import misc
14from utils import logger
15
16
17CROSTC_ROOT = "/usr/local/google/crostc"
18MAIL_PROGRAM = "~/var/bin/mail-sheriff"
19WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data")
20PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives")
21NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports")
22
23class GCCConfig(object):
24  def __init__(self, githash):
25    self.githash = githash
26
27
28class ToolchainConfig:
29  def __init__(self, gcc_config=None, binutils_config=None):
30    self.gcc_config = gcc_config
31
32
33class ChromeOSCheckout(object):
34  def __init__(self, board, chromeos_root):
35    self._board = board
36    self._chromeos_root = chromeos_root
37    self._ce = command_executer.GetCommandExecuter()
38    self._l = logger.GetLogger()
39    self._build_num = None
40
41  def _DeleteChroot(self):
42    command = "cd %s; cros_sdk --delete" % self._chromeos_root
43    return self._ce.RunCommand(command)
44
45  def _DeleteCcahe(self):
46    # crosbug.com/34956
47    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
48    return self._ce.RunCommand(command)
49
50  def _GetBuildNumber(self):
51    """ This function assumes a ChromeOS image has been built in the chroot.
52    It translates the 'latest' symlink in the
53    <chroot>/src/build/images/<board> directory, to find the actual
54    ChromeOS build number for the image that was built.  For example, if
55    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
56    This function would parse it out and assign 'R37-5982' to self._build_num.
57    This is used to determine the official, vanilla build to use for
58    comparison tests.
59    """
60    # Get the path to 'latest'
61    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
62                                              self._board),
63                             "latest")
64    # Translate the symlink to its 'real' path.
65    real_path = os.path.realpath(sym_path)
66    # Break up the path and get the last piece
67    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
68    path_pieces = real_path.split("/")
69    last_piece = path_pieces[-1]
70    # Break this piece into the image number + other pieces, and get the
71    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
72    image_parts = last_piece.split(".")
73    self._build_num = image_parts[0]
74
75  def _BuildLabelName(self, config, board):
76    pieces = config.split("/")
77    compiler_version = pieces[-1]
78    label = compiler_version +  "_tot_afdo"
79    return label
80
81  def _BuildAndImage(self, label=""):
82    if (not label or
83        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
84      build_chromeos_args = [build_chromeos.__file__,
85                             "--chromeos_root=%s" % self._chromeos_root,
86                             "--board=%s" % self._board,
87                             "--rebuild"]
88      if self._public:
89        build_chromeos_args.append("--env=USE=-chrome_internal")
90
91      ret = build_chromeos.Main(build_chromeos_args)
92      if ret != 0:
93        raise RuntimeError("Couldn't build ChromeOS!")
94
95      if not  self._build_num:
96        self._GetBuildNumber()
97      # Check to see if we need to create the symbolic link for the vanilla
98      # image, and do so if appropriate.
99      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
100        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
101        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
102                                          'chroot/tmp', build_name)
103        misc.LabelLatestImage(self._chromeos_root, self._board, label,
104                              full_vanilla_path)
105      else:
106        misc.LabelLatestImage(self._chromeos_root, self._board, label)
107    return label
108
109  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
110    env_string = misc.GetEnvStringFromDict(env_dict)
111    command = ("%s %s" %
112               (env_string,
113                misc.GetSetupBoardCommand(self._board,
114                                          usepkg=usepkg_flag,
115                                          force=clobber_flag)))
116    ret = self._ce.ChrootRunCommand(self._chromeos_root,
117                                    command)
118    error_str = "Could not setup board: '%s'" % command
119    assert ret == 0, error_str
120
121  def _UnInstallToolchain(self):
122    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
123               misc.GetCtargetFromBoard(self._board,
124                                   self._chromeos_root))
125    ret = self._ce.ChrootRunCommand(self._chromeos_root,
126                                    command)
127    if ret != 0:
128      raise RuntimeError("Couldn't uninstall the toolchain!")
129
130  def _CheckoutChromeOS(self):
131    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
132    if not os.path.exists(self._chromeos_root):
133      setup_chromeos_args = [setup_chromeos.__file__,
134                             "--dir=%s" % self._chromeos_root]
135      if self._public:
136        setup_chromeos_args.append("--public")
137      ret = setup_chromeos.Main(setup_chromeos_args)
138      if ret != 0:
139        raise RuntimeError("Couldn't run setup_chromeos!")
140
141
142  def _BuildToolchain(self, config):
143    # Call setup_board for basic, vanilla setup.
144    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
145    # Now uninstall the vanilla compiler and setup/build our custom
146    # compiler.
147    self._UnInstallToolchain()
148    envdict = {"USE": "git_gcc",
149               "GCC_GITHASH": config.gcc_config.githash,
150               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
151    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
152
153
154class ToolchainComparator(ChromeOSCheckout):
155  def __init__(self, board, remotes, configs, clean,
156               public, force_mismatch, noschedv2=False):
157    self._board = board
158    self._remotes = remotes
159    self._chromeos_root = "chromeos"
160    self._configs = configs
161    self._clean = clean
162    self._public = public
163    self._force_mismatch = force_mismatch
164    self._ce = command_executer.GetCommandExecuter()
165    self._l = logger.GetLogger()
166    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
167                                           "%Y-%m-%d_%H:%M:%S")
168    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
169        "%s.%s" % (timestamp, board),
170        )
171    self._noschedv2 = noschedv2
172    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
173
174
175  def _FinishSetup(self):
176    # Get correct .boto file
177    current_dir = os.getcwd()
178    src = "/usr/local/google/home/mobiletc-prebuild/.boto"
179    dest = os.path.join(current_dir, self._chromeos_root,
180                        "src/private-overlays/chromeos-overlay/"
181                        "googlestorage_account.boto")
182    # Copy the file to the correct place
183    copy_cmd = "cp %s %s" % (src, dest)
184    retval = self._ce.RunCommand(copy_cmd)
185    if retval != 0:
186      raise RuntimeError("Couldn't copy .boto file for google storage.")
187
188    # Fix protections on ssh key
189    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
190               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
191               "/testing_rsa")
192    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
193    if retval != 0:
194      raise RuntimeError("chmod for testing_rsa failed")
195
196  def _TestLabels(self, labels):
197    experiment_file = "toolchain_experiment.txt"
198    image_args = ""
199    if self._force_mismatch:
200      image_args = "--force-mismatch"
201    experiment_header = """
202    board: %s
203    remote: %s
204    retries: 1
205    """ % (self._board, self._remotes)
206    experiment_tests = """
207    benchmark: all_toolchain_perf {
208      suite: telemetry_Crosperf
209      iterations: 3
210    }
211    """
212    with open(experiment_file, "w") as f:
213      print >>f, experiment_header
214      print >>f, experiment_tests
215      for label in labels:
216        # TODO(asharif): Fix crosperf so it accepts labels with symbols
217        crosperf_label = label
218        crosperf_label = crosperf_label.replace("-", "_")
219        crosperf_label = crosperf_label.replace("+", "_")
220        crosperf_label = crosperf_label.replace(".", "")
221
222        # Use the official build instead of building vanilla ourselves.
223        if label == "vanilla":
224          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
225
226          # Now add 'official build' to test file.
227          official_image = """
228          official_image {
229            chromeos_root: %s
230            build: %s
231          }
232          """ % (self._chromeos_root, build_name)
233          print >>f, official_image
234
235        else:
236          experiment_image = """
237          %s {
238            chromeos_image: %s
239            image_args: %s
240          }
241          """ % (crosperf_label,
242                 os.path.join(misc.GetImageDir(self._chromeos_root,
243                                               self._board),
244                              label, "chromiumos_test_image.bin"),
245                 image_args)
246          print >>f, experiment_image
247
248    crosperf = os.path.join(os.path.dirname(__file__),
249                            "crosperf",
250                            "crosperf")
251    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
252    command = ("{crosperf} --no_email=True --results_dir={r_dir} "
253               "--json_report=True {noschedv2_opts} {exp_file}").format(
254                crosperf=crosperf,
255                r_dir=self._reports_dir,
256                noschedv2_opts=noschedv2_opts,
257                exp_file=experiment_file)
258
259    ret = self._ce.RunCommand(command)
260    if ret != 0:
261      raise RuntimeError("Couldn't run crosperf!")
262    else:
263      # Copy json report to pending archives directory.
264      command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR)
265      ret = self._ce.RunCommand(command)
266    return
267
268
269  def _CopyWeeklyReportFiles(self, labels):
270    """Create tar files of the custom and official images and copy them
271    to the weekly reports directory, so they exist when the weekly report
272    gets generated.  IMPORTANT NOTE: This function must run *after*
273    crosperf has been run; otherwise the vanilla images will not be there.
274    """
275    images_path = os.path.join(os.path.realpath(self._chromeos_root),
276                               "src/build/images", self._board)
277    weekday = time.strftime("%a")
278    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
279    dest_dir = os.path.join (data_dir, weekday)
280    if not os.path.exists(dest_dir):
281      os.makedirs(dest_dir)
282    # Make sure dest_dir is empty (clean out last week's data).
283    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
284    self._ce.RunCommand(cmd)
285    # Now create new tar files and copy them over.
286    for l in labels:
287      test_path = os.path.join(images_path, l)
288      if os.path.exists(test_path):
289        if l != "vanilla":
290          label_name = "test"
291        else:
292          label_name = "vanilla"
293        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
294        cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
295               "cp %s %s/.") % (images_path,
296                                tar_file_name,
297                                l, tar_file_name,
298                                dest_dir)
299        tar_ret = self._ce.RunCommand(cmd)
300        if tar_ret != 0:
301          self._l.LogOutput("Error while creating/copying test tar file(%s)."
302                            % tar_file_name)
303
304  def _SendEmail(self):
305    """Find email msesage generated by crosperf and send it."""
306    filename = os.path.join(self._reports_dir, "msg_body.html")
307    if (os.path.exists(filename) and
308        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
309      command = ('cat %s | %s -s "Nightly test results, %s" -team -html'
310                 % (filename, MAIL_PROGRAM, self._board))
311      self._ce.RunCommand(command)
312
313  def DoAll(self):
314    self._CheckoutChromeOS()
315    labels = []
316    labels.append("vanilla")
317    for config in self._configs:
318      label = self._BuildLabelName(config.gcc_config.githash, self._board)
319      if (not misc.DoesLabelExist(self._chromeos_root,
320                                  self._board,
321                                  label)):
322        self._BuildToolchain(config)
323        label = self._BuildAndImage(label)
324      labels.append(label)
325    self._FinishSetup()
326    self._TestLabels(labels)
327    self._SendEmail()
328    # Only try to copy the image files if the test runs ran successfully.
329    self._CopyWeeklyReportFiles(labels)
330    if self._clean:
331      ret = self._DeleteChroot()
332      if ret != 0:
333        return ret
334      ret = self._DeleteCcahe()
335      if ret != 0:
336        return ret
337    return 0
338
339
340def Main(argv):
341  """The main function."""
342  # Common initializations
343###  command_executer.InitCommandExecuter(True)
344  command_executer.InitCommandExecuter()
345  parser = optparse.OptionParser()
346  parser.add_option("--remote",
347                    dest="remote",
348                    help="Remote machines to run tests on.")
349  parser.add_option("--board",
350                    dest="board",
351                    default="x86-zgb",
352                    help="The target board.")
353  parser.add_option("--githashes",
354                    dest="githashes",
355                    default="master",
356                    help="The gcc githashes to test.")
357  parser.add_option("--clean",
358                    dest="clean",
359                    default=False,
360                    action="store_true",
361                    help="Clean the chroot after testing.")
362  parser.add_option("--public",
363                    dest="public",
364                    default=False,
365                    action="store_true",
366                    help="Use the public checkout/build.")
367  parser.add_option("--force-mismatch",
368                    dest="force_mismatch",
369                    default="",
370                    help="Force the image regardless of board mismatch")
371  parser.add_option("--noschedv2",
372                    dest="noschedv2",
373                    action="store_true",
374                    default=False,
375                    help="Pass --noschedv2 to crosperf.")
376  options, _ = parser.parse_args(argv)
377  if not options.board:
378    print "Please give a board."
379    return 1
380  if not options.remote:
381    print "Please give at least one remote machine."
382    return 1
383  toolchain_configs = []
384  for githash in options.githashes.split(","):
385    gcc_config = GCCConfig(githash=githash)
386    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
387    toolchain_configs.append(toolchain_config)
388  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
389                           options.clean, options.public,
390                           options.force_mismatch,
391                           options.noschedv2)
392  return fc.DoAll()
393
394
395if __name__ == "__main__":
396  retval = Main(sys.argv)
397  sys.exit(retval)
398