test_toolchains.py revision 4536ef6c9da0f8c0c544552798c0bcf516519f73 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import optparse
5import os
6import sys
7import build_chromeos
8import setup_chromeos
9import time
10from utils import command_executer
11from utils import misc
12from utils import logger
13
14
15WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
16
17class GCCConfig(object):
18  def __init__(self, githash):
19    self.githash = githash
20
21
22class ToolchainConfig:
23  def __init__(self, gcc_config=None, binutils_config=None):
24    self.gcc_config = gcc_config
25
26
27class ChromeOSCheckout(object):
28  def __init__(self, board, chromeos_root):
29    self._board = board
30    self._chromeos_root = chromeos_root
31    self._ce = command_executer.GetCommandExecuter()
32    self._l = logger.GetLogger()
33    self._build_num = None
34
35  def _DeleteChroot(self):
36    command = "cd %s; cros_sdk --delete" % self._chromeos_root
37    return self._ce.RunCommand(command)
38
39  def _DeleteCcahe(self):
40    # crosbug.com/34956
41    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
42    return self._ce.RunCommand(command)
43
44  def _GetBuildNumber(self):
45    """ This function assumes a ChromeOS image has been built in the chroot.
46    It translates the 'latest' symlink in the
47    <chroot>/src/build/images/<board> directory, to find the actual
48    ChromeOS build number for the image that was built.  For example, if
49    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
50    This function would parse it out and assign 'R37-5982' to self._build_num.
51    This is used to determine the official, vanilla build to use for
52    comparison tests.
53    """
54    # Get the path to 'latest'
55    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
56                                              self._board),
57                             "latest")
58    # Translate the symlink to its 'real' path.
59    real_path = os.path.realpath(sym_path)
60    # Break up the path and get the last piece
61    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
62    path_pieces = real_path.split("/")
63    last_piece = path_pieces[-1]
64    # Break this piece into the image number + other pieces, and get the
65    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
66    image_parts = last_piece.split(".")
67    self._build_num = image_parts[0]
68
69  def _BuildAndImage(self, label=""):
70    if (not label or
71        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
72      build_chromeos_args = [build_chromeos.__file__,
73                             "--chromeos_root=%s" % self._chromeos_root,
74                             "--board=%s" % self._board,
75                             "--rebuild"]
76      if self._public:
77        build_chromeos_args.append("--env=USE=-chrome_internal")
78
79      ret = build_chromeos.Main(build_chromeos_args)
80      if ret:
81        raise Exception("Couldn't build ChromeOS!")
82
83      if not  self._build_num:
84        self._GetBuildNumber()
85      # Check to see if we need to create the symbolic link for the vanilla
86      # image, and do so if appropriate.
87      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
88        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
89        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
90                                          'chroot/tmp', build_name)
91        misc.LabelLatestImage(self._chromeos_root, self._board, label,
92                              full_vanilla_path)
93      else:
94        misc.LabelLatestImage(self._chromeos_root, self._board, label)
95    return label
96
97  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
98    env_string = misc.GetEnvStringFromDict(env_dict)
99    command = ("%s %s" %
100               (env_string,
101                misc.GetSetupBoardCommand(self._board,
102                                          usepkg=usepkg_flag,
103                                          force=clobber_flag)))
104    ret = self._ce.ChrootRunCommand(self._chromeos_root,
105                                    command)
106    error_str = "Could not setup board: '%s'" % command
107    assert ret == 0, error_str
108
109  def _UnInstallToolchain(self):
110    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
111               misc.GetCtargetFromBoard(self._board,
112                                   self._chromeos_root))
113    ret = self._ce.ChrootRunCommand(self._chromeos_root,
114                                    command)
115    if ret:
116      raise Exception("Couldn't uninstall the toolchain!")
117
118  def _CheckoutChromeOS(self):
119    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
120    if not os.path.exists(self._chromeos_root):
121      setup_chromeos_args = [setup_chromeos.__file__,
122                             "--dir=%s" % self._chromeos_root]
123      if self._public:
124        setup_chromeos_args.append("--public")
125      ret = setup_chromeos.Main(setup_chromeos_args)
126      if ret:
127        raise Exception("Couldn't run setup_chromeos!")
128
129
130  def _BuildToolchain(self, config):
131    # Call setup_board for basic, vanilla setup.
132    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
133    # Now uninstall the vanilla compiler and setup/build our custom
134    # compiler.
135    self._UnInstallToolchain()
136    envdict = {"USE": "git_gcc",
137               "GCC_GITHASH": config.gcc_config.githash,
138               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
139    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
140
141
142class ToolchainComparator(ChromeOSCheckout):
143  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
144    self._board = board
145    self._remotes = remotes
146    self._chromeos_root = "chromeos"
147    self._configs = configs
148    self._clean = clean
149    self._public = public
150    self._force_mismatch = force_mismatch
151    self._ce = command_executer.GetCommandExecuter()
152    self._l = logger.GetLogger()
153    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
154
155
156  def _FinishSetup(self):
157    # Get correct .boto file
158    current_dir = os.getcwd()
159    src = "/home/mobiletc-prebuild/.boto"
160    dest = os.path.join(current_dir, self._chromeos_root,
161                        "src/private-overlays/chromeos-overlay/"
162                        "googlestorage_account.boto")
163    # Copy the file to the correct place
164    copy_cmd = "cp %s %s" % (src, dest)
165    retval = self._ce.RunCommand(copy_cmd)
166    if retval:
167      raise Exception("Couldn't copy .boto file for google storage.")
168
169    # Fix protections on ssh key
170    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
171               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
172               "/testing_rsa")
173    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
174    if retval:
175      raise Exception("chmod for testing_rsa failed")
176
177  def _TestLabels(self, labels):
178    experiment_file = "toolchain_experiment.txt"
179    image_args = ""
180    if self._force_mismatch:
181      image_args = "--force-mismatch"
182    experiment_header = """
183    board: %s
184    remote: %s
185    """ % (self._board, self._remotes)
186    experiment_tests = """
187    benchmark: all_perfv2 {
188      suite: telemetry_Crosperf
189      iterations: 3
190    }
191    """
192    with open(experiment_file, "w") as f:
193      print >>f, experiment_header
194      print >>f, experiment_tests
195      for label in labels:
196        # TODO(asharif): Fix crosperf so it accepts labels with symbols
197        crosperf_label = label
198        crosperf_label = crosperf_label.replace("-", "minus")
199        crosperf_label = crosperf_label.replace("+", "plus")
200        crosperf_label = crosperf_label.replace(".", "")
201
202        # Use the official build instead of building vanilla ourselves.
203        if label == "vanilla":
204          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
205
206          # Now add 'official build' to test file.
207          official_image = """
208          official_image {
209            chromeos_root: %s
210            build: %s
211          }
212          """ % (self._chromeos_root, build_name)
213          print >>f, official_image
214
215        else:
216          experiment_image = """
217          %s {
218            chromeos_image: %s
219            image_args: %s
220          }
221          """ % (crosperf_label,
222                 os.path.join(misc.GetImageDir(self._chromeos_root,
223                                               self._board),
224                              label, "chromiumos_test_image.bin"),
225                 image_args)
226          print >>f, experiment_image
227
228    crosperf = os.path.join(os.path.dirname(__file__),
229                            "crosperf",
230                            "crosperf")
231    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
232    ret = self._ce.RunCommand(command)
233    if ret:
234      raise Exception("Couldn't run crosperf!")
235    return ret
236
237
238  def _CopyWeeklyReportFiles(self, labels):
239    """Create tar files of the custom and official images and copy them
240    to the weekly reports directory, so they exist when the weekly report
241    gets generated.  IMPORTANT NOTE: This function must run *after*
242    crosperf has been run; otherwise the vanilla images will not be there.
243    """
244    images_path = os.path.join(os.path.realpath(self._chromeos_root),
245                               "src/build/images", self._board)
246    weekday = time.strftime("%a")
247    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
248    dest_dir = os.path.join (data_dir, weekday)
249    if not os.path.exists(dest_dir):
250      os.makedirs(dest_dir)
251    # Make sure dest_dir is empty (clean out last week's data).
252    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
253    self._ce.RunCommand(cmd)
254    # Now create new tar files and copy them over.
255    for l in labels:
256      test_path = os.path.join(images_path, l)
257      if os.path.exists(test_path):
258        if l != "vanilla":
259          label_name = "test"
260        else:
261          label_name = "vanilla"
262        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
263        cmd = "cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,
264                                                       tar_file_name,
265                                                       l, tar_file_name,
266                                                       dest_dir)
267        tar_ret = self._ce.RunCommand(cmd)
268        if tar_ret:
269          self._l.LogOutput("Error while creating/copying test tar file(%s)."
270                            % tar_file_name)
271
272
273  def DoAll(self):
274    self._CheckoutChromeOS()
275    labels = []
276    labels.append("vanilla")
277    for config in self._configs:
278      label = misc.GetFilenameFromString(config.gcc_config.githash)
279      if (not misc.DoesLabelExist(self._chromeos_root,
280                                  self._board,
281                                  label)):
282        self._BuildToolchain(config)
283        label = self._BuildAndImage(label)
284      labels.append(label)
285    self._FinishSetup()
286    if not self._TestLabels(labels):
287      # Only try to copy the image files if the test runs ran successfully.
288      self._CopyWeeklyReportFiles(labels)
289    if self._clean:
290      ret = self._DeleteChroot()
291      if ret: return ret
292      ret = self._DeleteCcahe()
293      if ret: return ret
294    return 0
295
296
297def Main(argv):
298  """The main function."""
299  # Common initializations
300###  command_executer.InitCommandExecuter(True)
301  command_executer.InitCommandExecuter()
302  parser = optparse.OptionParser()
303  parser.add_option("--remote",
304                    dest="remote",
305                    help="Remote machines to run tests on.")
306  parser.add_option("--board",
307                    dest="board",
308                    default="x86-zgb",
309                    help="The target board.")
310  parser.add_option("--githashes",
311                    dest="githashes",
312                    default="master",
313                    help="The gcc githashes to test.")
314  parser.add_option("--clean",
315                    dest="clean",
316                    default=False,
317                    action="store_true",
318                    help="Clean the chroot after testing.")
319  parser.add_option("--public",
320                    dest="public",
321                    default=False,
322                    action="store_true",
323                    help="Use the public checkout/build.")
324  parser.add_option("--force-mismatch",
325                    dest="force_mismatch",
326                    default="",
327                    help="Force the image regardless of board mismatch")
328  options, _ = parser.parse_args(argv)
329  if not options.board:
330    print "Please give a board."
331    return 1
332  if not options.remote:
333    print "Please give at least one remote machine."
334    return 1
335  toolchain_configs = []
336  for githash in options.githashes.split(","):
337    gcc_config = GCCConfig(githash=githash)
338    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
339    toolchain_configs.append(toolchain_config)
340  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
341                           options.clean, options.public,
342                           options.force_mismatch)
343  return fc.DoAll()
344
345
346if __name__ == "__main__":
347  retval = Main(sys.argv)
348  sys.exit(retval)
349