test_toolchains.py revision a6255d058ecf1fe6e9983bb5acf7bec5213c1317 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import optparse
5import os
6import sys
7import build_chromeos
8import setup_chromeos
9from utils import command_executer
10from utils import misc
11from utils import logger
12
13
14class GCCConfig(object):
15  def __init__(self, githash):
16    self.githash = githash
17
18
19class ToolchainConfig:
20  def __init__(self, gcc_config=None, binutils_config=None):
21    self.gcc_config = gcc_config
22
23
24class ChromeOSCheckout(object):
25  def __init__(self, board, chromeos_root):
26    self._board = board
27    self._chromeos_root = chromeos_root
28    self._ce = command_executer.GetCommandExecuter()
29    self._l = logger.GetLogger()
30
31  def _DeleteChroot(self):
32    command = "cd %s; cros_sdk --delete" % self._chromeos_root
33    return self._ce.RunCommand(command)
34
35  def _DeleteCcahe(self):
36    # crosbug.com/34956
37    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
38    return self._ce.RunCommand(command)
39
40  def _BuildAndImage(self, label=""):
41    if (not label or
42        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
43      build_chromeos_args = [build_chromeos.__file__,
44                             "--chromeos_root=%s" % self._chromeos_root,
45                             "--board=%s" % self._board,
46                             "--rebuild"]
47      if self._public:
48        build_chromeos_args.append("--env=USE=-chrome_internal")
49      if label == "vanilla":
50        build_chromeos_args.append("--vanilla_image")
51      ret = build_chromeos.Main(build_chromeos_args)
52      if ret:
53        raise Exception("Couldn't build ChromeOS!")
54      if label:
55        misc.LabelLatestImage(self._chromeos_root, self._board, label)
56    return label
57
58  def _SetupBoard(self, env_dict):
59    env_string = misc.GetEnvStringFromDict(env_dict)
60    command = ("%s %s" %
61               (env_string,
62                misc.GetSetupBoardCommand(self._board,
63                                          usepkg=False)))
64    ret = self._ce.ChrootRunCommand(self._chromeos_root,
65                                    command)
66    assert ret == 0, "Could not setup board with new toolchain."
67
68  def _UnInstallToolchain(self):
69    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
70               misc.GetCtargetFromBoard(self._board,
71                                   self._chromeos_root))
72    ret = self._ce.ChrootRunCommand(self._chromeos_root,
73                                    command)
74    if ret:
75      raise Exception("Couldn't uninstall the toolchain!")
76
77  def _CheckoutChromeOS(self):
78    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
79    if not os.path.exists(self._chromeos_root):
80      setup_chromeos_args = [setup_chromeos.__file__,
81                             "--dir=%s" % self._chromeos_root]
82      if self._public:
83        setup_chromeos_args.append("--public")
84      ret = setup_chromeos.Main(setup_chromeos_args)
85      if ret:
86        raise Exception("Couldn't run setup_chromeos!")
87
88
89  def _BuildToolchain(self, config):
90    self._UnInstallToolchain()
91    self._SetupBoard({"USE": "git_gcc",
92                      "GCC_GITHASH": config.gcc_config.githash,
93                      "EMERGE_DEFAULT_OPTS": "--exclude=gcc"})
94
95
96class ToolchainComparator(ChromeOSCheckout):
97  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
98    self._board = board
99    self._remotes = remotes
100    self._chromeos_root = "chromeos"
101    self._configs = configs
102    self._clean = clean
103    self._public = public
104    self._force_mismatch = force_mismatch
105    self._ce = command_executer.GetCommandExecuter()
106    self._l = logger.GetLogger()
107    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
108
109
110  def _FinishSetup(self):
111    # Get correct .boto file
112    current_dir = os.getcwd()
113    src = "/home/mobiletc-prebuild/.boto"
114    dest = os.path.join(current_dir, self._chromeos_root,
115                        "src/private-overlays/chromeos-overlay/"
116                        "googlestorage_account.boto")
117    # Copy the file to the correct place
118    copy_cmd = "cp %s %s" % (src, dest)
119    retval = self._ce.RunCommand(copy_cmd)
120    if retval:
121      raise Exception("Couldn't copy .boto file for google storage.")
122
123    # Fix protections on ssh key
124    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
125               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
126               "/testing_rsa")
127    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
128    if retval:
129      raise Exception("chmod for testing_rsa failed")
130
131  def _TestLabels(self, labels):
132    experiment_file = "toolchain_experiment.txt"
133    image_args = ""
134    if self._force_mismatch:
135      image_args = "--force-mismatch"
136    experiment_header = """
137    board: %s
138    remote: %s
139    """ % (self._board, self._remotes)
140    experiment_tests = """
141    benchmark: all_perfv2 {
142      suite: telemetry_Crosperf
143      iterations: 1
144    }
145    """
146    with open(experiment_file, "w") as f:
147      print >>f, experiment_header
148      print >>f, experiment_tests
149      for label in labels:
150        # TODO(asharif): Fix crosperf so it accepts labels with symbols
151        crosperf_label = label
152        crosperf_label = crosperf_label.replace("-", "minus")
153        crosperf_label = crosperf_label.replace("+", "plus")
154        crosperf_label = crosperf_label.replace(".", "")
155        experiment_image = """
156        %s {
157          chromeos_image: %s
158          image_args: %s
159        }
160        """ % (crosperf_label,
161               os.path.join(misc.GetImageDir(self._chromeos_root, self._board),
162                            label,
163                            "chromiumos_test_image.bin"),
164               image_args)
165        print >>f, experiment_image
166    crosperf = os.path.join(os.path.dirname(__file__),
167                            "crosperf",
168                            "crosperf")
169    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
170    ret = self._ce.RunCommand(command)
171    if ret:
172      raise Exception("Couldn't run crosperf!")
173
174  def DoAll(self):
175    self._CheckoutChromeOS()
176    labels = []
177    vanilla_label = self._BuildAndImage("vanilla")
178    labels.append(vanilla_label)
179    for config in self._configs:
180      label = misc.GetFilenameFromString(config.gcc_config.githash)
181      if (not misc.DoesLabelExist(self._chromeos_root,
182                                  self._board,
183                                  label)):
184        self._BuildToolchain(config)
185        label = self._BuildAndImage(label)
186      labels.append(label)
187    self._FinishSetup()
188    self._TestLabels(labels)
189    if self._clean:
190      ret = self._DeleteChroot()
191      if ret: return ret
192      ret = self._DeleteCcahe()
193      if ret: return ret
194    return 0
195
196
197def Main(argv):
198  """The main function."""
199  # Common initializations
200###  command_executer.InitCommandExecuter(True)
201  command_executer.InitCommandExecuter()
202  parser = optparse.OptionParser()
203  parser.add_option("--remote",
204                    dest="remote",
205                    help="Remote machines to run tests on.")
206  parser.add_option("--board",
207                    dest="board",
208                    default="x86-zgb",
209                    help="The target board.")
210  parser.add_option("--githashes",
211                    dest="githashes",
212                    default="master",
213                    help="The gcc githashes to test.")
214  parser.add_option("--clean",
215                    dest="clean",
216                    default=False,
217                    action="store_true",
218                    help="Clean the chroot after testing.")
219  parser.add_option("--public",
220                    dest="public",
221                    default=False,
222                    action="store_true",
223                    help="Use the public checkout/build.")
224  parser.add_option("--force-mismatch",
225                    dest="force_mismatch",
226                    default="",
227                    help="Force the image regardless of board mismatch")
228  options, _ = parser.parse_args(argv)
229  if not options.board:
230    print "Please give a board."
231    return 1
232  if not options.remote:
233    print "Please give at least one remote machine."
234    return 1
235  toolchain_configs = []
236  for githash in options.githashes.split(","):
237    gcc_config = GCCConfig(githash=githash)
238    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
239    toolchain_configs.append(toolchain_config)
240  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
241                           options.clean, options.public,
242                           options.force_mismatch)
243  return fc.DoAll()
244
245
246if __name__ == "__main__":
247  retval = Main(sys.argv)
248  sys.exit(retval)
249