test_toolchains.py revision 94bc470daff948021e754e6a63061da221faaf77 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import optparse
5import os
6import sys
7import build_chromeos
8import setup_chromeos
9import time
10from utils import command_executer
11from utils import misc
12from utils import logger
13
14
15WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
16
17class GCCConfig(object):
18  def __init__(self, githash):
19    self.githash = githash
20
21
22class ToolchainConfig:
23  def __init__(self, gcc_config=None, binutils_config=None):
24    self.gcc_config = gcc_config
25
26
27class ChromeOSCheckout(object):
28  def __init__(self, board, chromeos_root):
29    self._board = board
30    self._chromeos_root = chromeos_root
31    self._ce = command_executer.GetCommandExecuter()
32    self._l = logger.GetLogger()
33
34  def _DeleteChroot(self):
35    command = "cd %s; cros_sdk --delete" % self._chromeos_root
36    return self._ce.RunCommand(command)
37
38  def _DeleteCcahe(self):
39    # crosbug.com/34956
40    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
41    return self._ce.RunCommand(command)
42
43  def _BuildAndImage(self, label=""):
44    if (not label or
45        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
46      build_chromeos_args = [build_chromeos.__file__,
47                             "--chromeos_root=%s" % self._chromeos_root,
48                             "--board=%s" % self._board,
49                             "--rebuild"]
50      if self._public:
51        build_chromeos_args.append("--env=USE=-chrome_internal")
52      if label == "vanilla":
53        build_chromeos_args.append("--vanilla_image")
54      ret = build_chromeos.Main(build_chromeos_args)
55      if ret:
56        raise Exception("Couldn't build ChromeOS!")
57      if label:
58        misc.LabelLatestImage(self._chromeos_root, self._board, label)
59    return label
60
61  def _SetupBoard(self, env_dict):
62    env_string = misc.GetEnvStringFromDict(env_dict)
63    command = ("%s %s" %
64               (env_string,
65                misc.GetSetupBoardCommand(self._board,
66                                          usepkg=False)))
67    ret = self._ce.ChrootRunCommand(self._chromeos_root,
68                                    command)
69    assert ret == 0, "Could not setup board with new toolchain."
70
71  def _UnInstallToolchain(self):
72    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
73               misc.GetCtargetFromBoard(self._board,
74                                   self._chromeos_root))
75    ret = self._ce.ChrootRunCommand(self._chromeos_root,
76                                    command)
77    if ret:
78      raise Exception("Couldn't uninstall the toolchain!")
79
80  def _CheckoutChromeOS(self):
81    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
82    if not os.path.exists(self._chromeos_root):
83      setup_chromeos_args = [setup_chromeos.__file__,
84                             "--dir=%s" % self._chromeos_root]
85      if self._public:
86        setup_chromeos_args.append("--public")
87      ret = setup_chromeos.Main(setup_chromeos_args)
88      if ret:
89        raise Exception("Couldn't run setup_chromeos!")
90
91
92  def _BuildToolchain(self, config):
93    self._UnInstallToolchain()
94    self._SetupBoard({"USE": "git_gcc",
95                      "GCC_GITHASH": config.gcc_config.githash,
96                      "EMERGE_DEFAULT_OPTS": "--exclude=gcc"})
97
98
99class ToolchainComparator(ChromeOSCheckout):
100  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
101    self._board = board
102    self._remotes = remotes
103    self._chromeos_root = "chromeos"
104    self._configs = configs
105    self._clean = clean
106    self._public = public
107    self._force_mismatch = force_mismatch
108    self._ce = command_executer.GetCommandExecuter()
109    self._l = logger.GetLogger()
110    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
111
112
113  def _FinishSetup(self):
114    # Get correct .boto file
115    current_dir = os.getcwd()
116    src = "/home/mobiletc-prebuild/.boto"
117    dest = os.path.join(current_dir, self._chromeos_root,
118                        "src/private-overlays/chromeos-overlay/"
119                        "googlestorage_account.boto")
120    # Copy the file to the correct place
121    copy_cmd = "cp %s %s" % (src, dest)
122    retval = self._ce.RunCommand(copy_cmd)
123    if retval:
124      raise Exception("Couldn't copy .boto file for google storage.")
125
126    # Fix protections on ssh key
127    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
128               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
129               "/testing_rsa")
130    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
131    if retval:
132      raise Exception("chmod for testing_rsa failed")
133
134  def _TestLabels(self, labels):
135    experiment_file = "toolchain_experiment.txt"
136    image_args = ""
137    if self._force_mismatch:
138      image_args = "--force-mismatch"
139    experiment_header = """
140    board: %s
141    remote: %s
142    """ % (self._board, self._remotes)
143    experiment_tests = """
144    benchmark: all_perfv2 {
145      suite: telemetry_Crosperf
146      iterations: 3
147    }
148    """
149    with open(experiment_file, "w") as f:
150      print >>f, experiment_header
151      print >>f, experiment_tests
152      for label in labels:
153        # TODO(asharif): Fix crosperf so it accepts labels with symbols
154        crosperf_label = label
155        crosperf_label = crosperf_label.replace("-", "minus")
156        crosperf_label = crosperf_label.replace("+", "plus")
157        crosperf_label = crosperf_label.replace(".", "")
158        experiment_image = """
159        %s {
160          chromeos_image: %s
161          image_args: %s
162        }
163        """ % (crosperf_label,
164               os.path.join(misc.GetImageDir(self._chromeos_root, self._board),
165                            label,
166                            "chromiumos_test_image.bin"),
167               image_args)
168        print >>f, experiment_image
169    images_path = os.path.join(os.path.realpath(self._chromeos_root),
170                               "src/build/images", self._board)
171    weekday = time.strftime("%a")
172    for l in labels:
173      test_path = os.path.join(images_path, l)
174      if os.path.exists(test_path):
175        data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
176        if l == "vanilla":
177          label_name = l
178        else:
179          label_name = "test"
180        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
181        dest_dir = os.path.join (data_dir, weekday)
182        if not os.path.exists(dest_dir):
183          os.makedirs(dest_dir)
184        cmd="cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,tar_file_name,
185                                                     l, tar_file_name,
186                                                     dest_dir)
187        tar_ret = self._ce.RunCommand(cmd)
188        if tar_ret:
189          self._l.LogOutput("Error while creating/copying test tar file (%s)."
190                            % tar_file_name)
191    crosperf = os.path.join(os.path.dirname(__file__),
192                            "crosperf",
193                            "crosperf")
194    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
195    ret = self._ce.RunCommand(command)
196    if ret:
197      raise Exception("Couldn't run crosperf!")
198
199  def DoAll(self):
200    self._CheckoutChromeOS()
201    labels = []
202    vanilla_label = self._BuildAndImage("vanilla")
203    labels.append(vanilla_label)
204    for config in self._configs:
205      label = misc.GetFilenameFromString(config.gcc_config.githash)
206      if (not misc.DoesLabelExist(self._chromeos_root,
207                                  self._board,
208                                  label)):
209        self._BuildToolchain(config)
210        label = self._BuildAndImage(label)
211      labels.append(label)
212    self._FinishSetup()
213    self._TestLabels(labels)
214    if self._clean:
215      ret = self._DeleteChroot()
216      if ret: return ret
217      ret = self._DeleteCcahe()
218      if ret: return ret
219    return 0
220
221
222def Main(argv):
223  """The main function."""
224  # Common initializations
225###  command_executer.InitCommandExecuter(True)
226  command_executer.InitCommandExecuter()
227  parser = optparse.OptionParser()
228  parser.add_option("--remote",
229                    dest="remote",
230                    help="Remote machines to run tests on.")
231  parser.add_option("--board",
232                    dest="board",
233                    default="x86-zgb",
234                    help="The target board.")
235  parser.add_option("--githashes",
236                    dest="githashes",
237                    default="master",
238                    help="The gcc githashes to test.")
239  parser.add_option("--clean",
240                    dest="clean",
241                    default=False,
242                    action="store_true",
243                    help="Clean the chroot after testing.")
244  parser.add_option("--public",
245                    dest="public",
246                    default=False,
247                    action="store_true",
248                    help="Use the public checkout/build.")
249  parser.add_option("--force-mismatch",
250                    dest="force_mismatch",
251                    default="",
252                    help="Force the image regardless of board mismatch")
253  options, _ = parser.parse_args(argv)
254  if not options.board:
255    print "Please give a board."
256    return 1
257  if not options.remote:
258    print "Please give at least one remote machine."
259    return 1
260  toolchain_configs = []
261  for githash in options.githashes.split(","):
262    gcc_config = GCCConfig(githash=githash)
263    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
264    toolchain_configs.append(toolchain_config)
265  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
266                           options.clean, options.public,
267                           options.force_mismatch)
268  return fc.DoAll()
269
270
271if __name__ == "__main__":
272  retval = Main(sys.argv)
273  sys.exit(retval)
274