test_toolchains.py revision fe054f1eac0d0d739b74dee6bd45805af86568a2 
1#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import optparse
5import os
6import sys
7import build_chromeos
8import setup_chromeos
9import time
10from utils import command_executer
11from utils import misc
12from utils import logger
13
14
15WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
16AFDO_BOARDS = ["butterfly", "lumpy", "stumpy", "stout", "parrot", "parrot_ivb"]
17
18class GCCConfig(object):
19  def __init__(self, githash):
20    self.githash = githash
21
22
23class ToolchainConfig:
24  def __init__(self, gcc_config=None, binutils_config=None):
25    self.gcc_config = gcc_config
26
27
28class ChromeOSCheckout(object):
29  def __init__(self, board, chromeos_root):
30    self._board = board
31    self._chromeos_root = chromeos_root
32    self._ce = command_executer.GetCommandExecuter()
33    self._l = logger.GetLogger()
34    self._build_num = None
35
36  def _DeleteChroot(self):
37    command = "cd %s; cros_sdk --delete" % self._chromeos_root
38    return self._ce.RunCommand(command)
39
40  def _DeleteCcahe(self):
41    # crosbug.com/34956
42    command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
43    return self._ce.RunCommand(command)
44
45  def _GetBuildNumber(self):
46    """ This function assumes a ChromeOS image has been built in the chroot.
47    It translates the 'latest' symlink in the
48    <chroot>/src/build/images/<board> directory, to find the actual
49    ChromeOS build number for the image that was built.  For example, if
50    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
51    This function would parse it out and assign 'R37-5982' to self._build_num.
52    This is used to determine the official, vanilla build to use for
53    comparison tests.
54    """
55    # Get the path to 'latest'
56    sym_path = os.path.join (misc.GetImageDir(self._chromeos_root,
57                                              self._board),
58                             "latest")
59    # Translate the symlink to its 'real' path.
60    real_path = os.path.realpath(sym_path)
61    # Break up the path and get the last piece
62    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
63    path_pieces = real_path.split("/")
64    last_piece = path_pieces[-1]
65    # Break this piece into the image number + other pieces, and get the
66    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
67    image_parts = last_piece.split(".")
68    self._build_num = image_parts[0]
69
70  def _BuildAndImage(self, label=""):
71    if (not label or
72        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
73      build_chromeos_args = [build_chromeos.__file__,
74                             "--chromeos_root=%s" % self._chromeos_root,
75                             "--board=%s" % self._board,
76                             "--rebuild"]
77      if self._public:
78        build_chromeos_args.append("--env=USE=-chrome_internal")
79
80      if self._board in AFDO_BOARDS:
81        build_chromeos_args.append("--env=USE=afdo_use")
82
83      ret = build_chromeos.Main(build_chromeos_args)
84      if ret:
85        raise Exception("Couldn't build ChromeOS!")
86
87      if not  self._build_num:
88        self._GetBuildNumber()
89      # Check to see if we need to create the symbolic link for the vanilla
90      # image, and do so if appropriate.
91      if not misc.DoesLabelExist(self._chromeos_root, self._board, "vanilla"):
92        build_name = "%s-release/%s.0.0" % (self._board, self._build_num)
93        full_vanilla_path = os.path.join (os.getcwd(), self._chromeos_root,
94                                          'chroot/tmp', build_name)
95        misc.LabelLatestImage(self._chromeos_root, self._board, label,
96                              full_vanilla_path)
97      else:
98        misc.LabelLatestImage(self._chromeos_root, self._board, label)
99    return label
100
101  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
102    env_string = misc.GetEnvStringFromDict(env_dict)
103    command = ("%s %s" %
104               (env_string,
105                misc.GetSetupBoardCommand(self._board,
106                                          usepkg=usepkg_flag,
107                                          force=clobber_flag)))
108    ret = self._ce.ChrootRunCommand(self._chromeos_root,
109                                    command)
110    error_str = "Could not setup board: '%s'" % command
111    assert ret == 0, error_str
112
113  def _UnInstallToolchain(self):
114    command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
115               misc.GetCtargetFromBoard(self._board,
116                                   self._chromeos_root))
117    ret = self._ce.ChrootRunCommand(self._chromeos_root,
118                                    command)
119    if ret:
120      raise Exception("Couldn't uninstall the toolchain!")
121
122  def _CheckoutChromeOS(self):
123    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
124    if not os.path.exists(self._chromeos_root):
125      setup_chromeos_args = [setup_chromeos.__file__,
126                             "--dir=%s" % self._chromeos_root]
127      if self._public:
128        setup_chromeos_args.append("--public")
129      ret = setup_chromeos.Main(setup_chromeos_args)
130      if ret:
131        raise Exception("Couldn't run setup_chromeos!")
132
133
134  def _BuildToolchain(self, config):
135    # Call setup_board for basic, vanilla setup.
136    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
137    # Now uninstall the vanilla compiler and setup/build our custom
138    # compiler.
139    self._UnInstallToolchain()
140    envdict = {"USE": "git_gcc",
141               "GCC_GITHASH": config.gcc_config.githash,
142               "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}
143    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
144
145
146class ToolchainComparator(ChromeOSCheckout):
147  def __init__(self, board, remotes, configs, clean, public, force_mismatch):
148    self._board = board
149    self._remotes = remotes
150    self._chromeos_root = "chromeos"
151    self._configs = configs
152    self._clean = clean
153    self._public = public
154    self._force_mismatch = force_mismatch
155    self._ce = command_executer.GetCommandExecuter()
156    self._l = logger.GetLogger()
157    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
158
159
160  def _FinishSetup(self):
161    # Get correct .boto file
162    current_dir = os.getcwd()
163    src = "/home/mobiletc-prebuild/.boto"
164    dest = os.path.join(current_dir, self._chromeos_root,
165                        "src/private-overlays/chromeos-overlay/"
166                        "googlestorage_account.boto")
167    # Copy the file to the correct place
168    copy_cmd = "cp %s %s" % (src, dest)
169    retval = self._ce.RunCommand(copy_cmd)
170    if retval:
171      raise Exception("Couldn't copy .boto file for google storage.")
172
173    # Fix protections on ssh key
174    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
175               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
176               "/testing_rsa")
177    retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
178    if retval:
179      raise Exception("chmod for testing_rsa failed")
180
181  def _TestLabels(self, labels):
182    experiment_file = "toolchain_experiment.txt"
183    image_args = ""
184    if self._force_mismatch:
185      image_args = "--force-mismatch"
186    experiment_header = """
187    board: %s
188    remote: %s
189    """ % (self._board, self._remotes)
190    experiment_tests = """
191    benchmark: all_perfv2 {
192      suite: telemetry_Crosperf
193      iterations: 3
194    }
195    """
196    with open(experiment_file, "w") as f:
197      print >>f, experiment_header
198      print >>f, experiment_tests
199      for label in labels:
200        # TODO(asharif): Fix crosperf so it accepts labels with symbols
201        crosperf_label = label
202        crosperf_label = crosperf_label.replace("-", "minus")
203        crosperf_label = crosperf_label.replace("+", "plus")
204        crosperf_label = crosperf_label.replace(".", "")
205
206        # Use the official build instead of building vanilla ourselves.
207        if label == "vanilla":
208          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
209
210          # Now add 'official build' to test file.
211          official_image = """
212          official_image {
213            chromeos_root: %s
214            build: %s
215          }
216          """ % (self._chromeos_root, build_name)
217          print >>f, official_image
218
219        else:
220          experiment_image = """
221          %s {
222            chromeos_image: %s
223            image_args: %s
224          }
225          """ % (crosperf_label,
226                 os.path.join(misc.GetImageDir(self._chromeos_root,
227                                               self._board),
228                              label, "chromiumos_test_image.bin"),
229                 image_args)
230          print >>f, experiment_image
231
232    crosperf = os.path.join(os.path.dirname(__file__),
233                            "crosperf",
234                            "crosperf")
235    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
236    ret = self._ce.RunCommand(command)
237    if ret:
238      raise Exception("Couldn't run crosperf!")
239    return ret
240
241
242  def _CopyWeeklyReportFiles(self, labels):
243    """Create tar files of the custom and official images and copy them
244    to the weekly reports directory, so they exist when the weekly report
245    gets generated.  IMPORTANT NOTE: This function must run *after*
246    crosperf has been run; otherwise the vanilla images will not be there.
247    """
248    images_path = os.path.join(os.path.realpath(self._chromeos_root),
249                               "src/build/images", self._board)
250    weekday = time.strftime("%a")
251    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
252    dest_dir = os.path.join (data_dir, weekday)
253    if not os.path.exists(dest_dir):
254      os.makedirs(dest_dir)
255    # Make sure dest_dir is empty (clean out last week's data).
256    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, weekday)
257    self._ce.RunCommand(cmd)
258    # Now create new tar files and copy them over.
259    for l in labels:
260      test_path = os.path.join(images_path, l)
261      if os.path.exists(test_path):
262        if l != "vanilla":
263          label_name = "test"
264        else:
265          label_name = "vanilla"
266        tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
267        cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
268               "cp %s %s/.") % (images_path,
269                                tar_file_name,
270                                l, tar_file_name,
271                                dest_dir)
272        tar_ret = self._ce.RunCommand(cmd)
273        if tar_ret:
274          self._l.LogOutput("Error while creating/copying test tar file(%s)."
275                            % tar_file_name)
276
277
278  def DoAll(self):
279    self._CheckoutChromeOS()
280    labels = []
281    labels.append("vanilla")
282    for config in self._configs:
283      label = misc.GetFilenameFromString(config.gcc_config.githash)
284      if (not misc.DoesLabelExist(self._chromeos_root,
285                                  self._board,
286                                  label)):
287        self._BuildToolchain(config)
288        label = self._BuildAndImage(label)
289      labels.append(label)
290    self._FinishSetup()
291    if not self._TestLabels(labels):
292      # Only try to copy the image files if the test runs ran successfully.
293      self._CopyWeeklyReportFiles(labels)
294    if self._clean:
295      ret = self._DeleteChroot()
296      if ret: return ret
297      ret = self._DeleteCcahe()
298      if ret: return ret
299    return 0
300
301
302def Main(argv):
303  """The main function."""
304  # Common initializations
305###  command_executer.InitCommandExecuter(True)
306  command_executer.InitCommandExecuter()
307  parser = optparse.OptionParser()
308  parser.add_option("--remote",
309                    dest="remote",
310                    help="Remote machines to run tests on.")
311  parser.add_option("--board",
312                    dest="board",
313                    default="x86-zgb",
314                    help="The target board.")
315  parser.add_option("--githashes",
316                    dest="githashes",
317                    default="master",
318                    help="The gcc githashes to test.")
319  parser.add_option("--clean",
320                    dest="clean",
321                    default=False,
322                    action="store_true",
323                    help="Clean the chroot after testing.")
324  parser.add_option("--public",
325                    dest="public",
326                    default=False,
327                    action="store_true",
328                    help="Use the public checkout/build.")
329  parser.add_option("--force-mismatch",
330                    dest="force_mismatch",
331                    default="",
332                    help="Force the image regardless of board mismatch")
333  options, _ = parser.parse_args(argv)
334  if not options.board:
335    print "Please give a board."
336    return 1
337  if not options.remote:
338    print "Please give at least one remote machine."
339    return 1
340  toolchain_configs = []
341  for githash in options.githashes.split(","):
342    gcc_config = GCCConfig(githash=githash)
343    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
344    toolchain_configs.append(toolchain_config)
345  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
346                           options.clean, options.public,
347                           options.force_mismatch)
348  return fc.DoAll()
349
350
351if __name__ == "__main__":
352  retval = Main(sys.argv)
353  sys.exit(retval)
354