test_toolchains.py revision e54c75c2f1b65054ce7562132102f194cf1c7d48 
1#!/usr/bin/python2
2
3# Script to test different toolchains against ChromeOS benchmarks.
4"""Toolchain team nightly performance test script (local builds)."""
5
6from __future__ import print_function
7
8import argparse
9import datetime
10import os
11import sys
12import build_chromeos
13import setup_chromeos
14import time
15from cros_utils import command_executer
16from cros_utils import misc
17from cros_utils import logger
18
19CROSTC_ROOT = '/usr/local/google/crostc'
20MAIL_PROGRAM = '~/var/bin/mail-sheriff'
21PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
22NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
23
24
25class GCCConfig(object):
26  """GCC configuration class."""
27
28  def __init__(self, githash):
29    self.githash = githash
30
31
32class ToolchainConfig(object):
33  """Toolchain configuration class."""
34
35  def __init__(self, gcc_config=None):
36    self.gcc_config = gcc_config
37
38
39class ChromeOSCheckout(object):
40  """Main class for checking out, building and testing ChromeOS."""
41
42  def __init__(self, board, chromeos_root):
43    self._board = board
44    self._chromeos_root = chromeos_root
45    self._ce = command_executer.GetCommandExecuter()
46    self._l = logger.GetLogger()
47    self._build_num = None
48
49  def _DeleteChroot(self):
50    command = 'cd %s; cros_sdk --delete' % self._chromeos_root
51    return self._ce.RunCommand(command)
52
53  def _DeleteCcahe(self):
54    # crosbug.com/34956
55    command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache')
56    return self._ce.RunCommand(command)
57
58  def _GetBuildNumber(self):
59    """Get the build number of the ChromeOS image from the chroot.
60
61    This function assumes a ChromeOS image has been built in the chroot.
62    It translates the 'latest' symlink in the
63    <chroot>/src/build/images/<board> directory, to find the actual
64    ChromeOS build number for the image that was built.  For example, if
65    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
66    This function would parse it out and assign 'R37-5982' to self._build_num.
67    This is used to determine the official, vanilla build to use for
68    comparison tests.
69    """
70    # Get the path to 'latest'
71    sym_path = os.path.join(
72        misc.GetImageDir(self._chromeos_root, self._board), 'latest')
73    # Translate the symlink to its 'real' path.
74    real_path = os.path.realpath(sym_path)
75    # Break up the path and get the last piece
76    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
77    path_pieces = real_path.split('/')
78    last_piece = path_pieces[-1]
79    # Break this piece into the image number + other pieces, and get the
80    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
81    image_parts = last_piece.split('.')
82    self._build_num = image_parts[0]
83
84  def _BuildLabelName(self, config):
85    pieces = config.split('/')
86    compiler_version = pieces[-1]
87    label = compiler_version + '_tot_afdo'
88    return label
89
90  def _BuildAndImage(self, label=''):
91    if (not label or
92        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
93      build_chromeos_args = [
94          build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root,
95          '--board=%s' % self._board, '--rebuild'
96      ]
97      if self._public:
98        build_chromeos_args.append('--env=USE=-chrome_internal')
99
100      ret = build_chromeos.Main(build_chromeos_args)
101      if ret != 0:
102        raise RuntimeError("Couldn't build ChromeOS!")
103
104      if not self._build_num:
105        self._GetBuildNumber()
106      # Check to see if we need to create the symbolic link for the vanilla
107      # image, and do so if appropriate.
108      if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'):
109        build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
110        full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root,
111                                         'chroot/tmp', build_name)
112        misc.LabelLatestImage(self._chromeos_root, self._board, label,
113                              full_vanilla_path)
114      else:
115        misc.LabelLatestImage(self._chromeos_root, self._board, label)
116    return label
117
118  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
119    env_string = misc.GetEnvStringFromDict(env_dict)
120    command = ('%s %s' % (env_string, misc.GetSetupBoardCommand(
121        self._board, usepkg=usepkg_flag, force=clobber_flag)))
122    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
123    error_str = "Could not setup board: '%s'" % command
124    assert ret == 0, error_str
125
126  def _UnInstallToolchain(self):
127    command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' %
128               misc.GetCtargetFromBoard(self._board, self._chromeos_root))
129    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
130    if ret != 0:
131      raise RuntimeError("Couldn't uninstall the toolchain!")
132
133  def _CheckoutChromeOS(self):
134    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
135    if not os.path.exists(self._chromeos_root):
136      setup_chromeos_args = ['--dir=%s' % self._chromeos_root]
137      if self._public:
138        setup_chromeos_args.append('--public')
139      ret = setup_chromeos.Main(setup_chromeos_args)
140      if ret != 0:
141        raise RuntimeError("Couldn't run setup_chromeos!")
142
143  def _BuildToolchain(self, config):
144    # Call setup_board for basic, vanilla setup.
145    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
146    # Now uninstall the vanilla compiler and setup/build our custom
147    # compiler.
148    self._UnInstallToolchain()
149    envdict = {
150        'USE': 'git_gcc',
151        'GCC_GITHASH': config.gcc_config.githash,
152        'EMERGE_DEFAULT_OPTS': '--exclude=gcc'
153    }
154    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
155
156
157class ToolchainComparator(ChromeOSCheckout):
158  """Main class for running tests and generating reports."""
159
160  def __init__(self,
161               board,
162               remotes,
163               configs,
164               clean,
165               public,
166               force_mismatch,
167               noschedv2=False):
168    self._board = board
169    self._remotes = remotes
170    self._chromeos_root = 'chromeos'
171    self._configs = configs
172    self._clean = clean
173    self._public = public
174    self._force_mismatch = force_mismatch
175    self._ce = command_executer.GetCommandExecuter()
176    self._l = logger.GetLogger()
177    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
178                                           '%Y-%m-%d_%H:%M:%S')
179    self._reports_dir = os.path.join(
180        NIGHTLY_TESTS_DIR,
181        '%s.%s' % (timestamp, board),)
182    self._noschedv2 = noschedv2
183    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
184
185  def _FinishSetup(self):
186    # Get correct .boto file
187    current_dir = os.getcwd()
188    src = '/usr/local/google/home/mobiletc-prebuild/.boto'
189    dest = os.path.join(current_dir, self._chromeos_root,
190                        'src/private-overlays/chromeos-overlay/'
191                        'googlestorage_account.boto')
192    # Copy the file to the correct place
193    copy_cmd = 'cp %s %s' % (src, dest)
194    retv = self._ce.RunCommand(copy_cmd)
195    if retv != 0:
196      raise RuntimeError("Couldn't copy .boto file for google storage.")
197
198    # Fix protections on ssh key
199    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
200               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
201               '/testing_rsa')
202    retv = self._ce.ChrootRunCommand(self._chromeos_root, command)
203    if retv != 0:
204      raise RuntimeError('chmod for testing_rsa failed')
205
206  def _TestLabels(self, labels):
207    experiment_file = 'toolchain_experiment.txt'
208    image_args = ''
209    if self._force_mismatch:
210      image_args = '--force-mismatch'
211    experiment_header = """
212    board: %s
213    remote: %s
214    retries: 1
215    """ % (self._board, self._remotes)
216    experiment_tests = """
217    benchmark: all_toolchain_perf {
218      suite: telemetry_Crosperf
219      iterations: 3
220    }
221    """
222
223    with open(experiment_file, 'w') as f:
224      f.write(experiment_header)
225      f.write(experiment_tests)
226      for label in labels:
227        # TODO(asharif): Fix crosperf so it accepts labels with symbols
228        crosperf_label = label
229        crosperf_label = crosperf_label.replace('-', '_')
230        crosperf_label = crosperf_label.replace('+', '_')
231        crosperf_label = crosperf_label.replace('.', '')
232
233        # Use the official build instead of building vanilla ourselves.
234        if label == 'vanilla':
235          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
236
237          # Now add 'official build' to test file.
238          official_image = """
239          official_image {
240            chromeos_root: %s
241            build: %s
242          }
243          """ % (self._chromeos_root, build_name)
244          f.write(official_image)
245
246        else:
247          experiment_image = """
248          %s {
249            chromeos_image: %s
250            image_args: %s
251          }
252          """ % (crosperf_label, os.path.join(
253              misc.GetImageDir(self._chromeos_root, self._board), label,
254              'chromiumos_test_image.bin'), image_args)
255          f.write(experiment_image)
256
257    crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf')
258    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
259    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
260               '--json_report=True {noschedv2_opts} {exp_file}').format(
261                   crosperf=crosperf,
262                   r_dir=self._reports_dir,
263                   noschedv2_opts=noschedv2_opts,
264                   exp_file=experiment_file)
265
266    ret = self._ce.RunCommand(command)
267    if ret != 0:
268      raise RuntimeError('Crosperf execution error!')
269    else:
270      # Copy json report to pending archives directory.
271      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
272      ret = self._ce.RunCommand(command)
273    return
274
275  def _SendEmail(self):
276    """Find email msesage generated by crosperf and send it."""
277    filename = os.path.join(self._reports_dir, 'msg_body.html')
278    if (os.path.exists(filename) and
279        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
280      command = ('cat %s | %s -s "Nightly test results, %s" -team -html' %
281                 (filename, MAIL_PROGRAM, self._board))
282      self._ce.RunCommand(command)
283
284  def DoAll(self):
285    self._CheckoutChromeOS()
286    labels = []
287    labels.append('vanilla')
288    for config in self._configs:
289      label = self._BuildLabelName(config.gcc_config.githash)
290      if not misc.DoesLabelExist(self._chromeos_root, self._board, label):
291        self._BuildToolchain(config)
292        label = self._BuildAndImage(label)
293      labels.append(label)
294    self._FinishSetup()
295    self._TestLabels(labels)
296    self._SendEmail()
297    if self._clean:
298      ret = self._DeleteChroot()
299      if ret != 0:
300        return ret
301      ret = self._DeleteCcahe()
302      if ret != 0:
303        return ret
304    return 0
305
306
307def Main(argv):
308  """The main function."""
309  # Common initializations
310  ###  command_executer.InitCommandExecuter(True)
311  command_executer.InitCommandExecuter()
312  parser = argparse.ArgumentParser()
313  parser.add_argument(
314      '--remote', dest='remote', help='Remote machines to run tests on.')
315  parser.add_argument(
316      '--board', dest='board', default='x86-alex', help='The target board.')
317  parser.add_argument(
318      '--githashes',
319      dest='githashes',
320      default='master',
321      help='The gcc githashes to test.')
322  parser.add_argument(
323      '--clean',
324      dest='clean',
325      default=False,
326      action='store_true',
327      help='Clean the chroot after testing.')
328  parser.add_argument(
329      '--public',
330      dest='public',
331      default=False,
332      action='store_true',
333      help='Use the public checkout/build.')
334  parser.add_argument(
335      '--force-mismatch',
336      dest='force_mismatch',
337      default='',
338      help='Force the image regardless of board mismatch')
339  parser.add_argument(
340      '--noschedv2',
341      dest='noschedv2',
342      action='store_true',
343      default=False,
344      help='Pass --noschedv2 to crosperf.')
345  options = parser.parse_args(argv)
346  if not options.board:
347    print('Please give a board.')
348    return 1
349  if not options.remote:
350    print('Please give at least one remote machine.')
351    return 1
352  toolchain_configs = []
353  for githash in options.githashes.split(','):
354    gcc_config = GCCConfig(githash=githash)
355    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
356    toolchain_configs.append(toolchain_config)
357  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
358                           options.clean, options.public,
359                           options.force_mismatch, options.noschedv2)
360  return fc.DoAll()
361
362
363if __name__ == '__main__':
364  retval = Main(sys.argv[1:])
365  sys.exit(retval)
366