test_toolchains.py revision aee96b71ad4ffde231ca5bf9c0509d15b0e9b753 
1#!/usr/bin/python2
2
3# Script to test different toolchains against ChromeOS benchmarks.
4"""Toolchain team nightly performance test script (local builds)."""
5
6from __future__ import print_function
7
8import argparse
9import datetime
10import os
11import sys
12import build_chromeos
13import setup_chromeos
14import time
15from cros_utils import command_executer
16from cros_utils import misc
17from cros_utils import logger
18
19CROSTC_ROOT = '/usr/local/google/crostc'
20MAIL_PROGRAM = '~/var/bin/mail-sheriff'
21WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
22PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
23NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
24
25
26class GCCConfig(object):
27  """GCC configuration class."""
28
29  def __init__(self, githash):
30    self.githash = githash
31
32
33class ToolchainConfig(object):
34  """Toolchain configuration class."""
35
36  def __init__(self, gcc_config=None):
37    self.gcc_config = gcc_config
38
39
40class ChromeOSCheckout(object):
41  """Main class for checking out, building and testing ChromeOS."""
42
43  def __init__(self, board, chromeos_root):
44    self._board = board
45    self._chromeos_root = chromeos_root
46    self._ce = command_executer.GetCommandExecuter()
47    self._l = logger.GetLogger()
48    self._build_num = None
49
50  def _DeleteChroot(self):
51    command = 'cd %s; cros_sdk --delete' % self._chromeos_root
52    return self._ce.RunCommand(command)
53
54  def _DeleteCcahe(self):
55    # crosbug.com/34956
56    command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache')
57    return self._ce.RunCommand(command)
58
59  def _GetBuildNumber(self):
60    """Get the build number of the ChromeOS image from the chroot.
61
62    This function assumes a ChromeOS image has been built in the chroot.
63    It translates the 'latest' symlink in the
64    <chroot>/src/build/images/<board> directory, to find the actual
65    ChromeOS build number for the image that was built.  For example, if
66    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
67    This function would parse it out and assign 'R37-5982' to self._build_num.
68    This is used to determine the official, vanilla build to use for
69    comparison tests.
70    """
71    # Get the path to 'latest'
72    sym_path = os.path.join(
73        misc.GetImageDir(self._chromeos_root, self._board), 'latest')
74    # Translate the symlink to its 'real' path.
75    real_path = os.path.realpath(sym_path)
76    # Break up the path and get the last piece
77    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
78    path_pieces = real_path.split('/')
79    last_piece = path_pieces[-1]
80    # Break this piece into the image number + other pieces, and get the
81    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
82    image_parts = last_piece.split('.')
83    self._build_num = image_parts[0]
84
85  def _BuildLabelName(self, config):
86    pieces = config.split('/')
87    compiler_version = pieces[-1]
88    label = compiler_version + '_tot_afdo'
89    return label
90
91  def _BuildAndImage(self, label=''):
92    if (not label or
93        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
94      build_chromeos_args = [
95          build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root,
96          '--board=%s' % self._board, '--rebuild'
97      ]
98      if self._public:
99        build_chromeos_args.append('--env=USE=-chrome_internal')
100
101      ret = build_chromeos.Main(build_chromeos_args)
102      if ret != 0:
103        raise RuntimeError("Couldn't build ChromeOS!")
104
105      if not self._build_num:
106        self._GetBuildNumber()
107      # Check to see if we need to create the symbolic link for the vanilla
108      # image, and do so if appropriate.
109      if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'):
110        build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
111        full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root,
112                                         'chroot/tmp', build_name)
113        misc.LabelLatestImage(self._chromeos_root, self._board, label,
114                              full_vanilla_path)
115      else:
116        misc.LabelLatestImage(self._chromeos_root, self._board, label)
117    return label
118
119  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
120    env_string = misc.GetEnvStringFromDict(env_dict)
121    command = ('%s %s' % (env_string, misc.GetSetupBoardCommand(
122        self._board, usepkg=usepkg_flag, force=clobber_flag)))
123    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
124    error_str = "Could not setup board: '%s'" % command
125    assert ret == 0, error_str
126
127  def _UnInstallToolchain(self):
128    command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' %
129               misc.GetCtargetFromBoard(self._board, self._chromeos_root))
130    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
131    if ret != 0:
132      raise RuntimeError("Couldn't uninstall the toolchain!")
133
134  def _CheckoutChromeOS(self):
135    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
136    if not os.path.exists(self._chromeos_root):
137      setup_chromeos_args = ['--dir=%s' % self._chromeos_root]
138      if self._public:
139        setup_chromeos_args.append('--public')
140      ret = setup_chromeos.Main(setup_chromeos_args)
141      if ret != 0:
142        raise RuntimeError("Couldn't run setup_chromeos!")
143
144  def _BuildToolchain(self, config):
145    # Call setup_board for basic, vanilla setup.
146    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
147    # Now uninstall the vanilla compiler and setup/build our custom
148    # compiler.
149    self._UnInstallToolchain()
150    envdict = {
151        'USE': 'git_gcc',
152        'GCC_GITHASH': config.gcc_config.githash,
153        'EMERGE_DEFAULT_OPTS': '--exclude=gcc'
154    }
155    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
156
157
158class ToolchainComparator(ChromeOSCheckout):
159  """Main class for running tests and generating reports."""
160
161  def __init__(self,
162               board,
163               remotes,
164               configs,
165               clean,
166               public,
167               force_mismatch,
168               noschedv2=False):
169    self._board = board
170    self._remotes = remotes
171    self._chromeos_root = 'chromeos'
172    self._configs = configs
173    self._clean = clean
174    self._public = public
175    self._force_mismatch = force_mismatch
176    self._ce = command_executer.GetCommandExecuter()
177    self._l = logger.GetLogger()
178    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
179                                           '%Y-%m-%d_%H:%M:%S')
180    self._reports_dir = os.path.join(
181        NIGHTLY_TESTS_DIR,
182        '%s.%s' % (timestamp, board),)
183    self._noschedv2 = noschedv2
184    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
185
186  def _FinishSetup(self):
187    # Get correct .boto file
188    current_dir = os.getcwd()
189    src = '/usr/local/google/home/mobiletc-prebuild/.boto'
190    dest = os.path.join(current_dir, self._chromeos_root,
191                        'src/private-overlays/chromeos-overlay/'
192                        'googlestorage_account.boto')
193    # Copy the file to the correct place
194    copy_cmd = 'cp %s %s' % (src, dest)
195    retv = self._ce.RunCommand(copy_cmd)
196    if retv != 0:
197      raise RuntimeError("Couldn't copy .boto file for google storage.")
198
199    # Fix protections on ssh key
200    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
201               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
202               '/testing_rsa')
203    retv = self._ce.ChrootRunCommand(self._chromeos_root, command)
204    if retv != 0:
205      raise RuntimeError('chmod for testing_rsa failed')
206
207  def _TestLabels(self, labels):
208    experiment_file = 'toolchain_experiment.txt'
209    image_args = ''
210    if self._force_mismatch:
211      image_args = '--force-mismatch'
212    experiment_header = """
213    board: %s
214    remote: %s
215    retries: 1
216    """ % (self._board, self._remotes)
217    experiment_tests = """
218    benchmark: all_toolchain_perf {
219      suite: telemetry_Crosperf
220      iterations: 3
221    }
222    """
223
224    with open(experiment_file, 'w') as f:
225      f.write(experiment_header)
226      f.write(experiment_tests)
227      for label in labels:
228        # TODO(asharif): Fix crosperf so it accepts labels with symbols
229        crosperf_label = label
230        crosperf_label = crosperf_label.replace('-', '_')
231        crosperf_label = crosperf_label.replace('+', '_')
232        crosperf_label = crosperf_label.replace('.', '')
233
234        # Use the official build instead of building vanilla ourselves.
235        if label == 'vanilla':
236          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
237
238          # Now add 'official build' to test file.
239          official_image = """
240          official_image {
241            chromeos_root: %s
242            build: %s
243          }
244          """ % (self._chromeos_root, build_name)
245          f.write(official_image)
246
247        else:
248          experiment_image = """
249          %s {
250            chromeos_image: %s
251            image_args: %s
252          }
253          """ % (crosperf_label, os.path.join(
254              misc.GetImageDir(self._chromeos_root, self._board), label,
255              'chromiumos_test_image.bin'), image_args)
256          f.write(experiment_image)
257
258    crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf')
259    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
260    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
261               '--json_report=True {noschedv2_opts} {exp_file}').format(
262                   crosperf=crosperf,
263                   r_dir=self._reports_dir,
264                   noschedv2_opts=noschedv2_opts,
265                   exp_file=experiment_file)
266
267    ret = self._ce.RunCommand(command)
268    if ret != 0:
269      raise RuntimeError('Crosperf execution error!')
270    else:
271      # Copy json report to pending archives directory.
272      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
273      ret = self._ce.RunCommand(command)
274    return
275
276  def _CopyWeeklyReportFiles(self, labels):
277    """Move files into place for creating 7-day reports.
278
279    Create tar files of the custom and official images and copy them
280    to the weekly reports directory, so they exist when the weekly report
281    gets generated.  IMPORTANT NOTE: This function must run *after*
282    crosperf has been run; otherwise the vanilla images will not be there.
283    """
284    images_path = os.path.join(
285        os.path.realpath(self._chromeos_root), 'src/build/images', self._board)
286    weekday = time.strftime('%a')
287    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
288    dest_dir = os.path.join(data_dir, weekday)
289    if not os.path.exists(dest_dir):
290      os.makedirs(dest_dir)
291    # Make sure dest_dir is empty (clean out last week's data).
292    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, weekday)
293    self._ce.RunCommand(cmd)
294    # Now create new tar files and copy them over.
295    for l in labels:
296      test_path = os.path.join(images_path, l)
297      if os.path.exists(test_path):
298        if l != 'vanilla':
299          label_name = 'test'
300        else:
301          label_name = 'vanilla'
302        tar_file_name = '%s_%s_image.tar' % (weekday, label_name)
303        cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
304               'cp %s %s/.') % (images_path, tar_file_name, l, tar_file_name,
305                                dest_dir)
306        tar_ret = self._ce.RunCommand(cmd)
307        if tar_ret != 0:
308          self._l.LogOutput('Error while creating/copying test tar file(%s).' %
309                            tar_file_name)
310
311  def _SendEmail(self):
312    """Find email msesage generated by crosperf and send it."""
313    filename = os.path.join(self._reports_dir, 'msg_body.html')
314    if (os.path.exists(filename) and
315        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
316      command = ('cat %s | %s -s "Nightly test results, %s" -team -html' %
317                 (filename, MAIL_PROGRAM, self._board))
318      self._ce.RunCommand(command)
319
320  def DoAll(self):
321    self._CheckoutChromeOS()
322    labels = []
323    labels.append('vanilla')
324    for config in self._configs:
325      label = self._BuildLabelName(config.gcc_config.githash)
326      if not misc.DoesLabelExist(self._chromeos_root, self._board, label):
327        self._BuildToolchain(config)
328        label = self._BuildAndImage(label)
329      labels.append(label)
330    self._FinishSetup()
331    self._TestLabels(labels)
332    self._SendEmail()
333    # Only try to copy the image files if the test runs ran successfully.
334    self._CopyWeeklyReportFiles(labels)
335    if self._clean:
336      ret = self._DeleteChroot()
337      if ret != 0:
338        return ret
339      ret = self._DeleteCcahe()
340      if ret != 0:
341        return ret
342    return 0
343
344
345def Main(argv):
346  """The main function."""
347  # Common initializations
348  ###  command_executer.InitCommandExecuter(True)
349  command_executer.InitCommandExecuter()
350  parser = argparse.ArgumentParser()
351  parser.add_argument(
352      '--remote', dest='remote', help='Remote machines to run tests on.')
353  parser.add_argument(
354      '--board', dest='board', default='x86-alex', help='The target board.')
355  parser.add_argument(
356      '--githashes',
357      dest='githashes',
358      default='master',
359      help='The gcc githashes to test.')
360  parser.add_argument(
361      '--clean',
362      dest='clean',
363      default=False,
364      action='store_true',
365      help='Clean the chroot after testing.')
366  parser.add_argument(
367      '--public',
368      dest='public',
369      default=False,
370      action='store_true',
371      help='Use the public checkout/build.')
372  parser.add_argument(
373      '--force-mismatch',
374      dest='force_mismatch',
375      default='',
376      help='Force the image regardless of board mismatch')
377  parser.add_argument(
378      '--noschedv2',
379      dest='noschedv2',
380      action='store_true',
381      default=False,
382      help='Pass --noschedv2 to crosperf.')
383  options = parser.parse_args(argv)
384  if not options.board:
385    print('Please give a board.')
386    return 1
387  if not options.remote:
388    print('Please give at least one remote machine.')
389    return 1
390  toolchain_configs = []
391  for githash in options.githashes.split(','):
392    gcc_config = GCCConfig(githash=githash)
393    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
394    toolchain_configs.append(toolchain_config)
395  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
396                           options.clean, options.public,
397                           options.force_mismatch, options.noschedv2)
398  return fc.DoAll()
399
400
401if __name__ == '__main__':
402  retval = Main(sys.argv[1:])
403  sys.exit(retval)
404