buildbot_test_toolchains.py revision e58198707a6524ad3f731c61c9d5ef140a342451 
1#!/usr/bin/env python2
2#
3# Copyright 2016 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Script for running nightly compiler tests on ChromeOS.
8
9This script launches a buildbot to build ChromeOS with the latest compiler on
10a particular board; then it finds and downloads the trybot image and the
11corresponding official image, and runs crosperf performance tests comparing
12the two.  It then generates a report, emails it to the c-compiler-chrome, as
13well as copying the images into the seven-day reports directory.
14"""
15
16# Script to test different toolchains against ChromeOS benchmarks.
17
18from __future__ import print_function
19
20import argparse
21import datetime
22import os
23import re
24import sys
25import time
26
27from cros_utils import command_executer
28from cros_utils import logger
29
30from cros_utils import buildbot_utils
31
32# CL that updated GCC ebuilds to use 'next_gcc'.
33USE_NEXT_GCC_PATCH = '230260'
34
35# CL that uses LLVM to build the peppy image.
36USE_LLVM_PATCH = '295217'
37
38CROSTC_ROOT = '/usr/local/google/crostc'
39ROLE_ACCOUNT = 'mobiletc-prebuild'
40TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
41MAIL_PROGRAM = '~/var/bin/mail-sheriff'
42PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
43NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
44
45IMAGE_DIR = '{board}-{image_type}'
46IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}'
47IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR
48TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
49PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
50IMAGE_RE_GROUPS = {
51    'board': r'(?P<board>\S+)',
52    'image_type': r'(?P<image_type>\S+)',
53    'chrome_version': r'(?P<chrome_version>R\d+)',
54    'tip': r'(?P<tip>\d+)',
55    'branch': r'(?P<branch>\d+)',
56    'branch_branch': r'(?P<branch_branch>\d+)',
57    'build_id': r'(?P<build_id>b\d+)'
58}
59TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
60
61
62class ToolchainComparator(object):
63  """Class for doing the nightly tests work."""
64
65  def __init__(self,
66               board,
67               remotes,
68               chromeos_root,
69               weekday,
70               patches,
71               noschedv2=False):
72    self._board = board
73    self._remotes = remotes
74    self._chromeos_root = chromeos_root
75    self._base_dir = os.getcwd()
76    self._ce = command_executer.GetCommandExecuter()
77    self._l = logger.GetLogger()
78    self._build = '%s-release' % board
79    self._patches = patches.split(',')
80    self._patches_string = '_'.join(str(p) for p in self._patches)
81    self._noschedv2 = noschedv2
82
83    if not weekday:
84      self._weekday = time.strftime('%a')
85    else:
86      self._weekday = weekday
87    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
88                                           '%Y-%m-%d_%H:%M:%S')
89    self._reports_dir = os.path.join(
90        NIGHTLY_TESTS_DIR,
91        '%s.%s' % (timestamp, board),)
92
93  def _GetVanillaImageName(self, trybot_image):
94    """Given a trybot artifact name, get latest vanilla image name.
95
96    Args:
97      trybot_image: artifact name such as
98          'trybot-daisy-release/R40-6394.0.0-b1389'
99
100    Returns:
101      Latest official image name, e.g. 'daisy-release/R57-9089.0.0'.
102    """
103    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
104    assert mo
105    dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict())
106    version = buildbot_utils.GetGSContent(self._chromeos_root,
107                                          dirname + '/LATEST-master')
108    return dirname + '/' + version
109
110  def _GetNonAFDOImageName(self, trybot_image):
111    """Given a trybot artifact name, get corresponding non-AFDO image name.
112
113    We get the non-AFDO image from the PFQ builders. This image
114    is not generated for all the boards and, the closest PFQ image
115    was the one build for the previous ChromeOS version (the chrome
116    used in the current version is the one validated in the previous
117    version).
118    The previous ChromeOS does not always exist either. So, we try
119    a couple of versions before.
120
121    Args:
122      trybot_image: artifact name such as
123          'trybot-daisy-release/R40-6394.0.0-b1389'
124
125    Returns:
126      Corresponding chrome PFQ image name, e.g.
127      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
128    """
129    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
130    assert mo
131    image_dict = mo.groupdict()
132    image_dict['image_type'] = 'chrome-pfq'
133    for _ in xrange(2):
134      image_dict['tip'] = str(int(image_dict['tip']) - 1)
135      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
136      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
137        return nonafdo_image
138    return ''
139
140  def _FinishSetup(self):
141    """Make sure testing_rsa file is properly set up."""
142    # Fix protections on ssh key
143    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
144               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
145               '/testing_rsa')
146    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
147    if ret_val != 0:
148      raise RuntimeError('chmod for testing_rsa failed')
149
150  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
151    """Create crosperf experiment file.
152
153    Given the names of the trybot, vanilla and non-AFDO images, create the
154    appropriate crosperf experiment file and launch crosperf on it.
155    """
156    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
157    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
158
159    compiler_string = 'gcc'
160    if USE_LLVM_PATCH in self._patches_string:
161      experiment_file_name = '%s_llvm_experiment.txt' % self._board
162      compiler_string = 'llvm'
163
164    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
165    experiment_header = """
166    board: %s
167    remote: %s
168    retries: 1
169    """ % (self._board, self._remotes)
170    experiment_tests = """
171    benchmark: all_toolchain_perf {
172      suite: telemetry_Crosperf
173      iterations: 3
174    }
175
176    benchmark: page_cycler_v2.typical_25 {
177      suite: telemetry_Crosperf
178      iterations: 2
179      run_local: False
180      retries: 0
181    }
182    """
183
184    with open(experiment_file, 'w') as f:
185      f.write(experiment_header)
186      f.write(experiment_tests)
187
188      # Now add vanilla to test file.
189      official_image = """
190          vanilla_image {
191            chromeos_root: %s
192            build: %s
193            compiler: gcc
194          }
195          """ % (self._chromeos_root, vanilla_image)
196      f.write(official_image)
197
198      # Now add non-AFDO image to test file.
199      if nonafdo_image:
200        official_nonafdo_image = """
201          nonafdo_image {
202            chromeos_root: %s
203            build: %s
204            compiler: gcc
205          }
206          """ % (self._chromeos_root, nonafdo_image)
207        f.write(official_nonafdo_image)
208
209      label_string = '%s_trybot_image' % compiler_string
210      if USE_NEXT_GCC_PATCH in self._patches:
211        label_string = 'gcc_next_trybot_image'
212
213      experiment_image = """
214          %s {
215            chromeos_root: %s
216            build: %s
217            compiler: %s
218          }
219          """ % (label_string, self._chromeos_root, trybot_image,
220                 compiler_string)
221      f.write(experiment_image)
222
223    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
224    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
225    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
226               '--json_report=True {noschedv2_opts} {exp_file}').format(
227                   crosperf=crosperf,
228                   r_dir=self._reports_dir,
229                   noschedv2_opts=noschedv2_opts,
230                   exp_file=experiment_file)
231
232    ret = self._ce.RunCommand(command)
233    if ret != 0:
234      raise RuntimeError('Crosperf execution error!')
235    else:
236      # Copy json report to pending archives directory.
237      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
238      ret = self._ce.RunCommand(command)
239    return
240
241  def _SendEmail(self):
242    """Find email message generated by crosperf and send it."""
243    filename = os.path.join(self._reports_dir, 'msg_body.html')
244    if (os.path.exists(filename) and
245        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
246      email_title = 'buildbot test results'
247      if self._patches_string == USE_LLVM_PATCH:
248        email_title = 'buildbot llvm test results'
249      command = ('cat %s | %s -s "%s, %s" -team -html' %
250                 (filename, MAIL_PROGRAM, email_title, self._board))
251      self._ce.RunCommand(command)
252
253  def DoAll(self):
254    """Main function inside ToolchainComparator class.
255
256    Launch trybot, get image names, create crosperf experiment file, run
257    crosperf, and copy images into seven-day report directories.
258    """
259    date_str = datetime.date.today()
260    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
261                                       date_str)
262    build_id, trybot_image = buildbot_utils.GetTrybotImage(
263        self._chromeos_root,
264        self._build,
265        self._patches,
266        description,
267        other_flags=['--notests'],
268        build_toolchain=True)
269
270    print('trybot_url: \
271       https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s' \
272       % build_id)
273    if len(trybot_image) == 0:
274      self._l.LogError('Unable to find trybot_image for %s!' % description)
275      return 1
276
277    vanilla_image = self._GetVanillaImageName(trybot_image)
278    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
279
280    print('trybot_image: %s' % trybot_image)
281    print('vanilla_image: %s' % vanilla_image)
282    print('nonafdo_image: %s' % nonafdo_image)
283
284    if os.getlogin() == ROLE_ACCOUNT:
285      self._FinishSetup()
286
287    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
288    self._SendEmail()
289    return 0
290
291
292def Main(argv):
293  """The main function."""
294
295  # Common initializations
296  command_executer.InitCommandExecuter()
297  parser = argparse.ArgumentParser()
298  parser.add_argument(
299      '--remote', dest='remote', help='Remote machines to run tests on.')
300  parser.add_argument(
301      '--board', dest='board', default='x86-zgb', help='The target board.')
302  parser.add_argument(
303      '--chromeos_root',
304      dest='chromeos_root',
305      help='The chromeos root from which to run tests.')
306  parser.add_argument(
307      '--weekday',
308      default='',
309      dest='weekday',
310      help='The day of the week for which to run tests.')
311  parser.add_argument(
312      '--patch',
313      dest='patches',
314      help='The patches to use for the testing, '
315      "seprate the patch numbers with ',' "
316      'for more than one patches.')
317  parser.add_argument(
318      '--noschedv2',
319      dest='noschedv2',
320      action='store_true',
321      default=False,
322      help='Pass --noschedv2 to crosperf.')
323
324  options = parser.parse_args(argv[1:])
325  if not options.board:
326    print('Please give a board.')
327    return 1
328  if not options.remote:
329    print('Please give at least one remote machine.')
330    return 1
331  if not options.chromeos_root:
332    print('Please specify the ChromeOS root directory.')
333    return 1
334  if options.patches:
335    patches = options.patches
336  else:
337    patches = USE_NEXT_GCC_PATCH
338
339  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
340                           options.weekday, patches, options.noschedv2)
341  return fc.DoAll()
342
343
344if __name__ == '__main__':
345  retval = Main(sys.argv)
346  sys.exit(retval)
347