buildbot_test_toolchains.py revision e84ea3d3e9fbe92d9854a85d67ab01fb488781c7 
1#!/usr/bin/env python2
2"""Script for running nightly compiler tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the latest compiler on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs crosperf performance tests comparing
7the two.  It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the images into the seven-day reports directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import re
19import sys
20import time
21
22from cros_utils import command_executer
23from cros_utils import logger
24
25from cros_utils import buildbot_utils
26
27# CL that updated GCC ebuilds to use 'next_gcc'.
28USE_NEXT_GCC_PATCH = '230260'
29
30# CL that uses LLVM to build the peppy image.
31USE_LLVM_PATCH = '295217'
32
33CROSTC_ROOT = '/usr/local/google/crostc'
34ROLE_ACCOUNT = 'mobiletc-prebuild'
35TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
36MAIL_PROGRAM = '~/var/bin/mail-sheriff'
37PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
38NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
39
40IMAGE_FS = (r'{board}-{image_type}/{chrome_version}-{tip}\.' +
41            r'{branch}\.{branch_branch}')
42TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
43PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
44IMAGE_RE_GROUPS = {
45    'board': r'(?P<board>\S+)',
46    'image_type': r'(?P<image_type>\S+)',
47    'chrome_version': r'(?P<chrome_version>R\d+)',
48    'tip': r'(?P<tip>\d+)',
49    'branch': r'(?P<branch>\d+)',
50    'branch_branch': r'(?P<branch_branch>\d+)',
51    'build_id': r'(?P<build_id>b\d+)'
52}
53TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
54
55
56class ToolchainComparator(object):
57  """Class for doing the nightly tests work."""
58
59  def __init__(self,
60               board,
61               remotes,
62               chromeos_root,
63               weekday,
64               patches,
65               noschedv2=False):
66    self._board = board
67    self._remotes = remotes
68    self._chromeos_root = chromeos_root
69    self._base_dir = os.getcwd()
70    self._ce = command_executer.GetCommandExecuter()
71    self._l = logger.GetLogger()
72    self._build = '%s-release' % board
73    self._patches = patches.split(',')
74    self._patches_string = '_'.join(str(p) for p in self._patches)
75    self._noschedv2 = noschedv2
76
77    if not weekday:
78      self._weekday = time.strftime('%a')
79    else:
80      self._weekday = weekday
81    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
82                                           '%Y-%m-%d_%H:%M:%S')
83    self._reports_dir = os.path.join(
84        NIGHTLY_TESTS_DIR,
85        '%s.%s' % (timestamp, board),)
86
87  def _GetVanillaImageName(self, trybot_image):
88    """Given a trybot artifact name, get corresponding vanilla image name.
89
90    Args:
91      trybot_image: artifact name such as
92          'trybot-daisy-release/R40-6394.0.0-b1389'
93
94    Returns:
95      Corresponding official image name, e.g. 'daisy-release/R40-6394.0.0'.
96    """
97    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
98    assert mo
99    return IMAGE_FS.replace('\\', '').format(**mo.groupdict())
100
101  def _GetNonAFDOImageName(self, trybot_image):
102    """Given a trybot artifact name, get corresponding non-AFDO image name.
103
104    We get the non-AFDO image from the PFQ builders. This image
105    is not generated for all the boards and, the closest PFQ image
106    was the one build for the previous ChromeOS version (the chrome
107    used in the current version is the one validated in the previous
108    version).
109    The previous ChromeOS does not always exist either. So, we try
110    a couple of versions before.
111
112    Args:
113      trybot_image: artifact name such as
114          'trybot-daisy-release/R40-6394.0.0-b1389'
115
116    Returns:
117      Corresponding chrome PFQ image name, e.g.
118      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
119    """
120    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
121    assert mo
122    image_dict = mo.groupdict()
123    image_dict['image_type'] = 'chrome-pfq'
124    for _ in xrange(2):
125      image_dict['tip'] = str(int(image_dict['tip']) - 1)
126      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
127      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
128        return nonafdo_image
129    return ''
130
131  def _FinishSetup(self):
132    """Make sure testing_rsa file is properly set up."""
133    # Fix protections on ssh key
134    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
135               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
136               '/testing_rsa')
137    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
138    if ret_val != 0:
139      raise RuntimeError('chmod for testing_rsa failed')
140
141  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
142    """Create crosperf experiment file.
143
144    Given the names of the trybot, vanilla and non-AFDO images, create the
145    appropriate crosperf experiment file and launch crosperf on it.
146    """
147    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
148    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
149
150    compiler_string = 'gcc'
151    if USE_LLVM_PATCH in self._patches_string:
152      experiment_file_name = '%s_llvm_experiment.txt' % self._board
153      compiler_string = 'llvm'
154
155    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
156    experiment_header = """
157    board: %s
158    remote: %s
159    retries: 1
160    """ % (self._board, self._remotes)
161    experiment_tests = """
162    benchmark: all_toolchain_perf {
163      suite: telemetry_Crosperf
164      iterations: 3
165    }
166
167    benchmark: page_cycler_v2.typical_25 {
168      suite: telemetry_Crosperf
169      iterations: 2
170      run_local: False
171      retries: 0
172    }
173    """
174
175    with open(experiment_file, 'w') as f:
176      f.write(experiment_header)
177      f.write(experiment_tests)
178
179      # Now add vanilla to test file.
180      official_image = """
181          vanilla_image {
182            chromeos_root: %s
183            build: %s
184            compiler: gcc
185          }
186          """ % (self._chromeos_root, vanilla_image)
187      f.write(official_image)
188
189      # Now add non-AFDO image to test file.
190      if nonafdo_image:
191        official_nonafdo_image = """
192          nonafdo_image {
193            chromeos_root: %s
194            build: %s
195            compiler: gcc
196          }
197          """ % (self._chromeos_root, nonafdo_image)
198        f.write(official_nonafdo_image)
199
200      label_string = '%s_trybot_image' % compiler_string
201      if USE_NEXT_GCC_PATCH in self._patches:
202        label_string = 'gcc_next_trybot_image'
203
204      experiment_image = """
205          %s {
206            chromeos_root: %s
207            build: %s
208            compiler: %s
209          }
210          """ % (label_string, self._chromeos_root, trybot_image,
211                 compiler_string)
212      f.write(experiment_image)
213
214    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
215    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
216    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
217               '--json_report=True {noschedv2_opts} {exp_file}').format(
218                   crosperf=crosperf,
219                   r_dir=self._reports_dir,
220                   noschedv2_opts=noschedv2_opts,
221                   exp_file=experiment_file)
222
223    ret = self._ce.RunCommand(command)
224    if ret != 0:
225      raise RuntimeError('Crosperf execution error!')
226    else:
227      # Copy json report to pending archives directory.
228      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
229      ret = self._ce.RunCommand(command)
230    return
231
232  def _SendEmail(self):
233    """Find email message generated by crosperf and send it."""
234    filename = os.path.join(self._reports_dir, 'msg_body.html')
235    if (os.path.exists(filename) and
236        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
237      email_title = 'buildbot test results'
238      if self._patches_string == USE_LLVM_PATCH:
239        email_title = 'buildbot llvm test results'
240      command = ('cat %s | %s -s "%s, %s" -team -html' %
241                 (filename, MAIL_PROGRAM, email_title, self._board))
242      self._ce.RunCommand(command)
243
244  def DoAll(self):
245    """Main function inside ToolchainComparator class.
246
247    Launch trybot, get image names, create crosperf experiment file, run
248    crosperf, and copy images into seven-day report directories.
249    """
250    date_str = datetime.date.today()
251    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
252                                       date_str)
253    build_id, trybot_image = buildbot_utils.GetTrybotImage(
254        self._chromeos_root,
255        self._build,
256        self._patches,
257        description,
258        other_flags=['--notests'],
259        build_toolchain=True)
260
261    print('trybot_url: \
262       https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s' \
263       % build_id)
264    if len(trybot_image) == 0:
265      self._l.LogError('Unable to find trybot_image for %s!' % description)
266      return 1
267
268    vanilla_image = self._GetVanillaImageName(trybot_image)
269    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
270
271    # The trybot image is ready here, in some cases, the vanilla image
272    # is not ready, so we need to make sure vanilla image is available.
273    buildbot_utils.WaitForImage(self._chromeos_root, vanilla_image)
274    print('trybot_image: %s' % trybot_image)
275    print('vanilla_image: %s' % vanilla_image)
276    print('nonafdo_image: %s' % nonafdo_image)
277
278    if os.getlogin() == ROLE_ACCOUNT:
279      self._FinishSetup()
280
281    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
282    self._SendEmail()
283    return 0
284
285
286def Main(argv):
287  """The main function."""
288
289  # Common initializations
290  command_executer.InitCommandExecuter()
291  parser = argparse.ArgumentParser()
292  parser.add_argument(
293      '--remote', dest='remote', help='Remote machines to run tests on.')
294  parser.add_argument(
295      '--board', dest='board', default='x86-zgb', help='The target board.')
296  parser.add_argument(
297      '--chromeos_root',
298      dest='chromeos_root',
299      help='The chromeos root from which to run tests.')
300  parser.add_argument(
301      '--weekday',
302      default='',
303      dest='weekday',
304      help='The day of the week for which to run tests.')
305  parser.add_argument(
306      '--patch',
307      dest='patches',
308      help='The patches to use for the testing, '
309      "seprate the patch numbers with ',' "
310      'for more than one patches.')
311  parser.add_argument(
312      '--noschedv2',
313      dest='noschedv2',
314      action='store_true',
315      default=False,
316      help='Pass --noschedv2 to crosperf.')
317
318  options = parser.parse_args(argv[1:])
319  if not options.board:
320    print('Please give a board.')
321    return 1
322  if not options.remote:
323    print('Please give at least one remote machine.')
324    return 1
325  if not options.chromeos_root:
326    print('Please specify the ChromeOS root directory.')
327    return 1
328  if options.patches:
329    patches = options.patches
330  else:
331    patches = USE_NEXT_GCC_PATCH
332
333  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
334                           options.weekday, patches, options.noschedv2)
335  return fc.DoAll()
336
337
338if __name__ == '__main__':
339  retval = Main(sys.argv)
340  sys.exit(retval)
341