buildbot_test_toolchains.py revision 0ded515f8f340188634bd8d14e3184b98e4f06de 
1#!/usr/bin/python2
2"""Script for running nightly compiler tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the latest compiler on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs crosperf performance tests comparing
7the two.  It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the images into the seven-day reports directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import re
19import sys
20import time
21
22from cros_utils import command_executer
23from cros_utils import logger
24
25from cros_utils import buildbot_utils
26
27# CL that updated GCC ebuilds to use 'next_gcc'.
28USE_NEXT_GCC_PATCH = '230260'
29
30# CL that uses LLVM to build the peppy image.
31USE_LLVM_PATCH = '295217'
32
33# The boards on which we run weekly reports
34WEEKLY_REPORT_BOARDS = ['lumpy']
35
36CROSTC_ROOT = '/usr/local/google/crostc'
37ROLE_ACCOUNT = 'mobiletc-prebuild'
38TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
39MAIL_PROGRAM = '~/var/bin/mail-sheriff'
40WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
41PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
42NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
43
44IMAGE_FS = (r'{board}-{image_type}/{chrome_version}-{tip}\.' +
45            r'{branch}\.{branch_branch}')
46TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
47PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
48IMAGE_RE_GROUPS = {'board': r'(?P<board>\S+)',
49                   'image_type': r'(?P<image_type>\S+)',
50                   'chrome_version': r'(?P<chrome_version>R\d+)',
51                   'tip': r'(?P<tip>\d+)',
52                   'branch': r'(?P<branch>\d+)',
53                   'branch_branch': r'(?P<branch_branch>\d+)',
54                   'build_id': r'(?P<build_id>b\d+)'}
55TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
56
57
58class ToolchainComparator(object):
59  """Class for doing the nightly tests work."""
60
61  def __init__(self,
62               board,
63               remotes,
64               chromeos_root,
65               weekday,
66               patches,
67               noschedv2=False):
68    self._board = board
69    self._remotes = remotes
70    self._chromeos_root = chromeos_root
71    self._base_dir = os.getcwd()
72    self._ce = command_executer.GetCommandExecuter()
73    self._l = logger.GetLogger()
74    self._build = '%s-release' % board
75    self._patches = patches.split(',')
76    self._patches_string = '_'.join(str(p) for p in self._patches)
77    self._noschedv2 = noschedv2
78
79    if not weekday:
80      self._weekday = time.strftime('%a')
81    else:
82      self._weekday = weekday
83    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
84                                           '%Y-%m-%d_%H:%M:%S')
85    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
86                                     '%s.%s' % (timestamp, board),)
87
88  def _GetVanillaImageName(self, trybot_image):
89    """Given a trybot artifact name, get corresponding vanilla image name.
90
91    Args:
92      trybot_image: artifact name such as
93          'trybot-daisy-release/R40-6394.0.0-b1389'
94
95    Returns:
96      Corresponding official image name, e.g. 'daisy-release/R40-6394.0.0'.
97    """
98    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
99    assert mo
100    return IMAGE_FS.replace('\\', '').format(**mo.groupdict())
101
102  def _GetNonAFDOImageName(self, trybot_image):
103    """Given a trybot artifact name, get corresponding non-AFDO image name.
104
105    We get the non-AFDO image from the PFQ builders. This image
106    is not generated for all the boards and, the closest PFQ image
107    was the one build for the previous ChromeOS version (the chrome
108    used in the current version is the one validated in the previous
109    version).
110    The previous ChromeOS does not always exist either. So, we try
111    a couple of versions before.
112
113    Args:
114      trybot_image: artifact name such as
115          'trybot-daisy-release/R40-6394.0.0-b1389'
116
117    Returns:
118      Corresponding chrome PFQ image name, e.g.
119      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
120    """
121    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
122    assert mo
123    image_dict = mo.groupdict()
124    image_dict['image_type'] = 'chrome-pfq'
125    for _ in xrange(2):
126      image_dict['tip'] = str(int(image_dict['tip']) - 1)
127      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
128      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
129        return nonafdo_image
130    return ''
131
132  def _FinishSetup(self):
133    """Make sure testing_rsa file is properly set up."""
134    # Fix protections on ssh key
135    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
136               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
137               '/testing_rsa')
138    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
139    if ret_val != 0:
140      raise RuntimeError('chmod for testing_rsa failed')
141
142  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
143    """Create crosperf experiment file.
144
145    Given the names of the trybot, vanilla and non-AFDO images, create the
146    appropriate crosperf experiment file and launch crosperf on it.
147    """
148    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
149    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
150
151    compiler_string = 'gcc'
152    if USE_LLVM_PATCH in self._patches_string:
153      experiment_file_name = '%s_llvm_experiment.txt' % self._board
154      compiler_string = 'llvm'
155
156    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
157    experiment_header = """
158    board: %s
159    remote: %s
160    retries: 1
161    """ % (self._board, self._remotes)
162    experiment_tests = """
163    benchmark: all_toolchain_perf {
164      suite: telemetry_Crosperf
165      iterations: 3
166    }
167    """
168
169    with open(experiment_file, 'w') as f:
170      f.write(experiment_header)
171      f.write(experiment_tests)
172
173      # Now add vanilla to test file.
174      official_image = """
175          vanilla_image {
176            chromeos_root: %s
177            build: %s
178            compiler: gcc
179          }
180          """ % (self._chromeos_root, vanilla_image)
181      f.write(official_image)
182
183      # Now add non-AFDO image to test file.
184      if nonafdo_image:
185        official_nonafdo_image = """
186          nonafdo_image {
187            chromeos_root: %s
188            build: %s
189            compiler: gcc
190          }
191          """ % (self._chromeos_root, nonafdo_image)
192        f.write(official_nonafdo_image)
193
194      label_string = '%s_trybot_image' % compiler_string
195      if USE_NEXT_GCC_PATCH in self._patches:
196        label_string = 'gcc_next_trybot_image'
197
198      experiment_image = """
199          %s {
200            chromeos_root: %s
201            build: %s
202            compiler: %s
203          }
204          """ % (label_string, self._chromeos_root, trybot_image,
205                 compiler_string)
206      f.write(experiment_image)
207
208    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
209    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
210    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
211               '--json_report=True {noschedv2_opts} {exp_file}').format(
212                   crosperf=crosperf,
213                   r_dir=self._reports_dir,
214                   noschedv2_opts=noschedv2_opts,
215                   exp_file=experiment_file)
216
217    ret = self._ce.RunCommand(command)
218    if ret != 0:
219      raise RuntimeError("Couldn't run crosperf!")
220    else:
221      # Copy json report to pending archives directory.
222      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
223      ret = self._ce.RunCommand(command)
224    return
225
226  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image, nonafdo_image):
227    """Put files in place for running seven-day reports.
228
229    Create tar files of the custom and official images and copy them
230    to the weekly reports directory, so they exist when the weekly report
231    gets generated.  IMPORTANT NOTE: This function must run *after*
232    crosperf has been run; otherwise the vanilla images will not be there.
233    """
234
235    dry_run = False
236    if os.getlogin() != ROLE_ACCOUNT:
237      self._l.LogOutput('Running this from non-role account; not copying '
238                        'tar files for weekly reports.')
239      dry_run = True
240
241    images_path = os.path.join(
242        os.path.realpath(self._chromeos_root), 'chroot/tmp')
243
244    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
245    dest_dir = os.path.join(data_dir, self._weekday)
246    if not os.path.exists(dest_dir):
247      os.makedirs(dest_dir)
248
249    # Make sure dest_dir is empty (clean out last week's data).
250    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday)
251    if dry_run:
252      print('CMD: %s' % cmd)
253    else:
254      self._ce.RunCommand(cmd)
255
256    # Now create new tar files and copy them over.
257    labels = {'test': trybot_image, 'vanilla': vanilla_image}
258    if nonafdo_image:
259      labels['nonafdo'] = nonafdo_image
260    for label_name, test_path in labels.iteritems():
261      tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name)
262      cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
263             'cp %s %s/.') % (images_path, tar_file_name, test_path,
264                              tar_file_name, dest_dir)
265      if dry_run:
266        print('CMD: %s' % cmd)
267        tar_ret = 0
268      else:
269        tar_ret = self._ce.RunCommand(cmd)
270      if tar_ret:
271        self._l.LogOutput('Error while creating/copying test tar file(%s).' %
272                          tar_file_name)
273
274  def _SendEmail(self):
275    """Find email message generated by crosperf and send it."""
276    filename = os.path.join(self._reports_dir, 'msg_body.html')
277    if (os.path.exists(filename) and
278        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
279      email_title = 'buildbot test results'
280      if self._patches_string == USE_LLVM_PATCH:
281        email_title = 'buildbot llvm test results'
282      command = ('cat %s | %s -s "%s, %s" -team -html' %
283                 (filename, MAIL_PROGRAM, email_title, self._board))
284      self._ce.RunCommand(command)
285
286  def DoAll(self):
287    """Main function inside ToolchainComparator class.
288
289    Launch trybot, get image names, create crosperf experiment file, run
290    crosperf, and copy images into seven-day report directories.
291    """
292    date_str = datetime.date.today()
293    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
294                                       date_str)
295    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
296                                                 self._build,
297                                                 self._patches,
298                                                 description,
299                                                 other_flags=['--notests'],
300                                                 build_toolchain=True)
301
302    if len(trybot_image) == 0:
303      self._l.LogError('Unable to find trybot_image for %s!' % description)
304      return 1
305
306    vanilla_image = self._GetVanillaImageName(trybot_image)
307    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
308
309    # The trybot image is ready here, in some cases, the vanilla image
310    # is not ready, so we need to make sure vanilla image is available.
311    buildbot_utils.WaitForImage(self._chromeos_root, vanilla_image)
312    print('trybot_image: %s' % trybot_image)
313    print('vanilla_image: %s' % vanilla_image)
314    print('nonafdo_image: %s' % nonafdo_image)
315
316    if os.getlogin() == ROLE_ACCOUNT:
317      self._FinishSetup()
318
319    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
320    self._SendEmail()
321    if (self._patches_string == USE_NEXT_GCC_PATCH and
322        self._board in WEEKLY_REPORT_BOARDS):
323      # Only try to copy the image files if the test runs ran successfully.
324      self._CopyWeeklyReportFiles(trybot_image, vanilla_image, nonafdo_image)
325    return 0
326
327
328def Main(argv):
329  """The main function."""
330
331  # Common initializations
332  command_executer.InitCommandExecuter()
333  parser = argparse.ArgumentParser()
334  parser.add_argument('--remote',
335                      dest='remote',
336                      help='Remote machines to run tests on.')
337  parser.add_argument('--board',
338                      dest='board',
339                      default='x86-zgb',
340                      help='The target board.')
341  parser.add_argument('--chromeos_root',
342                      dest='chromeos_root',
343                      help='The chromeos root from which to run tests.')
344  parser.add_argument('--weekday',
345                      default='',
346                      dest='weekday',
347                      help='The day of the week for which to run tests.')
348  parser.add_argument('--patch',
349                      dest='patches',
350                      help='The patches to use for the testing, '
351                      "seprate the patch numbers with ',' "
352                      'for more than one patches.')
353  parser.add_argument('--noschedv2',
354                      dest='noschedv2',
355                      action='store_true',
356                      default=False,
357                      help='Pass --noschedv2 to crosperf.')
358
359  options = parser.parse_args(argv[1:])
360  if not options.board:
361    print('Please give a board.')
362    return 1
363  if not options.remote:
364    print('Please give at least one remote machine.')
365    return 1
366  if not options.chromeos_root:
367    print('Please specify the ChromeOS root directory.')
368    return 1
369  if options.patches:
370    patches = options.patches
371  else:
372    patches = USE_NEXT_GCC_PATCH
373
374  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
375                           options.weekday, patches, options.noschedv2)
376  return fc.DoAll()
377
378
379if __name__ == '__main__':
380  retval = Main(sys.argv)
381  sys.exit(retval)
382