buildbot_test_toolchains.py revision aee96b71ad4ffde231ca5bf9c0509d15b0e9b753 
1#!/usr/bin/python2
2"""Script for running nightly compiler tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the latest compiler on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs crosperf performance tests comparing
7the two.  It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the images into the seven-day reports directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import re
19import sys
20import time
21
22from cros_utils import command_executer
23from cros_utils import logger
24
25from cros_utils import buildbot_utils
26
27# CL that updated GCC ebuilds to use 'next_gcc'.
28USE_NEXT_GCC_PATCH = '230260'
29
30# CL that uses LLVM to build the peppy image.
31USE_LLVM_PATCH = '295217'
32
33# The boards on which we run weekly reports
34WEEKLY_REPORT_BOARDS = ['lumpy']
35
36CROSTC_ROOT = '/usr/local/google/crostc'
37ROLE_ACCOUNT = 'mobiletc-prebuild'
38TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
39MAIL_PROGRAM = '~/var/bin/mail-sheriff'
40WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
41PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
42NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
43
44IMAGE_FS = (r'{board}-{image_type}/{chrome_version}-{tip}\.' +
45            r'{branch}\.{branch_branch}')
46TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
47PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
48IMAGE_RE_GROUPS = {
49    'board': r'(?P<board>\S+)',
50    'image_type': r'(?P<image_type>\S+)',
51    'chrome_version': r'(?P<chrome_version>R\d+)',
52    'tip': r'(?P<tip>\d+)',
53    'branch': r'(?P<branch>\d+)',
54    'branch_branch': r'(?P<branch_branch>\d+)',
55    'build_id': r'(?P<build_id>b\d+)'
56}
57TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
58
59
60class ToolchainComparator(object):
61  """Class for doing the nightly tests work."""
62
63  def __init__(self,
64               board,
65               remotes,
66               chromeos_root,
67               weekday,
68               patches,
69               noschedv2=False):
70    self._board = board
71    self._remotes = remotes
72    self._chromeos_root = chromeos_root
73    self._base_dir = os.getcwd()
74    self._ce = command_executer.GetCommandExecuter()
75    self._l = logger.GetLogger()
76    self._build = '%s-release' % board
77    self._patches = patches.split(',')
78    self._patches_string = '_'.join(str(p) for p in self._patches)
79    self._noschedv2 = noschedv2
80
81    if not weekday:
82      self._weekday = time.strftime('%a')
83    else:
84      self._weekday = weekday
85    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
86                                           '%Y-%m-%d_%H:%M:%S')
87    self._reports_dir = os.path.join(
88        NIGHTLY_TESTS_DIR,
89        '%s.%s' % (timestamp, board),)
90
91  def _GetVanillaImageName(self, trybot_image):
92    """Given a trybot artifact name, get corresponding vanilla image name.
93
94    Args:
95      trybot_image: artifact name such as
96          'trybot-daisy-release/R40-6394.0.0-b1389'
97
98    Returns:
99      Corresponding official image name, e.g. 'daisy-release/R40-6394.0.0'.
100    """
101    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
102    assert mo
103    return IMAGE_FS.replace('\\', '').format(**mo.groupdict())
104
105  def _GetNonAFDOImageName(self, trybot_image):
106    """Given a trybot artifact name, get corresponding non-AFDO image name.
107
108    We get the non-AFDO image from the PFQ builders. This image
109    is not generated for all the boards and, the closest PFQ image
110    was the one build for the previous ChromeOS version (the chrome
111    used in the current version is the one validated in the previous
112    version).
113    The previous ChromeOS does not always exist either. So, we try
114    a couple of versions before.
115
116    Args:
117      trybot_image: artifact name such as
118          'trybot-daisy-release/R40-6394.0.0-b1389'
119
120    Returns:
121      Corresponding chrome PFQ image name, e.g.
122      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
123    """
124    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
125    assert mo
126    image_dict = mo.groupdict()
127    image_dict['image_type'] = 'chrome-pfq'
128    for _ in xrange(2):
129      image_dict['tip'] = str(int(image_dict['tip']) - 1)
130      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
131      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
132        return nonafdo_image
133    return ''
134
135  def _FinishSetup(self):
136    """Make sure testing_rsa file is properly set up."""
137    # Fix protections on ssh key
138    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
139               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
140               '/testing_rsa')
141    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
142    if ret_val != 0:
143      raise RuntimeError('chmod for testing_rsa failed')
144
145  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
146    """Create crosperf experiment file.
147
148    Given the names of the trybot, vanilla and non-AFDO images, create the
149    appropriate crosperf experiment file and launch crosperf on it.
150    """
151    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
152    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
153
154    compiler_string = 'gcc'
155    if USE_LLVM_PATCH in self._patches_string:
156      experiment_file_name = '%s_llvm_experiment.txt' % self._board
157      compiler_string = 'llvm'
158
159    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
160    experiment_header = """
161    board: %s
162    remote: %s
163    retries: 1
164    """ % (self._board, self._remotes)
165    experiment_tests = """
166    benchmark: all_toolchain_perf {
167      suite: telemetry_Crosperf
168      iterations: 3
169    }
170    """
171
172    with open(experiment_file, 'w') as f:
173      f.write(experiment_header)
174      f.write(experiment_tests)
175
176      # Now add vanilla to test file.
177      official_image = """
178          vanilla_image {
179            chromeos_root: %s
180            build: %s
181            compiler: gcc
182          }
183          """ % (self._chromeos_root, vanilla_image)
184      f.write(official_image)
185
186      # Now add non-AFDO image to test file.
187      if nonafdo_image:
188        official_nonafdo_image = """
189          nonafdo_image {
190            chromeos_root: %s
191            build: %s
192            compiler: gcc
193          }
194          """ % (self._chromeos_root, nonafdo_image)
195        f.write(official_nonafdo_image)
196
197      label_string = '%s_trybot_image' % compiler_string
198      if USE_NEXT_GCC_PATCH in self._patches:
199        label_string = 'gcc_next_trybot_image'
200
201      experiment_image = """
202          %s {
203            chromeos_root: %s
204            build: %s
205            compiler: %s
206          }
207          """ % (label_string, self._chromeos_root, trybot_image,
208                 compiler_string)
209      f.write(experiment_image)
210
211    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
212    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
213    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
214               '--json_report=True {noschedv2_opts} {exp_file}').format(
215                   crosperf=crosperf,
216                   r_dir=self._reports_dir,
217                   noschedv2_opts=noschedv2_opts,
218                   exp_file=experiment_file)
219
220    ret = self._ce.RunCommand(command)
221    if ret != 0:
222      raise RuntimeError('Crosperf execution error!')
223    else:
224      # Copy json report to pending archives directory.
225      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
226      ret = self._ce.RunCommand(command)
227    return
228
229  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image, nonafdo_image):
230    """Put files in place for running seven-day reports.
231
232    Create tar files of the custom and official images and copy them
233    to the weekly reports directory, so they exist when the weekly report
234    gets generated.  IMPORTANT NOTE: This function must run *after*
235    crosperf has been run; otherwise the vanilla images will not be there.
236    """
237
238    dry_run = False
239    if os.getlogin() != ROLE_ACCOUNT:
240      self._l.LogOutput('Running this from non-role account; not copying '
241                        'tar files for weekly reports.')
242      dry_run = True
243
244    images_path = os.path.join(
245        os.path.realpath(self._chromeos_root), 'chroot/tmp')
246
247    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
248    dest_dir = os.path.join(data_dir, self._weekday)
249    if not os.path.exists(dest_dir):
250      os.makedirs(dest_dir)
251
252    # Make sure dest_dir is empty (clean out last week's data).
253    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday)
254    if dry_run:
255      print('CMD: %s' % cmd)
256    else:
257      self._ce.RunCommand(cmd)
258
259    # Now create new tar files and copy them over.
260    labels = {'test': trybot_image, 'vanilla': vanilla_image}
261    if nonafdo_image:
262      labels['nonafdo'] = nonafdo_image
263    for label_name, test_path in labels.iteritems():
264      tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name)
265      cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
266             'cp %s %s/.') % (images_path, tar_file_name, test_path,
267                              tar_file_name, dest_dir)
268      if dry_run:
269        print('CMD: %s' % cmd)
270        tar_ret = 0
271      else:
272        tar_ret = self._ce.RunCommand(cmd)
273      if tar_ret:
274        self._l.LogOutput('Error while creating/copying test tar file(%s).' %
275                          tar_file_name)
276
277  def _SendEmail(self):
278    """Find email message generated by crosperf and send it."""
279    filename = os.path.join(self._reports_dir, 'msg_body.html')
280    if (os.path.exists(filename) and
281        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
282      email_title = 'buildbot test results'
283      if self._patches_string == USE_LLVM_PATCH:
284        email_title = 'buildbot llvm test results'
285      command = ('cat %s | %s -s "%s, %s" -team -html' %
286                 (filename, MAIL_PROGRAM, email_title, self._board))
287      self._ce.RunCommand(command)
288
289  def DoAll(self):
290    """Main function inside ToolchainComparator class.
291
292    Launch trybot, get image names, create crosperf experiment file, run
293    crosperf, and copy images into seven-day report directories.
294    """
295    date_str = datetime.date.today()
296    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
297                                       date_str)
298    trybot_image = buildbot_utils.GetTrybotImage(
299        self._chromeos_root,
300        self._build,
301        self._patches,
302        description,
303        other_flags=['--notests'],
304        build_toolchain=True)
305
306    if len(trybot_image) == 0:
307      self._l.LogError('Unable to find trybot_image for %s!' % description)
308      return 1
309
310    vanilla_image = self._GetVanillaImageName(trybot_image)
311    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
312
313    # The trybot image is ready here, in some cases, the vanilla image
314    # is not ready, so we need to make sure vanilla image is available.
315    buildbot_utils.WaitForImage(self._chromeos_root, vanilla_image)
316    print('trybot_image: %s' % trybot_image)
317    print('vanilla_image: %s' % vanilla_image)
318    print('nonafdo_image: %s' % nonafdo_image)
319
320    if os.getlogin() == ROLE_ACCOUNT:
321      self._FinishSetup()
322
323    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
324    self._SendEmail()
325    if (self._patches_string == USE_NEXT_GCC_PATCH and
326        self._board in WEEKLY_REPORT_BOARDS):
327      # Only try to copy the image files if the test runs ran successfully.
328      self._CopyWeeklyReportFiles(trybot_image, vanilla_image, nonafdo_image)
329    return 0
330
331
332def Main(argv):
333  """The main function."""
334
335  # Common initializations
336  command_executer.InitCommandExecuter()
337  parser = argparse.ArgumentParser()
338  parser.add_argument(
339      '--remote', dest='remote', help='Remote machines to run tests on.')
340  parser.add_argument(
341      '--board', dest='board', default='x86-zgb', help='The target board.')
342  parser.add_argument(
343      '--chromeos_root',
344      dest='chromeos_root',
345      help='The chromeos root from which to run tests.')
346  parser.add_argument(
347      '--weekday',
348      default='',
349      dest='weekday',
350      help='The day of the week for which to run tests.')
351  parser.add_argument(
352      '--patch',
353      dest='patches',
354      help='The patches to use for the testing, '
355      "seprate the patch numbers with ',' "
356      'for more than one patches.')
357  parser.add_argument(
358      '--noschedv2',
359      dest='noschedv2',
360      action='store_true',
361      default=False,
362      help='Pass --noschedv2 to crosperf.')
363
364  options = parser.parse_args(argv[1:])
365  if not options.board:
366    print('Please give a board.')
367    return 1
368  if not options.remote:
369    print('Please give at least one remote machine.')
370    return 1
371  if not options.chromeos_root:
372    print('Please specify the ChromeOS root directory.')
373    return 1
374  if options.patches:
375    patches = options.patches
376  else:
377    patches = USE_NEXT_GCC_PATCH
378
379  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
380                           options.weekday, patches, options.noschedv2)
381  return fc.DoAll()
382
383
384if __name__ == '__main__':
385  retval = Main(sys.argv)
386  sys.exit(retval)
387