buildbot_test_toolchains.py revision e82513b0aec27bf5d3ca51789edc48dde5ee439b 
1#!/usr/bin/python2
2"""Script for running nightly compiler tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the latest compiler on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs crosperf performance tests comparing
7the two.  It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the images into the seven-day reports directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import re
19import sys
20import time
21
22from cros_utils import command_executer
23from cros_utils import logger
24
25from cros_utils import buildbot_utils
26
27# CL that updated GCC ebuilds to use 'next_gcc'.
28USE_NEXT_GCC_PATCH = '230260'
29
30# CL that uses LLVM to build the peppy image.
31USE_LLVM_PATCH = '295217'
32
33# The boards on which we run weekly reports
34WEEKLY_REPORT_BOARDS = ['lumpy']
35
36CROSTC_ROOT = '/usr/local/google/crostc'
37ROLE_ACCOUNT = 'mobiletc-prebuild'
38TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
39MAIL_PROGRAM = '~/var/bin/mail-sheriff'
40WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
41PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
42NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
43
44IMAGE_FS = (r'{board}-{image_type}/{chrome_version}-{tip}\.' +
45            r'{branch}\.{branch_branch}')
46TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
47PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
48IMAGE_RE_GROUPS = {
49    'board': r'(?P<board>\S+)',
50    'image_type': r'(?P<image_type>\S+)',
51    'chrome_version': r'(?P<chrome_version>R\d+)',
52    'tip': r'(?P<tip>\d+)',
53    'branch': r'(?P<branch>\d+)',
54    'branch_branch': r'(?P<branch_branch>\d+)',
55    'build_id': r'(?P<build_id>b\d+)'
56}
57TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
58
59
60class ToolchainComparator(object):
61  """Class for doing the nightly tests work."""
62
63  def __init__(self,
64               board,
65               remotes,
66               chromeos_root,
67               weekday,
68               patches,
69               noschedv2=False):
70    self._board = board
71    self._remotes = remotes
72    self._chromeos_root = chromeos_root
73    self._base_dir = os.getcwd()
74    self._ce = command_executer.GetCommandExecuter()
75    self._l = logger.GetLogger()
76    self._build = '%s-release' % board
77    self._patches = patches.split(',')
78    self._patches_string = '_'.join(str(p) for p in self._patches)
79    self._noschedv2 = noschedv2
80
81    if not weekday:
82      self._weekday = time.strftime('%a')
83    else:
84      self._weekday = weekday
85    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
86                                           '%Y-%m-%d_%H:%M:%S')
87    self._reports_dir = os.path.join(
88        NIGHTLY_TESTS_DIR,
89        '%s.%s' % (timestamp, board),)
90
91  def _GetVanillaImageName(self, trybot_image):
92    """Given a trybot artifact name, get corresponding vanilla image name.
93
94    Args:
95      trybot_image: artifact name such as
96          'trybot-daisy-release/R40-6394.0.0-b1389'
97
98    Returns:
99      Corresponding official image name, e.g. 'daisy-release/R40-6394.0.0'.
100    """
101    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
102    assert mo
103    return IMAGE_FS.replace('\\', '').format(**mo.groupdict())
104
105  def _GetNonAFDOImageName(self, trybot_image):
106    """Given a trybot artifact name, get corresponding non-AFDO image name.
107
108    We get the non-AFDO image from the PFQ builders. This image
109    is not generated for all the boards and, the closest PFQ image
110    was the one build for the previous ChromeOS version (the chrome
111    used in the current version is the one validated in the previous
112    version).
113    The previous ChromeOS does not always exist either. So, we try
114    a couple of versions before.
115
116    Args:
117      trybot_image: artifact name such as
118          'trybot-daisy-release/R40-6394.0.0-b1389'
119
120    Returns:
121      Corresponding chrome PFQ image name, e.g.
122      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
123    """
124    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
125    assert mo
126    image_dict = mo.groupdict()
127    image_dict['image_type'] = 'chrome-pfq'
128    for _ in xrange(2):
129      image_dict['tip'] = str(int(image_dict['tip']) - 1)
130      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
131      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
132        return nonafdo_image
133    return ''
134
135  def _FinishSetup(self):
136    """Make sure testing_rsa file is properly set up."""
137    # Fix protections on ssh key
138    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
139               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
140               '/testing_rsa')
141    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
142    if ret_val != 0:
143      raise RuntimeError('chmod for testing_rsa failed')
144
145  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
146    """Create crosperf experiment file.
147
148    Given the names of the trybot, vanilla and non-AFDO images, create the
149    appropriate crosperf experiment file and launch crosperf on it.
150    """
151    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
152    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
153
154    compiler_string = 'gcc'
155    if USE_LLVM_PATCH in self._patches_string:
156      experiment_file_name = '%s_llvm_experiment.txt' % self._board
157      compiler_string = 'llvm'
158
159    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
160    experiment_header = """
161    board: %s
162    remote: %s
163    retries: 1
164    """ % (self._board, self._remotes)
165    experiment_tests = """
166    benchmark: all_toolchain_perf {
167      suite: telemetry_Crosperf
168      iterations: 3
169    }
170
171    benchmark: page_cycler_v2.typical_25 {
172      suite: telemetry_Crosperf
173      iterations: 2
174      run_local: False
175      retries: 0
176    }
177    """
178
179    with open(experiment_file, 'w') as f:
180      f.write(experiment_header)
181      f.write(experiment_tests)
182
183      # Now add vanilla to test file.
184      official_image = """
185          vanilla_image {
186            chromeos_root: %s
187            build: %s
188            compiler: gcc
189          }
190          """ % (self._chromeos_root, vanilla_image)
191      f.write(official_image)
192
193      # Now add non-AFDO image to test file.
194      if nonafdo_image:
195        official_nonafdo_image = """
196          nonafdo_image {
197            chromeos_root: %s
198            build: %s
199            compiler: gcc
200          }
201          """ % (self._chromeos_root, nonafdo_image)
202        f.write(official_nonafdo_image)
203
204      label_string = '%s_trybot_image' % compiler_string
205      if USE_NEXT_GCC_PATCH in self._patches:
206        label_string = 'gcc_next_trybot_image'
207
208      experiment_image = """
209          %s {
210            chromeos_root: %s
211            build: %s
212            compiler: %s
213          }
214          """ % (label_string, self._chromeos_root, trybot_image,
215                 compiler_string)
216      f.write(experiment_image)
217
218    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
219    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
220    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
221               '--json_report=True {noschedv2_opts} {exp_file}').format(
222                   crosperf=crosperf,
223                   r_dir=self._reports_dir,
224                   noschedv2_opts=noschedv2_opts,
225                   exp_file=experiment_file)
226
227    ret = self._ce.RunCommand(command)
228    if ret != 0:
229      raise RuntimeError('Crosperf execution error!')
230    else:
231      # Copy json report to pending archives directory.
232      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
233      ret = self._ce.RunCommand(command)
234    return
235
236  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image, nonafdo_image):
237    """Put files in place for running seven-day reports.
238
239    Create tar files of the custom and official images and copy them
240    to the weekly reports directory, so they exist when the weekly report
241    gets generated.  IMPORTANT NOTE: This function must run *after*
242    crosperf has been run; otherwise the vanilla images will not be there.
243    """
244
245    dry_run = False
246    if os.getlogin() != ROLE_ACCOUNT:
247      self._l.LogOutput('Running this from non-role account; not copying '
248                        'tar files for weekly reports.')
249      dry_run = True
250
251    images_path = os.path.join(
252        os.path.realpath(self._chromeos_root), 'chroot/tmp')
253
254    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
255    dest_dir = os.path.join(data_dir, self._weekday)
256    if not os.path.exists(dest_dir):
257      os.makedirs(dest_dir)
258
259    # Make sure dest_dir is empty (clean out last week's data).
260    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday)
261    if dry_run:
262      print('CMD: %s' % cmd)
263    else:
264      self._ce.RunCommand(cmd)
265
266    # Now create new tar files and copy them over.
267    labels = {'test': trybot_image, 'vanilla': vanilla_image}
268    if nonafdo_image:
269      labels['nonafdo'] = nonafdo_image
270    for label_name, test_path in labels.iteritems():
271      tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name)
272      cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
273             'cp %s %s/.') % (images_path, tar_file_name, test_path,
274                              tar_file_name, dest_dir)
275      if dry_run:
276        print('CMD: %s' % cmd)
277        tar_ret = 0
278      else:
279        tar_ret = self._ce.RunCommand(cmd)
280      if tar_ret:
281        self._l.LogOutput('Error while creating/copying test tar file(%s).' %
282                          tar_file_name)
283
284  def _SendEmail(self):
285    """Find email message generated by crosperf and send it."""
286    filename = os.path.join(self._reports_dir, 'msg_body.html')
287    if (os.path.exists(filename) and
288        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
289      email_title = 'buildbot test results'
290      if self._patches_string == USE_LLVM_PATCH:
291        email_title = 'buildbot llvm test results'
292      command = ('cat %s | %s -s "%s, %s" -team -html' %
293                 (filename, MAIL_PROGRAM, email_title, self._board))
294      self._ce.RunCommand(command)
295
296  def DoAll(self):
297    """Main function inside ToolchainComparator class.
298
299    Launch trybot, get image names, create crosperf experiment file, run
300    crosperf, and copy images into seven-day report directories.
301    """
302    date_str = datetime.date.today()
303    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
304                                       date_str)
305    trybot_image = buildbot_utils.GetTrybotImage(
306        self._chromeos_root,
307        self._build,
308        self._patches,
309        description,
310        other_flags=['--notests'],
311        build_toolchain=True)
312
313    if len(trybot_image) == 0:
314      self._l.LogError('Unable to find trybot_image for %s!' % description)
315      return 1
316
317    vanilla_image = self._GetVanillaImageName(trybot_image)
318    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
319
320    # The trybot image is ready here, in some cases, the vanilla image
321    # is not ready, so we need to make sure vanilla image is available.
322    buildbot_utils.WaitForImage(self._chromeos_root, vanilla_image)
323    print('trybot_image: %s' % trybot_image)
324    print('vanilla_image: %s' % vanilla_image)
325    print('nonafdo_image: %s' % nonafdo_image)
326
327    if os.getlogin() == ROLE_ACCOUNT:
328      self._FinishSetup()
329
330    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
331    self._SendEmail()
332    if (self._patches_string == USE_NEXT_GCC_PATCH and
333        self._board in WEEKLY_REPORT_BOARDS):
334      # Only try to copy the image files if the test runs ran successfully.
335      self._CopyWeeklyReportFiles(trybot_image, vanilla_image, nonafdo_image)
336    return 0
337
338
339def Main(argv):
340  """The main function."""
341
342  # Common initializations
343  command_executer.InitCommandExecuter()
344  parser = argparse.ArgumentParser()
345  parser.add_argument(
346      '--remote', dest='remote', help='Remote machines to run tests on.')
347  parser.add_argument(
348      '--board', dest='board', default='x86-zgb', help='The target board.')
349  parser.add_argument(
350      '--chromeos_root',
351      dest='chromeos_root',
352      help='The chromeos root from which to run tests.')
353  parser.add_argument(
354      '--weekday',
355      default='',
356      dest='weekday',
357      help='The day of the week for which to run tests.')
358  parser.add_argument(
359      '--patch',
360      dest='patches',
361      help='The patches to use for the testing, '
362      "seprate the patch numbers with ',' "
363      'for more than one patches.')
364  parser.add_argument(
365      '--noschedv2',
366      dest='noschedv2',
367      action='store_true',
368      default=False,
369      help='Pass --noschedv2 to crosperf.')
370
371  options = parser.parse_args(argv[1:])
372  if not options.board:
373    print('Please give a board.')
374    return 1
375  if not options.remote:
376    print('Please give at least one remote machine.')
377    return 1
378  if not options.chromeos_root:
379    print('Please specify the ChromeOS root directory.')
380    return 1
381  if options.patches:
382    patches = options.patches
383  else:
384    patches = USE_NEXT_GCC_PATCH
385
386  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
387                           options.weekday, patches, options.noschedv2)
388  return fc.DoAll()
389
390
391if __name__ == '__main__':
392  retval = Main(sys.argv)
393  sys.exit(retval)
394