buildbot_test_toolchains.py revision f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe 
1#!/usr/bin/python2
2"""Script for running nightly compiler tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the latest compiler on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs crosperf performance tests comparing
7the two.  It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the images into the seven-day reports directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import datetime
16import optparse
17import os
18import sys
19import time
20
21from utils import command_executer
22from utils import logger
23
24from utils import buildbot_utils
25
26# CL that updated GCC ebuilds to use 'next_gcc'.
27USE_NEXT_GCC_PATCH = '230260'
28
29# CL that uses LLVM to build the peppy image.
30USE_LLVM_PATCH = '295217'
31
32# The boards on which we run weekly reports
33WEEKLY_REPORT_BOARDS = ['lumpy']
34
35CROSTC_ROOT = '/usr/local/google/crostc'
36ROLE_ACCOUNT = 'mobiletc-prebuild'
37TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
38MAIL_PROGRAM = '~/var/bin/mail-sheriff'
39WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
40PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
41NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
42
43
44class ToolchainComparator(object):
45  """Class for doing the nightly tests work."""
46
47  def __init__(self,
48               board,
49               remotes,
50               chromeos_root,
51               weekday,
52               patches,
53               noschedv2=False):
54    self._board = board
55    self._remotes = remotes
56    self._chromeos_root = chromeos_root
57    self._base_dir = os.getcwd()
58    self._ce = command_executer.GetCommandExecuter()
59    self._l = logger.GetLogger()
60    self._build = '%s-release' % board
61    self._patches = patches.split(',')
62    self._patches_string = '_'.join(str(p) for p in self._patches)
63    self._noschedv2 = noschedv2
64
65    if not weekday:
66      self._weekday = time.strftime('%a')
67    else:
68      self._weekday = weekday
69    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
70                                           '%Y-%m-%d_%H:%M:%S')
71    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
72                                     '%s.%s' % (timestamp, board),)
73
74  def _ParseVanillaImage(self, trybot_image):
75    """Parse a trybot artifact name to get corresponding vanilla image.
76
77    This function takes an artifact name, such as
78    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
79    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
80    """
81    start_pos = trybot_image.find(self._build)
82    end_pos = trybot_image.rfind('-b')
83    vanilla_image = trybot_image[start_pos:end_pos]
84    return vanilla_image
85
86  def _FinishSetup(self):
87    """Make sure testing_rsa file is properly set up."""
88    # Fix protections on ssh key
89    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
90               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
91               '/testing_rsa')
92    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
93    if ret_val != 0:
94      raise RuntimeError('chmod for testing_rsa failed')
95
96  def _TestImages(self, trybot_image, vanilla_image):
97    """Create crosperf experiment file.
98
99    Given the names of the trybot and vanilla images, create the
100    appropriate crosperf experiment file and launch crosperf on it.
101    """
102    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
103    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
104
105    compiler_string = 'gcc'
106    if USE_LLVM_PATCH in self._patches_string:
107      experiment_file_name = '%s_llvm_experiment.txt' % self._board
108      compiler_string = 'llvm'
109
110    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
111    experiment_header = """
112    board: %s
113    remote: %s
114    retries: 1
115    """ % (self._board, self._remotes)
116    experiment_tests = """
117    benchmark: all_toolchain_perf {
118      suite: telemetry_Crosperf
119      iterations: 3
120    }
121    """
122
123    with open(experiment_file, 'w') as f:
124      f.write(experiment_header)
125      f.write(experiment_tests)
126
127      # Now add vanilla to test file.
128      official_image = """
129          vanilla_image {
130            chromeos_root: %s
131            build: %s
132            compiler: gcc
133          }
134          """ % (self._chromeos_root, vanilla_image)
135      f.write(official_image)
136
137      label_string = '%s_trybot_image' % compiler_string
138      if USE_NEXT_GCC_PATCH in self._patches:
139        label_string = 'gcc_next_trybot_image'
140
141      experiment_image = """
142          %s {
143            chromeos_root: %s
144            build: %s
145            compiler: %s
146          }
147          """ % (label_string, self._chromeos_root, trybot_image,
148                 compiler_string)
149      f.write(experiment_image)
150
151    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
152    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
153    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
154               '--json_report=True {noschedv2_opts} {exp_file}').format(
155                   crosperf=crosperf,
156                   r_dir=self._reports_dir,
157                   noschedv2_opts=noschedv2_opts,
158                   exp_file=experiment_file)
159
160    ret = self._ce.RunCommand(command)
161    if ret != 0:
162      raise RuntimeError("Couldn't run crosperf!")
163    else:
164      # Copy json report to pending archives directory.
165      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
166      ret = self._ce.RunCommand(command)
167    return
168
169  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
170    """Put files in place for running seven-day reports.
171
172    Create tar files of the custom and official images and copy them
173    to the weekly reports directory, so they exist when the weekly report
174    gets generated.  IMPORTANT NOTE: This function must run *after*
175    crosperf has been run; otherwise the vanilla images will not be there.
176    """
177
178    dry_run = False
179    if os.getlogin() != ROLE_ACCOUNT:
180      self._l.LogOutput('Running this from non-role account; not copying '
181                        'tar files for weekly reports.')
182      dry_run = True
183
184    images_path = os.path.join(
185        os.path.realpath(self._chromeos_root), 'chroot/tmp')
186
187    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
188    dest_dir = os.path.join(data_dir, self._weekday)
189    if not os.path.exists(dest_dir):
190      os.makedirs(dest_dir)
191
192    # Make sure dest_dir is empty (clean out last week's data).
193    cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday)
194    if dry_run:
195      print('CMD: %s' % cmd)
196    else:
197      self._ce.RunCommand(cmd)
198
199    # Now create new tar files and copy them over.
200    labels = ['test', 'vanilla']
201    for label_name in labels:
202      if label_name == 'test':
203        test_path = trybot_image
204      else:
205        test_path = vanilla_image
206      tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name)
207      cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
208             'cp %s %s/.') % (images_path, tar_file_name, test_path,
209                              tar_file_name, dest_dir)
210      if dry_run:
211        print('CMD: %s' % cmd)
212        tar_ret = 0
213      else:
214        tar_ret = self._ce.RunCommand(cmd)
215      if tar_ret:
216        self._l.LogOutput('Error while creating/copying test tar file(%s).' %
217                          tar_file_name)
218
219  def _SendEmail(self):
220    """Find email message generated by crosperf and send it."""
221    filename = os.path.join(self._reports_dir, 'msg_body.html')
222    if (os.path.exists(filename) and
223        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
224      email_title = 'buildbot test results'
225      if self._patches_string == USE_LLVM_PATCH:
226        email_title = 'buildbot llvm test results'
227      command = ('cat %s | %s -s "%s, %s" -team -html' %
228                 (filename, MAIL_PROGRAM, email_title, self._board))
229      self._ce.RunCommand(command)
230
231  def DoAll(self):
232    """Main function inside ToolchainComparator class.
233
234    Launch trybot, get image names, create crosperf experiment file, run
235    crosperf, and copy images into seven-day report directories.
236    """
237    date_str = datetime.date.today()
238    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
239                                       date_str)
240    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
241                                                 self._build,
242                                                 self._patches,
243                                                 description,
244                                                 build_toolchain=True)
245
246    vanilla_image = self._ParseVanillaImage(trybot_image)
247
248    print('trybot_image: %s' % trybot_image)
249    print('vanilla_image: %s' % vanilla_image)
250    if len(trybot_image) == 0:
251      self._l.LogError('Unable to find trybot_image for %s!' % description)
252      return 1
253    if len(vanilla_image) == 0:
254      self._l.LogError('Unable to find vanilla image for %s!' % description)
255      return 1
256    if os.getlogin() == ROLE_ACCOUNT:
257      self._FinishSetup()
258
259    self._TestImages(trybot_image, vanilla_image)
260    self._SendEmail()
261    if (self._patches_string == USE_NEXT_GCC_PATCH and
262        self._board in WEEKLY_REPORT_BOARDS):
263      # Only try to copy the image files if the test runs ran successfully.
264      self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
265    return 0
266
267
268def Main(argv):
269  """The main function."""
270
271  # Common initializations
272  command_executer.InitCommandExecuter()
273  parser = optparse.OptionParser()
274  parser.add_option('--remote',
275                    dest='remote',
276                    help='Remote machines to run tests on.')
277  parser.add_option('--board',
278                    dest='board',
279                    default='x86-zgb',
280                    help='The target board.')
281  parser.add_option('--chromeos_root',
282                    dest='chromeos_root',
283                    help='The chromeos root from which to run tests.')
284  parser.add_option('--weekday',
285                    default='',
286                    dest='weekday',
287                    help='The day of the week for which to run tests.')
288  parser.add_option('--patch',
289                    dest='patches',
290                    help='The patches to use for the testing, '
291                    "seprate the patch numbers with ',' "
292                    'for more than one patches.')
293  parser.add_option('--noschedv2',
294                    dest='noschedv2',
295                    action='store_true',
296                    default=False,
297                    help='Pass --noschedv2 to crosperf.')
298
299  options, _ = parser.parse_args(argv)
300  if not options.board:
301    print('Please give a board.')
302    return 1
303  if not options.remote:
304    print('Please give at least one remote machine.')
305    return 1
306  if not options.chromeos_root:
307    print('Please specify the ChromeOS root directory.')
308    return 1
309  if options.patches:
310    patches = options.patches
311  else:
312    patches = USE_NEXT_GCC_PATCH
313
314  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
315                           options.weekday, patches, options.noschedv2)
316  return fc.DoAll()
317
318
319if __name__ == '__main__':
320  retval = Main(sys.argv)
321  sys.exit(retval)
322