buildbot_test_llvm.py revision de600776d8342db1ae2579efd4ca07adbc310d6c 
1#!/usr/bin/env python2
2"""Script for running llvm validation tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the llvm on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs test for correctness.
7It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the result into a directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import sys
19import time
20
21from cros_utils import command_executer
22from cros_utils import logger
23
24from cros_utils import buildbot_utils
25
26# CL that uses LLVM to build the peppy image.
27USE_LLVM_PATCH = '295217'
28
29CROSTC_ROOT = '/usr/local/google/crostc'
30ROLE_ACCOUNT = 'mobiletc-prebuild'
31TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
32MAIL_PROGRAM = '~/var/bin/mail-sheriff'
33VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
34START_DATE = datetime.date(2016, 1, 1)
35TEST_PER_DAY = 2
36TEST_BOARD = [
37    'squawks',
38    'terra',
39    'lulu',
40    'peach_pit',
41    'peppy',
42    'link',
43    'sentry',
44    'chell',
45    'nyan_big',
46    'daisy',
47]
48
49
50class ToolchainVerifier(object):
51  """Class for the toolchain verifier."""
52
53  def __init__(self, board, chromeos_root, weekday, patches, compiler):
54    self._board = board
55    self._chromeos_root = chromeos_root
56    self._base_dir = os.getcwd()
57    self._ce = command_executer.GetCommandExecuter()
58    self._l = logger.GetLogger()
59    self._compiler = compiler
60    self._build = '%s-%s-toolchain' % (board, compiler)
61    self._patches = patches.split(',') if patches else []
62    self._patches_string = '_'.join(str(p) for p in self._patches)
63
64    if not weekday:
65      self._weekday = time.strftime('%a')
66    else:
67      self._weekday = weekday
68    self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
69
70  def _FinishSetup(self):
71    """Make sure testing_rsa file is properly set up."""
72    # Fix protections on ssh key
73    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
74               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
75               '/testing_rsa')
76    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
77    if ret_val != 0:
78      raise RuntimeError('chmod for testing_rsa failed')
79
80  def DoAll(self):
81    """Main function inside ToolchainComparator class.
82
83    Launch trybot, get image names, create crosperf experiment file, run
84    crosperf, and copy images into seven-day report directories.
85    """
86    flags = ['--hwtest']
87    date_str = datetime.date.today()
88    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
89                                       date_str)
90    trybot_image = buildbot_utils.GetTrybotImage(
91        self._chromeos_root,
92        self._build,
93        self._patches,
94        description,
95        other_flags=flags)
96    if len(trybot_image) == 0:
97      self._l.LogError('Unable to find trybot_image for %s!' % description)
98      return 1
99
100    if os.getlogin() == ROLE_ACCOUNT:
101      self._FinishSetup()
102
103    return 0
104
105
106def SendEmail(start_board, compiler):
107  """Send out the test results for all the boards."""
108
109  # This is no longer the correct way to get results.  Until
110  # this is fixed, don't send any email at all.
111  return 0
112
113  results = ''
114  for i in range(len(TEST_BOARD)):
115    board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
116    f = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
117    if not os.path.exists(f):
118      continue
119    results += board
120    results += '\n'
121    file_name = os.path.join(VALIDATION_RESULT_DIR, f)
122    with open(file_name, 'r') as readin:
123      read_data = readin.read()
124      results += read_data
125
126  output = os.path.join(VALIDATION_RESULT_DIR, compiler, 'result')
127  with open(output, 'w') as out:
128    out.write(results)
129
130  ce = command_executer.GetCommandExecuter()
131  if os.path.exists(os.path.expanduser(MAIL_PROGRAM)):
132    email_title = '%s validation test results' % compiler
133    command = ('cat %s | %s -s "%s" -team' %
134               (output, MAIL_PROGRAM, email_title))
135    ce.RunCommand(command)
136
137
138def Main(argv):
139  """The main function."""
140
141  # Common initializations
142  command_executer.InitCommandExecuter()
143  parser = argparse.ArgumentParser()
144  parser.add_argument(
145      '--chromeos_root',
146      dest='chromeos_root',
147      help='The chromeos root from which to run tests.')
148  parser.add_argument(
149      '--weekday',
150      default='',
151      dest='weekday',
152      help='The day of the week for which to run tests.')
153  parser.add_argument(
154      '--board', default='', dest='board', help='The board to test.')
155  parser.add_argument(
156      '--patch',
157      dest='patches',
158      default='',
159      help='The patches to use for the testing, '
160      "seprate the patch numbers with ',' "
161      'for more than one patches.')
162  parser.add_argument(
163      '--compiler',
164      dest='compiler',
165      help='Which compiler (llvm, llvm-next or gcc) to use for '
166      'testing.')
167
168  options = parser.parse_args(argv[1:])
169  if not options.chromeos_root:
170    print('Please specify the ChromeOS root directory.')
171    return 1
172  if not options.compiler:
173    print('Please specify which compiler to test (gcc, llvm, or llvm-next).')
174    return 1
175  patches = options.patches
176  if not patches and options.compiler == 'llvm':
177    patches = USE_LLVM_PATCH
178
179  if options.board:
180    fv = ToolchainVerifier(options.board, options.chromeos_root,
181                           options.weekday, patches, options.compiler)
182    return fv.Doall()
183
184  today = datetime.date.today()
185  delta = today - START_DATE
186  days = delta.days
187
188  start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
189  for i in range(TEST_PER_DAY):
190    try:
191      board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
192      fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
193                             patches, options.compiler)
194      fv.DoAll()
195    except SystemExit:
196      logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
197      with open(logfile, 'w') as f:
198        f.write('Verifier got an exception, please check the log.\n')
199
200  SendEmail(start_board, options.compiler)
201
202
203if __name__ == '__main__':
204  retval = Main(sys.argv)
205  sys.exit(retval)
206