buildbot_test_llvm.py revision 314ea5683c0b4c7658f6c3f659cb5c53218a613c 
1#!/usr/bin/python2
2"""Script for running llvm validation tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the llvm on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs test for correctness.
7It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the result into a directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import sys
19import time
20
21from utils import command_executer
22from utils import logger
23
24from utils import buildbot_utils
25
26# CL that uses LLVM to build the peppy image.
27USE_LLVM_PATCH = '295217'
28
29
30CROSTC_ROOT = '/usr/local/google/crostc'
31ROLE_ACCOUNT = 'mobiletc-prebuild'
32TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
33MAIL_PROGRAM = '~/var/bin/mail-sheriff'
34VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
35START_DATE = datetime.date(2016, 1, 1)
36TEST_PER_DAY = 2
37TEST_BOARD = [
38    'squawks',
39    'terra',
40    'lulu',
41    'peach_pit',
42    'falco',
43    'oak',
44    'veyron_jaq',
45    'lumpy',
46    'sentry',
47    'chell',
48    'nyan_big',
49    ]
50
51TEST = [
52    'bvt-inline',
53    'bvt-cq',
54    'paygen_au_canary',
55    'security',
56    #'kernel_per-build_regression',
57    #'kernel_per-build_benchmarks',
58    'kernal_daily_regression',
59    'kernel_daily_benchmarks',
60    #'stress',
61    ]
62
63class ToolchainVerifier(object):
64  """Class for the toolchain verifier."""
65
66  def __init__(self,
67               board,
68               chromeos_root,
69               weekday,
70               patches,
71               compiler):
72    self._board = board
73    self._chromeos_root = chromeos_root
74    self._base_dir = os.getcwd()
75    self._ce = command_executer.GetCommandExecuter()
76    self._l = logger.GetLogger()
77    self._compiler = compiler
78    self._build = '%s-release' % board
79    self._patches = patches.split(',')
80    self._patches_string = '_'.join(str(p) for p in self._patches)
81
82    if not weekday:
83      self._weekday = time.strftime('%a')
84    else:
85      self._weekday = weekday
86    self._reports = os.path.join(VALIDATION_RESULT_DIR, board)
87
88  def _FinishSetup(self):
89    """Make sure testing_rsa file is properly set up."""
90    # Fix protections on ssh key
91    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
92               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
93               '/testing_rsa')
94    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
95    if ret_val != 0:
96      raise RuntimeError('chmod for testing_rsa failed')
97
98  def _TestImages(self, image):
99    to_file = ''
100    for test in TEST:
101      command = ('test_that --board {board} :lab: suite:{test} '
102                 '-i {image} --fast --autotest_dir '
103                 '~/trunk/src/third_party/autotest/files '
104                 '--web  cautotest.corp.google.com'.format(
105                     board=self._board,
106                     test=test,
107                     image=image))
108      ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
109      timestamp = datetime.datetime.strftime(datetime.datetime.now(),
110                                             '%Y-%m-%d_%H:%M:%S')
111      if ret_val:
112        out = 'FAILED'
113      else:
114        out = '      '
115      to_file += out + ' ' + test + ' ' + timestamp + '\n'
116      with open(self._reports, 'w') as f:
117        f.write(to_file)
118
119  def DoAll(self):
120    """Main function inside ToolchainComparator class.
121
122    Launch trybot, get image names, create crosperf experiment file, run
123    crosperf, and copy images into seven-day report directories.
124    """
125    date_str = datetime.date.today()
126    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
127                                       date_str)
128    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
129                                                 self._build,
130                                                 self._patches,
131                                                 description,
132                                                 build_toolchain=True)
133    if len(trybot_image) == 0:
134      self._l.LogError('Unable to find trybot_image for %s!' % description)
135      return 1
136
137    if os.getlogin() == ROLE_ACCOUNT:
138      self._FinishSetup()
139
140    self._TestImages(trybot_image)
141    return 0
142
143def SendEmail(start_board, compiler):
144  """Send out the test results for all the boards."""
145  results = ""
146  for i in range(len(TEST_BOARD)):
147    board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
148    f = os.path.join(VALIDATION_RESULT_DIR, board)
149    if not os.path.exists(f):
150      continue
151    results += board
152    results += '\n'
153    file_name = os.path.join(VALIDATION_RESULT_DIR, f)
154    with open(file_name, 'r') as readin:
155      read_data = readin.read()
156      results += read_data
157
158  output = os.path.join(VALIDATION_RESULT_DIR, "result")
159  with open(output, 'w') as out:
160    out.write(results)
161
162  ce = command_executer.GetCommandExecuter()
163  if os.path.exists(os.path.expanduser(MAIL_PROGRAM)):
164    email_title = '%s validation test results' % compiler
165    command = ('cat %s | %s -s "%s" -team' %
166               (output, MAIL_PROGRAM, email_title))
167    ce.RunCommand(command)
168
169
170def Main(argv):
171  """The main function."""
172
173  # Common initializations
174  command_executer.InitCommandExecuter()
175  parser = argparse.ArgumentParser()
176  parser.add_argument('--chromeos_root',
177                      dest='chromeos_root',
178                      help='The chromeos root from which to run tests.')
179  parser.add_argument('--weekday',
180                      default='',
181                      dest='weekday',
182                      help='The day of the week for which to run tests.')
183  parser.add_argument('--board',
184                      default='',
185                      dest='board',
186                      help='The board to test.')
187  parser.add_argument('--patch',
188                      dest='patches',
189                      help='The patches to use for the testing, '
190                      "seprate the patch numbers with ',' "
191                      'for more than one patches.')
192  parser.add_argument('--compiler',
193                      dest='compiler',
194                      help='Which compiler (llvm or gcc) to use for '
195                      'testing.')
196
197  options = parser.parse_args(argv[1:])
198  if not options.chromeos_root:
199    print('Please specify the ChromeOS root directory.')
200    return 1
201  if not options.compiler:
202    print('Please specify which compiler to test (gcc or llvm).')
203    return 1
204  if options.patches:
205    patches = options.patches
206  elif options.compiler == 'llvm':
207    patches = USE_LLVM_PATCH
208
209  if options.board:
210    fv = ToolchainVerifier(options.board, options.chromeos_root,
211                           options.weekday, patches, options.compiler)
212    return fv.Doall()
213
214  today = datetime.date.today()
215  delta = today - START_DATE
216  days = delta.days
217
218  start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
219  for i in range(TEST_PER_DAY):
220    try:
221      board = TEST_BOARD[(start_board + i)  % len(TEST_BOARD)]
222      fv = ToolchainVerifier(board, options.chromeos_root,
223                             options.weekday, patches, options.compiler)
224      fv.DoAll()
225    except SystemExit:
226      logfile = os.path.join(VALIDATION_RESULT_DIR, board)
227      with open(logfile, 'w') as f:
228        f.write("Verifier got an exception, please check the log.\n")
229
230  SendEmail(start_board, options.compiler)
231
232if __name__ == '__main__':
233  retval = Main(sys.argv)
234  sys.exit(retval)
235