buildbot_test_llvm.py revision 6395d39689b42e519aa37c6ac9ca0fedd765b391 
1#!/usr/bin/python2
2"""Script for running llvm validation tests on ChromeOS.
3
4This script launches a buildbot to build ChromeOS with the llvm on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs test for correctness.
7It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the result into a directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
12
13from __future__ import print_function
14
15import argparse
16import datetime
17import os
18import sys
19import time
20
21from cros_utils import command_executer
22from cros_utils import logger
23
24from cros_utils import buildbot_utils
25
26# CL that uses LLVM to build the peppy image.
27USE_LLVM_PATCH = '295217'
28
29CROSTC_ROOT = '/usr/local/google/crostc'
30ROLE_ACCOUNT = 'mobiletc-prebuild'
31TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
32MAIL_PROGRAM = '~/var/bin/mail-sheriff'
33VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
34START_DATE = datetime.date(2016, 1, 1)
35TEST_PER_DAY = 2
36TEST_BOARD = [
37    'squawks',
38    'terra',
39    'lulu',
40    'peach_pit',
41    'peppy',
42    'link',
43    'veyron_jaq',
44    'lumpy',
45    'sentry',
46    'chell',
47    'nyan_big',
48]
49
50TEST = [
51    'bvt-inline',
52    'bvt-cq',
53    'paygen_au_canary',
54    'security',
55    #'kernel_per-build_regression',
56    #'kernel_per-build_benchmarks',
57    'kernel_daily_regression',
58    'kernel_daily_benchmarks',
59    #'stress',
60]
61
62
63class ToolchainVerifier(object):
64  """Class for the toolchain verifier."""
65
66  def __init__(self, board, chromeos_root, weekday, patches, compiler):
67    self._board = board
68    self._chromeos_root = chromeos_root
69    self._base_dir = os.getcwd()
70    self._ce = command_executer.GetCommandExecuter()
71    self._l = logger.GetLogger()
72    self._compiler = compiler
73    self._build = '%s-release' % board
74    self._patches = patches.split(',')
75    self._patches_string = '_'.join(str(p) for p in self._patches)
76
77    if not weekday:
78      self._weekday = time.strftime('%a')
79    else:
80      self._weekday = weekday
81    self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
82
83  def _FinishSetup(self):
84    """Make sure testing_rsa file is properly set up."""
85    # Fix protections on ssh key
86    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
87               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
88               '/testing_rsa')
89    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
90    if ret_val != 0:
91      raise RuntimeError('chmod for testing_rsa failed')
92
93  def _TestImages(self, image):
94    to_file = ''
95    for test in TEST:
96      # Do not run the kernel tests with the LLVM compiler.
97      if self._compiler == 'gcc' or not 'kernel' in test:
98        command = ('test_that --board {board} :lab: suite:{test} '
99                   '-i {image} --fast --autotest_dir '
100                   '~/trunk/src/third_party/autotest/files '
101                   '--web  cautotest.corp.google.com'.format(
102                       board=self._board, test=test, image=image))
103        ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
104        timestamp = datetime.datetime.strftime(datetime.datetime.now(),
105                                               '%Y-%m-%d_%H:%M:%S')
106        if ret_val:
107          out = 'FAILED'
108        else:
109          out = '      '
110        to_file += out + ' ' + test + ' ' + timestamp + '\n'
111        with open(self._reports, 'w') as f:
112          f.write(to_file)
113
114  def DoAll(self):
115    """Main function inside ToolchainComparator class.
116
117    Launch trybot, get image names, create crosperf experiment file, run
118    crosperf, and copy images into seven-day report directories.
119    """
120    date_str = datetime.date.today()
121    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
122                                       date_str)
123    trybot_image = buildbot_utils.GetTrybotImage(
124        self._chromeos_root,
125        self._build,
126        self._patches,
127        description,
128        other_flags=['--hwtest'],
129        build_toolchain=True)
130    if len(trybot_image) == 0:
131      self._l.LogError('Unable to find trybot_image for %s!' % description)
132      return 1
133
134    if os.getlogin() == ROLE_ACCOUNT:
135      self._FinishSetup()
136
137    self._TestImages(trybot_image)
138    return 0
139
140
141def SendEmail(start_board, compiler):
142  """Send out the test results for all the boards."""
143  results = ''
144  for i in range(len(TEST_BOARD)):
145    board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
146    f = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
147    if not os.path.exists(f):
148      continue
149    results += board
150    results += '\n'
151    file_name = os.path.join(VALIDATION_RESULT_DIR, f)
152    with open(file_name, 'r') as readin:
153      read_data = readin.read()
154      results += read_data
155
156  output = os.path.join(VALIDATION_RESULT_DIR, compiler, 'result')
157  with open(output, 'w') as out:
158    out.write(results)
159
160  ce = command_executer.GetCommandExecuter()
161  if os.path.exists(os.path.expanduser(MAIL_PROGRAM)):
162    email_title = '%s validation test results' % compiler
163    command = ('cat %s | %s -s "%s" -team' %
164               (output, MAIL_PROGRAM, email_title))
165    ce.RunCommand(command)
166
167
168def Main(argv):
169  """The main function."""
170
171  # Common initializations
172  command_executer.InitCommandExecuter()
173  parser = argparse.ArgumentParser()
174  parser.add_argument(
175      '--chromeos_root',
176      dest='chromeos_root',
177      help='The chromeos root from which to run tests.')
178  parser.add_argument(
179      '--weekday',
180      default='',
181      dest='weekday',
182      help='The day of the week for which to run tests.')
183  parser.add_argument(
184      '--board', default='', dest='board', help='The board to test.')
185  parser.add_argument(
186      '--patch',
187      dest='patches',
188      help='The patches to use for the testing, '
189      "seprate the patch numbers with ',' "
190      'for more than one patches.')
191  parser.add_argument(
192      '--compiler',
193      dest='compiler',
194      help='Which compiler (llvm or gcc) to use for '
195      'testing.')
196
197  options = parser.parse_args(argv[1:])
198  if not options.chromeos_root:
199    print('Please specify the ChromeOS root directory.')
200    return 1
201  if not options.compiler:
202    print('Please specify which compiler to test (gcc or llvm).')
203    return 1
204  if options.patches:
205    patches = options.patches
206  elif options.compiler == 'llvm':
207    patches = USE_LLVM_PATCH
208
209  if options.board:
210    fv = ToolchainVerifier(options.board, options.chromeos_root,
211                           options.weekday, patches, options.compiler)
212    return fv.Doall()
213
214  today = datetime.date.today()
215  delta = today - START_DATE
216  days = delta.days
217
218  start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
219  for i in range(TEST_PER_DAY):
220    try:
221      board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
222      fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
223                             patches, options.compiler)
224      fv.DoAll()
225    except SystemExit:
226      logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
227      with open(logfile, 'w') as f:
228        f.write('Verifier got an exception, please check the log.\n')
229
230  SendEmail(start_board, options.compiler)
231
232
233if __name__ == '__main__':
234  retval = Main(sys.argv)
235  sys.exit(retval)
236