buildbot_test_llvm.py revision 4bd70465c2e5c60802460aec4fdc698e1cfab8e3
1#!/usr/bin/env python2 2"""Script for running llvm validation tests on ChromeOS. 3 4This script launches a buildbot to build ChromeOS with the llvm on 5a particular board; then it finds and downloads the trybot image and the 6corresponding official image, and runs test for correctness. 7It then generates a report, emails it to the c-compiler-chrome, as 8well as copying the result into a directory. 9""" 10 11# Script to test different toolchains against ChromeOS benchmarks. 12 13from __future__ import print_function 14 15import argparse 16import datetime 17import os 18import sys 19import time 20 21from cros_utils import command_executer 22from cros_utils import logger 23 24from cros_utils import buildbot_utils 25 26# CL that uses LLVM to build the peppy image. 27USE_LLVM_PATCH = '295217' 28 29CROSTC_ROOT = '/usr/local/google/crostc' 30ROLE_ACCOUNT = 'mobiletc-prebuild' 31TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 32MAIL_PROGRAM = '~/var/bin/mail-sheriff' 33VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result') 34START_DATE = datetime.date(2016, 1, 1) 35TEST_PER_DAY = 2 36TEST_BOARD = [ 37 'squawks', 38 'terra', 39 'lulu', 40 'peach_pit', 41 'peppy', 42 'link', 43 'sentry', 44 'chell', 45 'nyan_big', 46 'daisy', 47] 48 49 50class ToolchainVerifier(object): 51 """Class for the toolchain verifier.""" 52 53 def __init__(self, board, chromeos_root, weekday, patches, compiler): 54 self._board = board 55 self._chromeos_root = chromeos_root 56 self._base_dir = os.getcwd() 57 self._ce = command_executer.GetCommandExecuter() 58 self._l = logger.GetLogger() 59 self._compiler = compiler 60 self._build = '%s-%s-toolchain' % (board, compiler) 61 self._patches = patches.split(',') 62 self._patches_string = '_'.join(str(p) for p in self._patches) 63 64 if not weekday: 65 self._weekday = time.strftime('%a') 66 else: 67 self._weekday = weekday 68 self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 69 70 def _FinishSetup(self): 71 """Make sure testing_rsa file is properly set up.""" 72 # Fix protections on ssh key 73 command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' 74 '/chrome-src-internal/src/third_party/chromite/ssh_keys' 75 '/testing_rsa') 76 ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 77 if ret_val != 0: 78 raise RuntimeError('chmod for testing_rsa failed') 79 80 def DoAll(self): 81 """Main function inside ToolchainComparator class. 82 83 Launch trybot, get image names, create crosperf experiment file, run 84 crosperf, and copy images into seven-day report directories. 85 """ 86 date_str = datetime.date.today() 87 description = 'master_%s_%s_%s' % (self._patches_string, self._build, 88 date_str) 89 trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, 90 self._build, self._patches, 91 description) 92 if len(trybot_image) == 0: 93 self._l.LogError('Unable to find trybot_image for %s!' % description) 94 return 1 95 96 if os.getlogin() == ROLE_ACCOUNT: 97 self._FinishSetup() 98 99 return 0 100 101 102def SendEmail(start_board, compiler): 103 """Send out the test results for all the boards.""" 104 results = '' 105 for i in range(len(TEST_BOARD)): 106 board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 107 f = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 108 if not os.path.exists(f): 109 continue 110 results += board 111 results += '\n' 112 file_name = os.path.join(VALIDATION_RESULT_DIR, f) 113 with open(file_name, 'r') as readin: 114 read_data = readin.read() 115 results += read_data 116 117 output = os.path.join(VALIDATION_RESULT_DIR, compiler, 'result') 118 with open(output, 'w') as out: 119 out.write(results) 120 121 ce = command_executer.GetCommandExecuter() 122 if os.path.exists(os.path.expanduser(MAIL_PROGRAM)): 123 email_title = '%s validation test results' % compiler 124 command = ('cat %s | %s -s "%s" -team' % 125 (output, MAIL_PROGRAM, email_title)) 126 ce.RunCommand(command) 127 128 129def Main(argv): 130 """The main function.""" 131 132 # Common initializations 133 command_executer.InitCommandExecuter() 134 parser = argparse.ArgumentParser() 135 parser.add_argument( 136 '--chromeos_root', 137 dest='chromeos_root', 138 help='The chromeos root from which to run tests.') 139 parser.add_argument( 140 '--weekday', 141 default='', 142 dest='weekday', 143 help='The day of the week for which to run tests.') 144 parser.add_argument( 145 '--board', default='', dest='board', help='The board to test.') 146 parser.add_argument( 147 '--patch', 148 dest='patches', 149 help='The patches to use for the testing, ' 150 "seprate the patch numbers with ',' " 151 'for more than one patches.') 152 parser.add_argument( 153 '--compiler', 154 dest='compiler', 155 help='Which compiler (llvm, llvm-next or gcc) to use for ' 156 'testing.') 157 158 options = parser.parse_args(argv[1:]) 159 if not options.chromeos_root: 160 print('Please specify the ChromeOS root directory.') 161 return 1 162 if not options.compiler: 163 print('Please specify which compiler to test (gcc, llvm, or llvm-next).') 164 return 1 165 if options.patches: 166 patches = options.patches 167 elif options.compiler == 'llvm': 168 patches = USE_LLVM_PATCH 169 170 if options.board: 171 fv = ToolchainVerifier(options.board, options.chromeos_root, 172 options.weekday, patches, options.compiler) 173 return fv.Doall() 174 175 today = datetime.date.today() 176 delta = today - START_DATE 177 days = delta.days 178 179 start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) 180 for i in range(TEST_PER_DAY): 181 try: 182 board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 183 fv = ToolchainVerifier(board, options.chromeos_root, options.weekday, 184 patches, options.compiler) 185 fv.DoAll() 186 except SystemExit: 187 logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board) 188 with open(logfile, 'w') as f: 189 f.write('Verifier got an exception, please check the log.\n') 190 191 SendEmail(start_board, options.compiler) 192 193 194if __name__ == '__main__': 195 retval = Main(sys.argv) 196 sys.exit(retval) 197