buildbot_test_llvm.py revision 1ba6d5764234e1df0ff56a256a2e2cae94f28353
1#!/usr/bin/env python2 2"""Script for running llvm validation tests on ChromeOS. 3 4This script launches a buildbot to build ChromeOS with the llvm on 5a particular board; then it finds and downloads the trybot image and the 6corresponding official image, and runs test for correctness. 7It then generates a report, emails it to the c-compiler-chrome, as 8well as copying the result into a directory. 9""" 10 11# Script to test different toolchains against ChromeOS benchmarks. 12 13from __future__ import print_function 14 15import argparse 16import datetime 17import os 18import sys 19import time 20 21from cros_utils import command_executer 22from cros_utils import logger 23 24from cros_utils import buildbot_utils 25 26# CL that uses LLVM to build the peppy image. 27USE_LLVM_PATCH = '295217' 28 29CROSTC_ROOT = '/usr/local/google/crostc' 30ROLE_ACCOUNT = 'mobiletc-prebuild' 31TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 32MAIL_PROGRAM = '~/var/bin/mail-sheriff' 33VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result') 34START_DATE = datetime.date(2016, 1, 1) 35TEST_PER_DAY = 2 36TEST_BOARD = [ 37 'squawks', 38 'terra', 39 'lulu', 40 'peach_pit', 41 'peppy', 42 'link', 43 'sentry', 44 'chell', 45 'nyan_big', 46 'daisy', 47] 48 49 50class ToolchainVerifier(object): 51 """Class for the toolchain verifier.""" 52 53 def __init__(self, board, chromeos_root, weekday, patches, compiler): 54 self._board = board 55 self._chromeos_root = chromeos_root 56 self._base_dir = os.getcwd() 57 self._ce = command_executer.GetCommandExecuter() 58 self._l = logger.GetLogger() 59 self._compiler = compiler 60 self._build = '%s-%s-toolchain' % (board, compiler) 61 self._patches = patches.split(',') 62 self._patches_string = '_'.join(str(p) for p in self._patches) 63 64 if not weekday: 65 self._weekday = time.strftime('%a') 66 else: 67 self._weekday = weekday 68 self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 69 70 def _FinishSetup(self): 71 """Make sure testing_rsa file is properly set up.""" 72 # Fix protections on ssh key 73 command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' 74 '/chrome-src-internal/src/third_party/chromite/ssh_keys' 75 '/testing_rsa') 76 ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 77 if ret_val != 0: 78 raise RuntimeError('chmod for testing_rsa failed') 79 80 def DoAll(self): 81 """Main function inside ToolchainComparator class. 82 83 Launch trybot, get image names, create crosperf experiment file, run 84 crosperf, and copy images into seven-day report directories. 85 """ 86 flags = ['--hwtest'] 87 date_str = datetime.date.today() 88 description = 'master_%s_%s_%s' % (self._patches_string, self._build, 89 date_str) 90 trybot_image = buildbot_utils.GetTrybotImage( 91 self._chromeos_root, 92 self._build, 93 self._patches, 94 description, 95 other_flags=flags) 96 if len(trybot_image) == 0: 97 self._l.LogError('Unable to find trybot_image for %s!' % description) 98 return 1 99 100 if os.getlogin() == ROLE_ACCOUNT: 101 self._FinishSetup() 102 103 return 0 104 105 106def SendEmail(start_board, compiler): 107 """Send out the test results for all the boards.""" 108 results = '' 109 for i in range(len(TEST_BOARD)): 110 board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 111 f = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 112 if not os.path.exists(f): 113 continue 114 results += board 115 results += '\n' 116 file_name = os.path.join(VALIDATION_RESULT_DIR, f) 117 with open(file_name, 'r') as readin: 118 read_data = readin.read() 119 results += read_data 120 121 output = os.path.join(VALIDATION_RESULT_DIR, compiler, 'result') 122 with open(output, 'w') as out: 123 out.write(results) 124 125 ce = command_executer.GetCommandExecuter() 126 if os.path.exists(os.path.expanduser(MAIL_PROGRAM)): 127 email_title = '%s validation test results' % compiler 128 command = ('cat %s | %s -s "%s" -team' % 129 (output, MAIL_PROGRAM, email_title)) 130 ce.RunCommand(command) 131 132 133def Main(argv): 134 """The main function.""" 135 136 # Common initializations 137 command_executer.InitCommandExecuter() 138 parser = argparse.ArgumentParser() 139 parser.add_argument( 140 '--chromeos_root', 141 dest='chromeos_root', 142 help='The chromeos root from which to run tests.') 143 parser.add_argument( 144 '--weekday', 145 default='', 146 dest='weekday', 147 help='The day of the week for which to run tests.') 148 parser.add_argument( 149 '--board', default='', dest='board', help='The board to test.') 150 parser.add_argument( 151 '--patch', 152 dest='patches', 153 help='The patches to use for the testing, ' 154 "seprate the patch numbers with ',' " 155 'for more than one patches.') 156 parser.add_argument( 157 '--compiler', 158 dest='compiler', 159 help='Which compiler (llvm, llvm-next or gcc) to use for ' 160 'testing.') 161 162 options = parser.parse_args(argv[1:]) 163 if not options.chromeos_root: 164 print('Please specify the ChromeOS root directory.') 165 return 1 166 if not options.compiler: 167 print('Please specify which compiler to test (gcc, llvm, or llvm-next).') 168 return 1 169 if options.patches: 170 patches = options.patches 171 elif options.compiler == 'llvm': 172 patches = USE_LLVM_PATCH 173 174 if options.board: 175 fv = ToolchainVerifier(options.board, options.chromeos_root, 176 options.weekday, patches, options.compiler) 177 return fv.Doall() 178 179 today = datetime.date.today() 180 delta = today - START_DATE 181 days = delta.days 182 183 start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) 184 for i in range(TEST_PER_DAY): 185 try: 186 board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 187 fv = ToolchainVerifier(board, options.chromeos_root, options.weekday, 188 patches, options.compiler) 189 fv.DoAll() 190 except SystemExit: 191 logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board) 192 with open(logfile, 'w') as f: 193 f.write('Verifier got an exception, please check the log.\n') 194 195 SendEmail(start_board, options.compiler) 196 197 198if __name__ == '__main__': 199 retval = Main(sys.argv) 200 sys.exit(retval) 201