buildbot_test_llvm.py revision a8517eb004edaffb972a429915dac2b30bb22dff
1#!/usr/bin/env python2 2"""Script for running llvm validation tests on ChromeOS. 3 4This script launches a buildbot to build ChromeOS with the llvm on 5a particular board; then it finds and downloads the trybot image and the 6corresponding official image, and runs test for correctness. 7It then generates a report, emails it to the c-compiler-chrome, as 8well as copying the result into a directory. 9""" 10 11# Script to test different toolchains against ChromeOS benchmarks. 12 13from __future__ import print_function 14 15import argparse 16import datetime 17import os 18import sys 19import time 20 21from cros_utils import command_executer 22from cros_utils import logger 23 24from cros_utils import buildbot_utils 25 26# CL that uses LLVM to build the peppy image. 27USE_LLVM_PATCH = '295217' 28 29CROSTC_ROOT = '/usr/local/google/crostc' 30ROLE_ACCOUNT = 'mobiletc-prebuild' 31TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 32MAIL_PROGRAM = '~/var/bin/mail-sheriff' 33VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result') 34START_DATE = datetime.date(2016, 1, 1) 35TEST_PER_DAY = 2 36TEST_BOARD = [ 37 'squawks', # x86_64, rambi (baytrail) 38 'terra', # x86_64, strago (braswell) 39 'lulu', # x86_64, auron (broadwell) 40 'peach_pit', # arm, peach (exynos-5420) 41 'peppy', # x86_64, slippy (haswell celeron) 42 'link', # x86_64, ivybridge (ivybridge) 43 'nyan_big', # arm, nyan (tegra) 44 'sentry', # x86_64, kunimitsu (skylake-u) 45 'chell', # x86_64, glados (skylake-y) 46 'daisy', # arm, daisy (exynos) 47] 48 49 50class ToolchainVerifier(object): 51 """Class for the toolchain verifier.""" 52 53 def __init__(self, board, chromeos_root, weekday, patches, compiler): 54 self._board = board 55 self._chromeos_root = chromeos_root 56 self._base_dir = os.getcwd() 57 self._ce = command_executer.GetCommandExecuter() 58 self._l = logger.GetLogger() 59 self._compiler = compiler 60 self._build = '%s-%s-toolchain' % (board, compiler) 61 self._patches = patches.split(',') if patches else [] 62 self._patches_string = '_'.join(str(p) for p in self._patches) 63 64 if not weekday: 65 self._weekday = time.strftime('%a') 66 else: 67 self._weekday = weekday 68 self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 69 70 def _FinishSetup(self): 71 """Make sure testing_rsa file is properly set up.""" 72 # Fix protections on ssh key 73 command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' 74 '/chrome-src-internal/src/third_party/chromite/ssh_keys' 75 '/testing_rsa') 76 ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 77 if ret_val != 0: 78 raise RuntimeError('chmod for testing_rsa failed') 79 80 def DoAll(self): 81 """Main function inside ToolchainComparator class. 82 83 Launch trybot, get image names, create crosperf experiment file, run 84 crosperf, and copy images into seven-day report directories. 85 """ 86 flags = ['--hwtest'] 87 date_str = datetime.date.today() 88 description = 'master_%s_%s_%s' % (self._patches_string, self._build, 89 date_str) 90 _ = buildbot_utils.GetTrybotImage( 91 self._chromeos_root, 92 self._build, 93 self._patches, 94 description, 95 other_flags=flags, 96 async=True) 97 98 return 0 99 100def Main(argv): 101 """The main function.""" 102 103 # Common initializations 104 command_executer.InitCommandExecuter() 105 parser = argparse.ArgumentParser() 106 parser.add_argument( 107 '--chromeos_root', 108 dest='chromeos_root', 109 help='The chromeos root from which to run tests.') 110 parser.add_argument( 111 '--weekday', 112 default='', 113 dest='weekday', 114 help='The day of the week for which to run tests.') 115 parser.add_argument( 116 '--board', default='', dest='board', help='The board to test.') 117 parser.add_argument( 118 '--patch', 119 dest='patches', 120 default='', 121 help='The patches to use for the testing, ' 122 "seprate the patch numbers with ',' " 123 'for more than one patches.') 124 parser.add_argument( 125 '--compiler', 126 dest='compiler', 127 help='Which compiler (llvm, llvm-next or gcc) to use for ' 128 'testing.') 129 130 options = parser.parse_args(argv[1:]) 131 if not options.chromeos_root: 132 print('Please specify the ChromeOS root directory.') 133 return 1 134 if not options.compiler: 135 print('Please specify which compiler to test (gcc, llvm, or llvm-next).') 136 return 1 137 patches = options.patches 138 if not patches and options.compiler == 'llvm': 139 patches = USE_LLVM_PATCH 140 141 if options.board: 142 fv = ToolchainVerifier(options.board, options.chromeos_root, 143 options.weekday, patches, options.compiler) 144 return fv.Doall() 145 146 today = datetime.date.today() 147 delta = today - START_DATE 148 days = delta.days 149 150 start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) 151 for i in range(TEST_PER_DAY): 152 try: 153 board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 154 fv = ToolchainVerifier(board, options.chromeos_root, options.weekday, 155 patches, options.compiler) 156 fv.DoAll() 157 except SystemExit: 158 logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board) 159 with open(logfile, 'w') as f: 160 f.write('Verifier got an exception, please check the log.\n') 161 162 163if __name__ == '__main__': 164 retval = Main(sys.argv) 165 sys.exit(retval) 166