buildbot_test_llvm.py revision 0ded515f8f340188634bd8d14e3184b98e4f06de
19106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn#!/usr/bin/python2 29106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn"""Script for running llvm validation tests on ChromeOS. 39106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn 49106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournThis script launches a buildbot to build ChromeOS with the llvm on 59106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourna particular board; then it finds and downloads the trybot image and the 69106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourncorresponding official image, and runs test for correctness. 79106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournIt then generates a report, emails it to the c-compiler-chrome, as 89106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbournwell as copying the result into a directory. 99106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn""" 109106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn 119106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn# Script to test different toolchains against ChromeOS benchmarks. 129106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn 139106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbournfrom __future__ import print_function 149106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn 159106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbournimport argparse 169106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbournimport datetime 179106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbournimport os 189240e796bc63422c28f2707840bd99c48573279bDake Guimport sys 199240e796bc63422c28f2707840bd99c48573279bDake Guimport time 209240e796bc63422c28f2707840bd99c48573279bDake Gu 219240e796bc63422c28f2707840bd99c48573279bDake Gufrom cros_utils import command_executer 229240e796bc63422c28f2707840bd99c48573279bDake Gufrom cros_utils import logger 239240e796bc63422c28f2707840bd99c48573279bDake Gu 249240e796bc63422c28f2707840bd99c48573279bDake Gufrom cros_utils import buildbot_utils 259240e796bc63422c28f2707840bd99c48573279bDake Gu 269240e796bc63422c28f2707840bd99c48573279bDake Gu# CL that uses LLVM to build the peppy image. 279106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournUSE_LLVM_PATCH = '295217' 289106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn 299106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn 30f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig StoutCROSTC_ROOT = '/usr/local/google/crostc' 319240e796bc63422c28f2707840bd99c48573279bDake GuROLE_ACCOUNT = 'mobiletc-prebuild' 329106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournTOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 339106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournMAIL_PROGRAM = '~/var/bin/mail-sheriff' 34a00bada00bff4a58436a39472ab14ccb7a8f619dCraig StoutVALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result') 359106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournSTART_DATE = datetime.date(2016, 1, 1) 3608c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake GuTEST_PER_DAY = 2 3708c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake GuTEST_BOARD = [ 3808c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'squawks', 3908c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'terra', 40c9a859537b0871f84afeeb706a5b425fe3f2b4ddAurimas Liutikas 'lulu', 4108c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'peach_pit', 4208c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'falco', 4308c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'oak', 4408c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'veyron_jaq', 4508c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'lumpy', 4608c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'sentry', 4708c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'chell', 4808c56822b71ab0aa0b9bb03e5fd45e28f6e358b8Dake Gu 'nyan_big', 499106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn ] 50a8a3b898da49324e83ea32c3f08776a481312166Tim Kilbourn 519106804a84e5e8733e0b9313f749fa1f726e5d11Tim KilbournTEST = [ 529240e796bc63422c28f2707840bd99c48573279bDake Gu 'bvt-inline', 539240e796bc63422c28f2707840bd99c48573279bDake Gu 'bvt-cq', 549240e796bc63422c28f2707840bd99c48573279bDake Gu 'paygen_au_canary', 559240e796bc63422c28f2707840bd99c48573279bDake Gu 'security', 569240e796bc63422c28f2707840bd99c48573279bDake Gu #'kernel_per-build_regression', 579240e796bc63422c28f2707840bd99c48573279bDake Gu #'kernel_per-build_benchmarks', 589240e796bc63422c28f2707840bd99c48573279bDake Gu 'kernal_daily_regression', 599240e796bc63422c28f2707840bd99c48573279bDake Gu 'kernel_daily_benchmarks', 609240e796bc63422c28f2707840bd99c48573279bDake Gu #'stress', 619240e796bc63422c28f2707840bd99c48573279bDake Gu ] 629240e796bc63422c28f2707840bd99c48573279bDake Gu 639240e796bc63422c28f2707840bd99c48573279bDake Guclass ToolchainVerifier(object): 649240e796bc63422c28f2707840bd99c48573279bDake Gu """Class for the toolchain verifier.""" 659240e796bc63422c28f2707840bd99c48573279bDake Gu 669106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn def __init__(self, 679106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn board, 689106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn chromeos_root, 699106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn weekday, 709106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn patches, 719106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn compiler): 729106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._board = board 739106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._chromeos_root = chromeos_root 749106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._base_dir = os.getcwd() 759106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._ce = command_executer.GetCommandExecuter() 769106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._l = logger.GetLogger() 779106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._compiler = compiler 789106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._build = '%s-release' % board 799106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._patches = patches.split(',') 809106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._patches_string = '_'.join(str(p) for p in self._patches) 81a8a3b898da49324e83ea32c3f08776a481312166Tim Kilbourn 829106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn if not weekday: 83f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout self._weekday = time.strftime('%a') 849106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn else: 859106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn self._weekday = weekday 863e91a20766ee7aea2bd2a8282425b0d61cd44376Craig Stout self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 879240e796bc63422c28f2707840bd99c48573279bDake Gu 889240e796bc63422c28f2707840bd99c48573279bDake Gu def _FinishSetup(self): 899106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn """Make sure testing_rsa file is properly set up.""" 909106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn # Fix protections on ssh key 91f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' 92f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout '/chrome-src-internal/src/third_party/chromite/ssh_keys' 9346443cb5b092f1d9156342645088eead9da026f6Dake Gu '/testing_rsa') 9446443cb5b092f1d9156342645088eead9da026f6Dake Gu ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 9546443cb5b092f1d9156342645088eead9da026f6Dake Gu if ret_val != 0: 96f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout raise RuntimeError('chmod for testing_rsa failed') 97f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout 98f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout def _TestImages(self, image): 999106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn to_file = '' 100a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout for test in TEST: 1019106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn command = ('test_that --board {board} :lab: suite:{test} ' 1029106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn '-i {image} --fast --autotest_dir ' 1039106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn '~/trunk/src/third_party/autotest/files ' 1049106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn '--web cautotest.corp.google.com'.format( 1059106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn board=self._board, 1069106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn test=test, 1079106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn image=image)) 108a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 109f272f7533fcb5aba341e9ab2f4ff0421d668a8caCraig Stout timestamp = datetime.datetime.strftime(datetime.datetime.now(), 110a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout '%Y-%m-%d_%H:%M:%S') 111a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout if ret_val: 112a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout out = 'FAILED' 1139106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn else: 1149106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn out = ' ' 1159106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn to_file += out + ' ' + test + ' ' + timestamp + '\n' 1169106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn with open(self._reports, 'w') as f: 1179106804a84e5e8733e0b9313f749fa1f726e5d11Tim Kilbourn f.write(to_file) 1189240e796bc63422c28f2707840bd99c48573279bDake Gu 1199240e796bc63422c28f2707840bd99c48573279bDake Gu def DoAll(self): 120a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout """Main function inside ToolchainComparator class. 1219240e796bc63422c28f2707840bd99c48573279bDake Gu 1229240e796bc63422c28f2707840bd99c48573279bDake Gu Launch trybot, get image names, create crosperf experiment file, run 1239240e796bc63422c28f2707840bd99c48573279bDake Gu crosperf, and copy images into seven-day report directories. 1249240e796bc63422c28f2707840bd99c48573279bDake Gu """ 1259240e796bc63422c28f2707840bd99c48573279bDake Gu date_str = datetime.date.today() 1269240e796bc63422c28f2707840bd99c48573279bDake Gu description = 'master_%s_%s_%s' % (self._patches_string, self._build, 1279240e796bc63422c28f2707840bd99c48573279bDake Gu date_str) 1289240e796bc63422c28f2707840bd99c48573279bDake Gu trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, 1299240e796bc63422c28f2707840bd99c48573279bDake Gu self._build, 130f0e485de16a48547b1c6b272cf005d0b80b92e79Craig Stout self._patches, 1319240e796bc63422c28f2707840bd99c48573279bDake Gu description, 1329240e796bc63422c28f2707840bd99c48573279bDake Gu other_flags=['--hwtest'], 1339240e796bc63422c28f2707840bd99c48573279bDake Gu build_toolchain=True) 1349240e796bc63422c28f2707840bd99c48573279bDake Gu if len(trybot_image) == 0: 135a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout self._l.LogError('Unable to find trybot_image for %s!' % description) 1369240e796bc63422c28f2707840bd99c48573279bDake Gu return 1 1379240e796bc63422c28f2707840bd99c48573279bDake Gu 1389240e796bc63422c28f2707840bd99c48573279bDake Gu if os.getlogin() == ROLE_ACCOUNT: 1399240e796bc63422c28f2707840bd99c48573279bDake Gu self._FinishSetup() 1409240e796bc63422c28f2707840bd99c48573279bDake Gu 1419240e796bc63422c28f2707840bd99c48573279bDake Gu self._TestImages(trybot_image) 142a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout return 0 1439240e796bc63422c28f2707840bd99c48573279bDake Gu 1449240e796bc63422c28f2707840bd99c48573279bDake Gudef SendEmail(start_board, compiler): 1459240e796bc63422c28f2707840bd99c48573279bDake Gu """Send out the test results for all the boards.""" 1469240e796bc63422c28f2707840bd99c48573279bDake Gu results = "" 1479240e796bc63422c28f2707840bd99c48573279bDake Gu for i in range(len(TEST_BOARD)): 1489240e796bc63422c28f2707840bd99c48573279bDake Gu board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 1499240e796bc63422c28f2707840bd99c48573279bDake Gu f = os.path.join(VALIDATION_RESULT_DIR, compiler, board) 1509240e796bc63422c28f2707840bd99c48573279bDake Gu if not os.path.exists(f): 1519240e796bc63422c28f2707840bd99c48573279bDake Gu continue 1529240e796bc63422c28f2707840bd99c48573279bDake Gu results += board 1539240e796bc63422c28f2707840bd99c48573279bDake Gu results += '\n' 1549240e796bc63422c28f2707840bd99c48573279bDake Gu file_name = os.path.join(VALIDATION_RESULT_DIR, f) 1559240e796bc63422c28f2707840bd99c48573279bDake Gu with open(file_name, 'r') as readin: 1569240e796bc63422c28f2707840bd99c48573279bDake Gu read_data = readin.read() 1579240e796bc63422c28f2707840bd99c48573279bDake Gu results += read_data 158a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout 1599240e796bc63422c28f2707840bd99c48573279bDake Gu output = os.path.join(VALIDATION_RESULT_DIR, compiler, "result") 1609240e796bc63422c28f2707840bd99c48573279bDake Gu with open(output, 'w') as out: 1619240e796bc63422c28f2707840bd99c48573279bDake Gu out.write(results) 1629240e796bc63422c28f2707840bd99c48573279bDake Gu 1639240e796bc63422c28f2707840bd99c48573279bDake Gu ce = command_executer.GetCommandExecuter() 1649240e796bc63422c28f2707840bd99c48573279bDake Gu if os.path.exists(os.path.expanduser(MAIL_PROGRAM)): 165a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout email_title = '%s validation test results' % compiler 1669240e796bc63422c28f2707840bd99c48573279bDake Gu command = ('cat %s | %s -s "%s" -team' % 1679240e796bc63422c28f2707840bd99c48573279bDake Gu (output, MAIL_PROGRAM, email_title)) 1689240e796bc63422c28f2707840bd99c48573279bDake Gu ce.RunCommand(command) 1699240e796bc63422c28f2707840bd99c48573279bDake Gu 1709240e796bc63422c28f2707840bd99c48573279bDake Gu 1719240e796bc63422c28f2707840bd99c48573279bDake Gudef Main(argv): 1729240e796bc63422c28f2707840bd99c48573279bDake Gu """The main function.""" 1739240e796bc63422c28f2707840bd99c48573279bDake Gu 1749240e796bc63422c28f2707840bd99c48573279bDake Gu # Common initializations 1759240e796bc63422c28f2707840bd99c48573279bDake Gu command_executer.InitCommandExecuter() 1769240e796bc63422c28f2707840bd99c48573279bDake Gu parser = argparse.ArgumentParser() 177a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout parser.add_argument('--chromeos_root', 1789240e796bc63422c28f2707840bd99c48573279bDake Gu dest='chromeos_root', 1799240e796bc63422c28f2707840bd99c48573279bDake Gu help='The chromeos root from which to run tests.') 1809240e796bc63422c28f2707840bd99c48573279bDake Gu parser.add_argument('--weekday', 1819240e796bc63422c28f2707840bd99c48573279bDake Gu default='', 1829240e796bc63422c28f2707840bd99c48573279bDake Gu dest='weekday', 1839240e796bc63422c28f2707840bd99c48573279bDake Gu help='The day of the week for which to run tests.') 1849240e796bc63422c28f2707840bd99c48573279bDake Gu parser.add_argument('--board', 1859240e796bc63422c28f2707840bd99c48573279bDake Gu default='', 186a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout dest='board', 1879240e796bc63422c28f2707840bd99c48573279bDake Gu help='The board to test.') 1889240e796bc63422c28f2707840bd99c48573279bDake Gu parser.add_argument('--patch', 1899240e796bc63422c28f2707840bd99c48573279bDake Gu dest='patches', 1909240e796bc63422c28f2707840bd99c48573279bDake Gu help='The patches to use for the testing, ' 1919240e796bc63422c28f2707840bd99c48573279bDake Gu "seprate the patch numbers with ',' " 1929240e796bc63422c28f2707840bd99c48573279bDake Gu 'for more than one patches.') 1939240e796bc63422c28f2707840bd99c48573279bDake Gu parser.add_argument('--compiler', 1949240e796bc63422c28f2707840bd99c48573279bDake Gu dest='compiler', 1959240e796bc63422c28f2707840bd99c48573279bDake Gu help='Which compiler (llvm or gcc) to use for ' 196f0e485de16a48547b1c6b272cf005d0b80b92e79Craig Stout 'testing.') 1979240e796bc63422c28f2707840bd99c48573279bDake Gu 1989240e796bc63422c28f2707840bd99c48573279bDake Gu options = parser.parse_args(argv[1:]) 1999240e796bc63422c28f2707840bd99c48573279bDake Gu if not options.chromeos_root: 2009240e796bc63422c28f2707840bd99c48573279bDake Gu print('Please specify the ChromeOS root directory.') 201a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout return 1 2029240e796bc63422c28f2707840bd99c48573279bDake Gu if not options.compiler: 2039240e796bc63422c28f2707840bd99c48573279bDake Gu print('Please specify which compiler to test (gcc or llvm).') 2049240e796bc63422c28f2707840bd99c48573279bDake Gu return 1 2059240e796bc63422c28f2707840bd99c48573279bDake Gu if options.patches: 2069240e796bc63422c28f2707840bd99c48573279bDake Gu patches = options.patches 2079240e796bc63422c28f2707840bd99c48573279bDake Gu elif options.compiler == 'llvm': 208a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout patches = USE_LLVM_PATCH 2099240e796bc63422c28f2707840bd99c48573279bDake Gu 2109240e796bc63422c28f2707840bd99c48573279bDake Gu if options.board: 2119240e796bc63422c28f2707840bd99c48573279bDake Gu fv = ToolchainVerifier(options.board, options.chromeos_root, 2129240e796bc63422c28f2707840bd99c48573279bDake Gu options.weekday, patches, options.compiler) 2139240e796bc63422c28f2707840bd99c48573279bDake Gu return fv.Doall() 2149240e796bc63422c28f2707840bd99c48573279bDake Gu 2159240e796bc63422c28f2707840bd99c48573279bDake Gu today = datetime.date.today() 2169240e796bc63422c28f2707840bd99c48573279bDake Gu delta = today - START_DATE 2179240e796bc63422c28f2707840bd99c48573279bDake Gu days = delta.days 2189240e796bc63422c28f2707840bd99c48573279bDake Gu 2199240e796bc63422c28f2707840bd99c48573279bDake Gu start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) 2209240e796bc63422c28f2707840bd99c48573279bDake Gu for i in range(TEST_PER_DAY): 2219240e796bc63422c28f2707840bd99c48573279bDake Gu try: 2229240e796bc63422c28f2707840bd99c48573279bDake Gu board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] 2239240e796bc63422c28f2707840bd99c48573279bDake Gu fv = ToolchainVerifier(board, options.chromeos_root, 224a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout options.weekday, patches, options.compiler) 2259240e796bc63422c28f2707840bd99c48573279bDake Gu fv.DoAll() 2269240e796bc63422c28f2707840bd99c48573279bDake Gu except SystemExit: 2279240e796bc63422c28f2707840bd99c48573279bDake Gu logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board) 2289240e796bc63422c28f2707840bd99c48573279bDake Gu with open(logfile, 'w') as f: 2299240e796bc63422c28f2707840bd99c48573279bDake Gu f.write("Verifier got an exception, please check the log.\n") 2309240e796bc63422c28f2707840bd99c48573279bDake Gu 231a00bada00bff4a58436a39472ab14ccb7a8f619dCraig Stout SendEmail(start_board, options.compiler) 2329240e796bc63422c28f2707840bd99c48573279bDake Gu 2339240e796bc63422c28f2707840bd99c48573279bDake Guif __name__ == '__main__': 2349240e796bc63422c28f2707840bd99c48573279bDake Gu retval = Main(sys.argv) 2359240e796bc63422c28f2707840bd99c48573279bDake Gu sys.exit(retval) 2369240e796bc63422c28f2707840bd99c48573279bDake Gu