buildbot_test_toolchains.py revision 9db7ae58ddb0c2d5650c9464427c40a3070ec03a
18d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt#!/usr/bin/python 28d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt""" 38d520ff1dc2da35cdca849e982051b86468016d8Dmitry ShmidtScript for running nightly compiler tests on ChromeOS. 48d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 5c5ec7f57ead87efa365800228aa0b09a12d9e6c4Dmitry ShmidtThis script launches a buildbot to build ChromeOS with the latest compiler on 6c5ec7f57ead87efa365800228aa0b09a12d9e6c4Dmitry Shmidta particular board; then it finds and downloads the trybot image and the 78d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtcorresponding official image, and runs crosperf performance tests comparing 88d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtthe two. It then generates a report, emails it to the c-compiler-chrome, as 98d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtwell as copying the images into the seven-day reports directory. 108d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt""" 118d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 128d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt# Script to test different toolchains against ChromeOS benchmarks. 138d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtimport datetime 148d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtimport optparse 158d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtimport os 168d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtimport sys 178d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtimport time 188d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtimport urllib2 198d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 208d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtfrom utils import command_executer 218d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtfrom utils import logger 228d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 238d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtfrom utils import buildbot_utils 248d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 258d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt# CL that updated GCC ebuilds to use 'next_gcc'. 268d520ff1dc2da35cdca849e982051b86468016d8Dmitry ShmidtUSE_NEXT_GCC_PATCH ="230260" 278d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 288d520ff1dc2da35cdca849e982051b86468016d8Dmitry ShmidtWEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data" 298d520ff1dc2da35cdca849e982051b86468016d8Dmitry ShmidtROLE_ACCOUNT = "mobiletc-prebuild" 308d520ff1dc2da35cdca849e982051b86468016d8Dmitry ShmidtTOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 318d520ff1dc2da35cdca849e982051b86468016d8Dmitry ShmidtMAIL_PROGRAM = "~/var/bin/mail-sheriff" 328d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 338d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidtclass ToolchainComparator(): 348d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 358d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Class for doing the nightly tests work. 368d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 378d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 388d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt def __init__(self, board, remotes, chromeos_root, weekday): 398d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._board = board 408d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._remotes = remotes 418d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._chromeos_root = chromeos_root 428d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._base_dir = os.getcwd() 438d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._ce = command_executer.GetCommandExecuter() 448d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._l = logger.GetLogger() 458d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._build = "%s-release" % board 468d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if not weekday: 478d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._weekday = time.strftime("%a") 488d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt else: 498d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._weekday = weekday 508d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt timestamp = datetime.datetime.strftime(datetime.datetime.now(), 518d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "%Y-%m-%d_%H:%M:%S") 528d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._reports_dir = os.path.join( 538d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt os.path.expanduser("~/nightly_test_reports"), 548d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "%s.%s" % (timestamp, board), 558d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt ) 568d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 578d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt def _ParseVanillaImage(self, trybot_image): 588d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 598d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Parse a trybot artifact name to get corresponding vanilla image. 608d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 618d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt This function takes an artifact name, such as 628d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the 638d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'. 648d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 658d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt start_pos = trybot_image.find(self._build) 668d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt end_pos = trybot_image.rfind("-b") 678d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt vanilla_image = trybot_image[start_pos:end_pos] 688d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt return vanilla_image 698d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 708d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt def _FinishSetup(self): 718d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 728d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Make sure testing_rsa file is properly set up. 738d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 748d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt # Fix protections on ssh key 758d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target" 768d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "/chrome-src-internal/src/third_party/chromite/ssh_keys" 778d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "/testing_rsa") 788d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 798d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if ret_val != 0: 808d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt raise RuntimeError("chmod for testing_rsa failed") 818d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 828d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt def _TestImages(self, trybot_image, vanilla_image): 838d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 848d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Create crosperf experiment file. 858d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 868d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Given the names of the trybot and vanilla images, create the 878d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt appropriate crosperf experiment file and launch crosperf on it. 888d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 898d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_file_dir = os.path.join (self._chromeos_root, "..", 908d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._weekday) 918d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_file_name = "%s_toolchain_experiment.txt" % self._board 928d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_file = os.path.join (experiment_file_dir, 938d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_file_name) 948d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_header = """ 958d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt board: %s 968d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt remote: %s 978d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt retries: 1 988d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ % (self._board, self._remotes) 998d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_tests = """ 1008d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt benchmark: all_perfv2 { 1018d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt suite: telemetry_Crosperf 1028d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt iterations: 3 1038d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt } 1048d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 1058d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt with open(experiment_file, "w") as f: 1068d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt print >> f, experiment_header 1078d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt print >> f, experiment_tests 1088d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1098d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt # Now add vanilla to test file. 1108d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt official_image = """ 1118d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt vanilla_image { 1128d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt chromeos_root: %s 1138d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt build: %s 1148d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt } 1158d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ % (self._chromeos_root, vanilla_image) 1168d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt print >> f, official_image 1178d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1188d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt experiment_image = """ 1198d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt test_image { 1208d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt chromeos_root: %s 1218d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt build: %s 1228d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt } 1238d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ % (self._chromeos_root, trybot_image) 1248d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt print >> f, experiment_image 1258d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1268d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt crosperf = os.path.join(TOOLCHAIN_DIR, 1278d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "crosperf", 1288d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "crosperf") 1298d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt command = ("%s --no_email=True --results_dir=%s %s" % 1308d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt (crosperf, self._reports_dir, experiment_file)) 1318d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1328d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt ret = self._ce.RunCommand(command) 1338d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if ret != 0: 1348d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt raise RuntimeError("Couldn't run crosperf!") 1358d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt return 1368d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1378d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image): 1388d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 1398d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Put files in place for running seven-day reports. 1408d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1418d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt Create tar files of the custom and official images and copy them 1428d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt to the weekly reports directory, so they exist when the weekly report 1438d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt gets generated. IMPORTANT NOTE: This function must run *after* 1448d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt crosperf has been run; otherwise the vanilla images will not be there. 1458d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt """ 1468d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1478d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt dry_run = False 1488d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if (os.getlogin() != ROLE_ACCOUNT): 1498d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._l.LogOutput("Running this from non-role account; not copying " 1508d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "tar files for weekly reports.") 1518d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt dry_run = True 1528d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1538d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt images_path = os.path.join(os.path.realpath(self._chromeos_root), 1548d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "chroot/tmp") 1558d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1568d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board) 1578d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt dest_dir = os.path.join (data_dir, self._weekday) 1588d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if not os.path.exists(dest_dir): 1598d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt os.makedirs(dest_dir) 1608d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1618d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt # Make sure dest_dir is empty (clean out last week's data). 1628d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday) 1638d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if dry_run: 1648d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt print "CMD: %s" % cmd 1658d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt else: 1668d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt self._ce.RunCommand(cmd) 1678d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt 1688d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt # Now create new tar files and copy them over. 1698d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt labels = [ "test", "vanilla" ] 1708d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt for label_name in labels: 1718d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if label_name == "test": 1728d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt test_path = trybot_image 1738d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt else: 1748d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt test_path = vanilla_image 1758d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name) 1768d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; " 1778d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt "cp %s %s/.") % (images_path, 1788d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt tar_file_name, 1798d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt test_path, 1808d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt tar_file_name, 1818d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt dest_dir) 1828d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if dry_run: 1838d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt print "CMD: %s" % cmd 1848d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt tar_ret = 0 1858d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt else: 1868d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt tar_ret = self._ce.RunCommand(cmd) 1878d520ff1dc2da35cdca849e982051b86468016d8Dmitry Shmidt if tar_ret: 188 self._l.LogOutput("Error while creating/copying test tar file(%s)." 189 % tar_file_name) 190 191 def _SendEmail(self): 192 """Find email message generated by crosperf and send it.""" 193 filename = os.path.join(self._reports_dir, 194 "msg_body.html") 195 if (os.path.exists(filename) and 196 os.path.exists(os.path.expanduser(MAIL_PROGRAM))): 197 command = ('cat %s | %s -s "buildbot test results, %s" -team -html' 198 % (filename, MAIL_PROGRAM, self._board)) 199 self._ce.RunCommand(command) 200 201 def DoAll(self): 202 """ 203 Main function inside ToolchainComparator class. 204 205 Launch trybot, get image names, create crosperf experiment file, run 206 crosperf, and copy images into seven-day report directories. 207 """ 208 date_str = datetime.date.today() 209 description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH, 210 self._build, 211 date_str) 212 trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, 213 self._build, 214 [ USE_NEXT_GCC_PATCH ], 215 description, 216 build_toolchain=True) 217 218 vanilla_image = self._ParseVanillaImage(trybot_image) 219 220 print ("trybot_image: %s" % trybot_image) 221 print ("vanilla_image: %s" % vanilla_image) 222 if len(trybot_image) == 0: 223 self._l.LogError("Unable to find trybot_image for %s!" % description) 224 return 1 225 if len(vanilla_image) == 0: 226 self._l.LogError("Unable to find vanilla image for %s!" % description) 227 return 1 228 if os.getlogin() == ROLE_ACCOUNT: 229 self._FinishSetup() 230 231 self._TestImages(trybot_image, vanilla_image) 232 self._SendEmail() 233 # Only try to copy the image files if the test runs ran successfully. 234 self._CopyWeeklyReportFiles(trybot_image, vanilla_image) 235 return 0 236 237 238def Main(argv): 239 """The main function.""" 240 241 # Common initializations 242 command_executer.InitCommandExecuter() 243 parser = optparse.OptionParser() 244 parser.add_option("--remote", 245 dest="remote", 246 help="Remote machines to run tests on.") 247 parser.add_option("--board", 248 dest="board", 249 default="x86-zgb", 250 help="The target board.") 251 parser.add_option("--chromeos_root", 252 dest="chromeos_root", 253 help="The chromeos root from which to run tests.") 254 parser.add_option("--weekday", default="", 255 dest="weekday", 256 help="The day of the week for which to run tests.") 257 options, _ = parser.parse_args(argv) 258 if not options.board: 259 print "Please give a board." 260 return 1 261 if not options.remote: 262 print "Please give at least one remote machine." 263 return 1 264 if not options.chromeos_root: 265 print "Please specify the ChromeOS root directory." 266 return 1 267 268 fc = ToolchainComparator(options.board, options.remote, 269 options.chromeos_root, options.weekday) 270 return fc.DoAll() 271 272 273if __name__ == "__main__": 274 retval = Main(sys.argv) 275 sys.exit(retval) 276