results_cache.py revision 5ea9f006b5de0d882d5b51da243806b7cac69938
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Module to deal with result cache.""" 8 9import getpass 10import glob 11import hashlib 12import os 13import pickle 14import re 15import tempfile 16import json 17import sys 18 19from utils import command_executer 20from utils import misc 21 22from image_checksummer import ImageChecksummer 23 24import results_report 25 26SCRATCH_DIR = os.path.expanduser("~/cros_scratch") 27RESULTS_FILE = "results.txt" 28MACHINE_FILE = "machine.txt" 29AUTOTEST_TARBALL = "autotest.tbz2" 30PERF_RESULTS_FILE = "perf-results.txt" 31CACHE_KEYS_FILE = "cache_keys.txt" 32 33class Result(object): 34 """ This class manages what exactly is stored inside the cache without knowing 35 what the key of the cache is. For runs with perf, it stores perf.data, 36 perf.report, etc. The key generation is handled by the ResultsCache class. 37 """ 38 39 def __init__(self, logger, label, log_level, cmd_exec=None): 40 self._chromeos_root = label.chromeos_root 41 self._logger = logger 42 self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger, 43 log_level=log_level) 44 self._temp_dir = None 45 self.label = label 46 self.results_dir = None 47 self.log_level = log_level 48 self.perf_data_files = [] 49 self.perf_report_files = [] 50 51 def _CopyFilesTo(self, dest_dir, files_to_copy): 52 file_index = 0 53 for file_to_copy in files_to_copy: 54 if not os.path.isdir(dest_dir): 55 command = "mkdir -p %s" % dest_dir 56 self._ce.RunCommand(command) 57 dest_file = os.path.join(dest_dir, 58 ("%s.%s" % (os.path.basename(file_to_copy), 59 file_index))) 60 ret = self._ce.CopyFiles(file_to_copy, 61 dest_file, 62 recursive=False) 63 if ret: 64 raise Exception("Could not copy results file: %s" % file_to_copy) 65 66 def CopyResultsTo(self, dest_dir): 67 self._CopyFilesTo(dest_dir, self.perf_data_files) 68 self._CopyFilesTo(dest_dir, self.perf_report_files) 69 if len(self.perf_data_files) or len(self.perf_report_files): 70 self._logger.LogOutput("Perf results files stored in %s." % dest_dir) 71 72 def _GetNewKeyvals(self, keyvals_dict): 73 # Initialize 'units' dictionary. 74 units_dict = {} 75 for k in keyvals_dict: 76 units_dict[k] = "" 77 results_files = self._GetDataMeasurementsFiles() 78 for f in results_files: 79 # Make sure we can find the results file 80 if os.path.exists(f): 81 data_filename = f 82 else: 83 # Otherwise get the base filename and create the correct 84 # path for it. 85 f_dir, f_base = misc.GetRoot(f) 86 data_filename = os.path.join(self._chromeos_root, "chroot/tmp", 87 self._temp_dir, f_base) 88 if os.path.exists(data_filename): 89 with open(data_filename, "r") as data_file: 90 lines = data_file.readlines() 91 for line in lines: 92 tmp_dict = json.loads(line) 93 graph_name = tmp_dict["graph"] 94 graph_str = (graph_name + "__") if graph_name else "" 95 key = graph_str + tmp_dict["description"] 96 keyvals_dict[key] = tmp_dict["value"] 97 units_dict[key] = tmp_dict["units"] 98 99 return keyvals_dict, units_dict 100 101 102 def _AppendTelemetryUnits(self, keyvals_dict, units_dict): 103 """ 104 keyvals_dict is the dictionary of key-value pairs that is used for 105 generating Crosperf reports. 106 107 units_dict is a dictionary of the units for the return values in 108 keyvals_dict. We need to associate the units with the return values, 109 for Telemetry tests, so that we can include the units in the reports. 110 This function takes each value in keyvals_dict, finds the corresponding 111 unit in the units_dict, and replaces the old value with a list of the 112 old value and the units. This later gets properly parsed in the 113 ResultOrganizer class, for generating the reports. 114 115 """ 116 117 results_dict = {} 118 for k in keyvals_dict: 119 # We don't want these lines in our reports; they add no useful data. 120 if k == "" or k == "telemetry_Crosperf": 121 continue 122 val = keyvals_dict[k] 123 units = units_dict[k] 124 new_val = [ val, units ] 125 results_dict[k] = new_val 126 return results_dict 127 128 def _GetKeyvals(self, show_all): 129 results_in_chroot = os.path.join(self._chromeos_root, 130 "chroot", "tmp") 131 if not self._temp_dir: 132 self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 133 command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir) 134 self._ce.RunCommand(command) 135 136 command = ("python generate_test_report --no-color --csv %s" % 137 (os.path.join("/tmp", os.path.basename(self._temp_dir)))) 138 [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root, 139 command, 140 return_output=True) 141 keyvals_dict = {} 142 tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root, 143 self._temp_dir) 144 for line in out.splitlines(): 145 tokens = re.split("=|,", line) 146 key = tokens[-2] 147 if key.startswith(tmp_dir_in_chroot): 148 key = key[len(tmp_dir_in_chroot) + 1:] 149 value = tokens[-1] 150 keyvals_dict[key] = value 151 152 # Check to see if there is a perf_measurements file and get the 153 # data from it if so. 154 keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict) 155 if self.suite == "telemetry_Crosperf": 156 # For telemtry_Crosperf results, append the units to the return 157 # results, for use in generating the reports. 158 keyvals_dict = self._AppendTelemetryUnits(keyvals_dict, 159 units_dict) 160 return keyvals_dict 161 162 def _GetResultsDir(self): 163 mo = re.search(r"Results placed in (\S+)", self.out) 164 if mo: 165 result = mo.group(1) 166 return result 167 raise Exception("Could not find results directory.") 168 169 def _FindFilesInResultsDir(self, find_args): 170 if not self.results_dir: 171 return None 172 173 command = "find %s %s" % (self.results_dir, 174 find_args) 175 ret, out, _ = self._ce.RunCommand(command, return_output=True) 176 if ret: 177 raise Exception("Could not run find command!") 178 return out 179 180 def _GetPerfDataFiles(self): 181 return self._FindFilesInResultsDir("-name perf.data").splitlines() 182 183 def _GetPerfReportFiles(self): 184 return self._FindFilesInResultsDir("-name perf.data.report").splitlines() 185 186 def _GetDataMeasurementsFiles(self): 187 return self._FindFilesInResultsDir("-name perf_measurements").splitlines() 188 189 def _GeneratePerfReportFiles(self): 190 perf_report_files = [] 191 for perf_data_file in self.perf_data_files: 192 # Generate a perf.report and store it side-by-side with the perf.data 193 # file. 194 chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root, 195 perf_data_file) 196 perf_report_file = "%s.report" % perf_data_file 197 if os.path.exists(perf_report_file): 198 raise Exception("Perf report file already exists: %s" % 199 perf_report_file) 200 chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root, 201 perf_report_file) 202 perf_path = os.path.join (self._chromeos_root, 203 "chroot", 204 "usr/bin/perf") 205 206 perf_file = "/usr/sbin/perf" 207 if os.path.exists(perf_path): 208 perf_file = "/usr/bin/perf" 209 210 # The following is a hack, to use the perf.static binary that 211 # was given to us by Stephane Eranian, until he can figure out 212 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 213 # Get the directory containing the 'crosperf' script. 214 dirname, _ = misc.GetRoot(sys.argv[0]) 215 perf_path = os.path.join (dirname, "..", "perf.static") 216 if os.path.exists(perf_path): 217 # copy the executable into the chroot so that it can be found. 218 src_path = perf_path 219 dst_path = os.path.join (self._chromeos_root, "chroot", 220 "tmp/perf.static") 221 command = "cp %s %s" % (src_path,dst_path) 222 self._ce.RunCommand (command) 223 perf_file = "/tmp/perf.static" 224 225 command = ("%s report " 226 "-n " 227 "--symfs /build/%s " 228 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux " 229 "--kallsyms /build/%s/boot/System.map-* " 230 "-i %s --stdio " 231 "> %s" % 232 (perf_file, 233 self._board, 234 self._board, 235 self._board, 236 chroot_perf_data_file, 237 chroot_perf_report_file)) 238 self._ce.ChrootRunCommand(self._chromeos_root, 239 command) 240 241 # Add a keyval to the dictionary for the events captured. 242 perf_report_files.append( 243 misc.GetOutsideChrootPath(self._chromeos_root, 244 chroot_perf_report_file)) 245 return perf_report_files 246 247 def _GatherPerfResults(self): 248 report_id = 0 249 for perf_report_file in self.perf_report_files: 250 with open(perf_report_file, "r") as f: 251 report_contents = f.read() 252 for group in re.findall(r"Events: (\S+) (\S+)", report_contents): 253 num_events = group[0] 254 event_name = group[1] 255 key = "perf_%s_%s" % (report_id, event_name) 256 value = str(misc.UnitToNumber(num_events)) 257 self.keyvals[key] = value 258 259 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 260 self._board = self.label.board 261 self.out = out 262 self.err = err 263 self.retval = retval 264 self.test_name = test 265 self.suite = suite 266 self.chroot_results_dir = self._GetResultsDir() 267 self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root, 268 self.chroot_results_dir) 269 self.perf_data_files = self._GetPerfDataFiles() 270 # Include all perf.report data in table. 271 self.perf_report_files = self._GeneratePerfReportFiles() 272 # TODO(asharif): Do something similar with perf stat. 273 274 # Grab keyvals from the directory. 275 self._ProcessResults(show_all) 276 277 def _ProcessResults(self, show_all): 278 # Note that this function doesn't know anything about whether there is a 279 # cache hit or miss. It should process results agnostic of the cache hit 280 # state. 281 self.keyvals = self._GetKeyvals(show_all) 282 self.keyvals["retval"] = self.retval 283 # Generate report from all perf.data files. 284 # Now parse all perf report files and include them in keyvals. 285 self._GatherPerfResults() 286 287 def _PopulateFromCacheDir(self, cache_dir, show_all, test, suite): 288 self.test_name = test 289 self.suite = suite 290 # Read in everything from the cache directory. 291 with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: 292 self.out = pickle.load(f) 293 self.err = pickle.load(f) 294 self.retval = pickle.load(f) 295 296 # Untar the tarball to a temporary directory 297 self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root, 298 "chroot", "tmp")) 299 300 command = ("cd %s && tar xf %s" % 301 (self._temp_dir, 302 os.path.join(cache_dir, AUTOTEST_TARBALL))) 303 ret = self._ce.RunCommand(command) 304 if ret: 305 raise Exception("Could not untar cached tarball") 306 self.results_dir = self._temp_dir 307 self.perf_data_files = self._GetPerfDataFiles() 308 self.perf_report_files = self._GetPerfReportFiles() 309 self._ProcessResults(show_all) 310 311 def CleanUp(self, rm_chroot_tmp): 312 if rm_chroot_tmp and self.results_dir: 313 dirname, basename = misc.GetRoot(self.results_dir) 314 if basename.find("test_that_results_") != -1: 315 command = "rm -rf %s" % self.results_dir 316 else: 317 command = "rm -rf %s" % dirname 318 self._ce.RunCommand(command) 319 if self._temp_dir: 320 command = "rm -rf %s" % self._temp_dir 321 self._ce.RunCommand(command) 322 323 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 324 # Create the dir if it doesn't exist. 325 temp_dir = tempfile.mkdtemp() 326 327 # Store to the temp directory. 328 with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f: 329 pickle.dump(self.out, f) 330 pickle.dump(self.err, f) 331 pickle.dump(self.retval, f) 332 333 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), "w") as f: 334 for k in key_list: 335 f.write(k) 336 f.write("\n") 337 338 if self.results_dir: 339 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 340 command = ("cd %s && " 341 "tar " 342 "--exclude=var/spool " 343 "--exclude=var/log " 344 "-cjf %s ." % (self.results_dir, tarball)) 345 ret = self._ce.RunCommand(command) 346 if ret: 347 raise Exception("Couldn't store autotest output directory.") 348 # Store machine info. 349 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 350 # this function. 351 with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: 352 f.write(machine_manager.machine_checksum_string[self.label.name]) 353 354 if os.path.exists(cache_dir): 355 command = "rm -rf {0}".format(cache_dir) 356 self._ce.RunCommand(command) 357 358 command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir)) 359 command += "chmod g+x {0} && ".format(temp_dir) 360 command += "mv {0} {1}".format(temp_dir, cache_dir) 361 ret = self._ce.RunCommand(command) 362 if ret: 363 command = "rm -rf {0}".format(temp_dir) 364 self._ce.RunCommand(command) 365 raise Exception("Could not move dir %s to dir %s" % 366 (temp_dir, cache_dir)) 367 368 @classmethod 369 def CreateFromRun(cls, logger, log_level, label, out, err, retval, show_all, 370 test, suite="telemetry_Crosperf"): 371 if suite == "telemetry": 372 result = TelemetryResult(logger, label, log_level) 373 else: 374 result = cls(logger, label, log_level) 375 result._PopulateFromRun(out, err, retval, show_all, test, suite) 376 return result 377 378 @classmethod 379 def CreateFromCacheHit(cls, logger, log_level, label, cache_dir, 380 show_all, test, suite="telemetry_Crosperf"): 381 if suite == "telemetry": 382 result = TelemetryResult(logger, label) 383 else: 384 result = cls(logger, label, log_level) 385 try: 386 result._PopulateFromCacheDir(cache_dir, show_all, test, suite) 387 388 except Exception as e: 389 logger.LogError("Exception while using cache: %s" % e) 390 return None 391 return result 392 393 394class TelemetryResult(Result): 395 396 def __init__(self, logger, label, log_level, cmd_exec=None): 397 super(TelemetryResult, self).__init__(logger, label, log_level, cmd_exec) 398 399 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 400 self.out = out 401 self.err = err 402 self.retval = retval 403 404 self._ProcessResults() 405 406 def _ProcessResults(self): 407 # The output is: 408 # url,average_commit_time (ms),... 409 # www.google.com,33.4,21.2,... 410 # We need to convert to this format: 411 # {"www.google.com:average_commit_time (ms)": "33.4", 412 # "www.google.com:...": "21.2"} 413 # Added note: Occasionally the output comes back 414 # with "JSON.stringify(window.automation.GetResults())" on 415 # the first line, and then the rest of the output as 416 # described above. 417 418 lines = self.out.splitlines() 419 self.keyvals = {} 420 421 if lines: 422 if lines[0].startswith("JSON.stringify"): 423 lines = lines[1:] 424 425 if not lines: 426 return 427 labels = lines[0].split(",") 428 for line in lines[1:]: 429 fields = line.split(",") 430 if len(fields) != len(labels): 431 continue 432 for i in range(1, len(labels)): 433 key = "%s %s" % (fields[0], labels[i]) 434 value = fields[i] 435 self.keyvals[key] = value 436 self.keyvals["retval"] = self.retval 437 438 def _PopulateFromCacheDir(self, cache_dir): 439 with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: 440 self.out = pickle.load(f) 441 self.err = pickle.load(f) 442 self.retval = pickle.load(f) 443 self._ProcessResults() 444 445 446class CacheConditions(object): 447 # Cache hit only if the result file exists. 448 CACHE_FILE_EXISTS = 0 449 450 # Cache hit if the checksum of cpuinfo and totalmem of 451 # the cached result and the new run match. 452 MACHINES_MATCH = 1 453 454 # Cache hit if the image checksum of the cached result and the new run match. 455 CHECKSUMS_MATCH = 2 456 457 # Cache hit only if the cached result was successful 458 RUN_SUCCEEDED = 3 459 460 # Never a cache hit. 461 FALSE = 4 462 463 # Cache hit if the image path matches the cached image path. 464 IMAGE_PATH_MATCH = 5 465 466 # Cache hit if the uuid of hard disk mataches the cached one 467 468 SAME_MACHINE_MATCH = 6 469 470 471class ResultsCache(object): 472 473 """ This class manages the key of the cached runs without worrying about what 474 is exactly stored (value). The value generation is handled by the Results 475 class. 476 """ 477 CACHE_VERSION = 6 478 479 def Init(self, chromeos_image, chromeos_root, test_name, iteration, 480 test_args, profiler_args, machine_manager, machine, board, 481 cache_conditions, logger_to_use, log_level, label, share_cache, 482 suite, show_all_results, run_local): 483 self.chromeos_image = chromeos_image 484 self.chromeos_root = chromeos_root 485 self.test_name = test_name 486 self.iteration = iteration 487 self.test_args = test_args 488 self.profiler_args = profiler_args 489 self.board = board 490 self.cache_conditions = cache_conditions 491 self.machine_manager = machine_manager 492 self.machine = machine 493 self._logger = logger_to_use 494 self._ce = command_executer.GetCommandExecuter(self._logger, 495 log_level=log_level) 496 self.label = label 497 self.share_cache = share_cache 498 self.suite = suite 499 self.log_level = log_level 500 self.show_all = show_all_results 501 self.run_local = run_local 502 503 def _GetCacheDirForRead(self): 504 matching_dirs = [] 505 for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)): 506 matching_dirs += glob.glob(glob_path) 507 508 if matching_dirs: 509 # Cache file found. 510 return matching_dirs[0] 511 else: 512 return None 513 514 def _GetCacheDirForWrite(self, get_keylist=False): 515 cache_path = self._FormCacheDir(self._GetCacheKeyList(False))[0] 516 if get_keylist: 517 args_str = "%s_%s_%s" % (self.test_args, self.profiler_args, self.run_local) 518 version, image = results_report.ParseChromeosImage(self.label.chromeos_image) 519 keylist = [ version, image, self.label.board, 520 self.machine.name, self.test_name, str(self.iteration), 521 args_str] 522 return cache_path, keylist 523 return cache_path 524 525 def _FormCacheDir(self, list_of_strings): 526 cache_key = " ".join(list_of_strings) 527 cache_dir = misc.GetFilenameFromString(cache_key) 528 if self.label.cache_dir: 529 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 530 cache_path = [os.path.join(cache_home, cache_dir)] 531 else: 532 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 533 534 if len(self.share_cache): 535 for path in [x.strip() for x in self.share_cache.split(",")]: 536 if os.path.exists(path): 537 cache_path.append(os.path.join(path, cache_dir)) 538 else: 539 self._logger.LogFatal("Unable to find shared cache: %s" % path) 540 541 return cache_path 542 543 def _GetCacheKeyList(self, read): 544 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 545 machine_checksum = "*" 546 else: 547 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 548 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 549 checksum = "*" 550 elif self.label.image_type == "trybot": 551 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 552 elif self.label.image_type == "official": 553 checksum = "*" 554 else: 555 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 556 557 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 558 image_path_checksum = "*" 559 else: 560 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 561 562 machine_id_checksum = "" 563 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 564 machine_id_checksum = "*" 565 else: 566 if self.machine and self.machine.name in self.label.remote: 567 machine_id_checksum = self.machine.machine_id_checksum 568 else: 569 for machine in self.machine_manager.GetMachines(self.label): 570 if machine.name == self.label.remote[0]: 571 machine_id_checksum = machine.machine_id_checksum 572 break 573 574 temp_test_args = "%s %s %s" % (self.test_args, self.profiler_args, self.run_local) 575 test_args_checksum = hashlib.md5( 576 "".join(temp_test_args)).hexdigest() 577 return (image_path_checksum, 578 self.test_name, str(self.iteration), 579 test_args_checksum, 580 checksum, 581 machine_checksum, 582 machine_id_checksum, 583 str(self.CACHE_VERSION)) 584 585 def ReadResult(self): 586 if CacheConditions.FALSE in self.cache_conditions: 587 cache_dir = self._GetCacheDirForWrite() 588 command = "rm -rf {0}".format(cache_dir) 589 self._ce.RunCommand(command) 590 return None 591 cache_dir = self._GetCacheDirForRead() 592 593 if not cache_dir: 594 return None 595 596 if not os.path.isdir(cache_dir): 597 return None 598 599 self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir) 600 result = Result.CreateFromCacheHit(self._logger, 601 self.log_level, 602 self.label, 603 cache_dir, 604 self.show_all, 605 self.test_name, 606 self.suite) 607 if not result: 608 return None 609 610 if (result.retval == 0 or 611 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 612 return result 613 614 return None 615 616 def StoreResult(self, result): 617 cache_dir, keylist = self._GetCacheDirForWrite(get_keylist=True) 618 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 619 620 621class MockResultsCache(ResultsCache): 622 def Init(self, *args): 623 pass 624 625 def ReadResult(self): 626 return None 627 628 def StoreResult(self, result): 629 pass 630 631 632class MockResult(Result): 633 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 634 self.out = out 635 self.err = err 636 self.retval = retval 637