results_cache.py revision 036c9233742004aa773a374df381b1cf137484f5
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Module to deal with result cache.""" 8 9import getpass 10import glob 11import hashlib 12import os 13import pickle 14import re 15import tempfile 16import json 17import sys 18 19from cros_utils import command_executer 20from cros_utils import misc 21 22from image_checksummer import ImageChecksummer 23 24import results_report 25import test_flag 26 27SCRATCH_DIR = os.path.expanduser("~/cros_scratch") 28RESULTS_FILE = "results.txt" 29MACHINE_FILE = "machine.txt" 30AUTOTEST_TARBALL = "autotest.tbz2" 31PERF_RESULTS_FILE = "perf-results.txt" 32CACHE_KEYS_FILE = "cache_keys.txt" 33 34class Result(object): 35 """ This class manages what exactly is stored inside the cache without knowing 36 what the key of the cache is. For runs with perf, it stores perf.data, 37 perf.report, etc. The key generation is handled by the ResultsCache class. 38 """ 39 40 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 41 self._chromeos_root = label.chromeos_root 42 self._logger = logger 43 self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger, 44 log_level=log_level) 45 self._temp_dir = None 46 self.label = label 47 self.results_dir = None 48 self.log_level = log_level 49 self.machine = machine 50 self.perf_data_files = [] 51 self.perf_report_files = [] 52 53 def _CopyFilesTo(self, dest_dir, files_to_copy): 54 file_index = 0 55 for file_to_copy in files_to_copy: 56 if not os.path.isdir(dest_dir): 57 command = "mkdir -p %s" % dest_dir 58 self._ce.RunCommand(command) 59 dest_file = os.path.join(dest_dir, 60 ("%s.%s" % (os.path.basename(file_to_copy), 61 file_index))) 62 ret = self._ce.CopyFiles(file_to_copy, 63 dest_file, 64 recursive=False) 65 if ret: 66 raise Exception("Could not copy results file: %s" % file_to_copy) 67 68 def CopyResultsTo(self, dest_dir): 69 self._CopyFilesTo(dest_dir, self.perf_data_files) 70 self._CopyFilesTo(dest_dir, self.perf_report_files) 71 if len(self.perf_data_files) or len(self.perf_report_files): 72 self._logger.LogOutput("Perf results files stored in %s." % dest_dir) 73 74 def _GetNewKeyvals(self, keyvals_dict): 75 # Initialize 'units' dictionary. 76 units_dict = {} 77 for k in keyvals_dict: 78 units_dict[k] = "" 79 results_files = self._GetDataMeasurementsFiles() 80 for f in results_files: 81 # Make sure we can find the results file 82 if os.path.exists(f): 83 data_filename = f 84 else: 85 # Otherwise get the base filename and create the correct 86 # path for it. 87 f_dir, f_base = misc.GetRoot(f) 88 data_filename = os.path.join(self._chromeos_root, "chroot/tmp", 89 self._temp_dir, f_base) 90 if os.path.exists(data_filename): 91 with open(data_filename, "r") as data_file: 92 lines = data_file.readlines() 93 for line in lines: 94 tmp_dict = json.loads(line) 95 graph_name = tmp_dict["graph"] 96 graph_str = (graph_name + "__") if graph_name else "" 97 key = graph_str + tmp_dict["description"] 98 keyvals_dict[key] = tmp_dict["value"] 99 units_dict[key] = tmp_dict["units"] 100 101 return keyvals_dict, units_dict 102 103 104 def _AppendTelemetryUnits(self, keyvals_dict, units_dict): 105 """ 106 keyvals_dict is the dictionary of key-value pairs that is used for 107 generating Crosperf reports. 108 109 units_dict is a dictionary of the units for the return values in 110 keyvals_dict. We need to associate the units with the return values, 111 for Telemetry tests, so that we can include the units in the reports. 112 This function takes each value in keyvals_dict, finds the corresponding 113 unit in the units_dict, and replaces the old value with a list of the 114 old value and the units. This later gets properly parsed in the 115 ResultOrganizer class, for generating the reports. 116 117 """ 118 119 results_dict = {} 120 for k in keyvals_dict: 121 # We don't want these lines in our reports; they add no useful data. 122 if k == "" or k == "telemetry_Crosperf": 123 continue 124 val = keyvals_dict[k] 125 units = units_dict[k] 126 new_val = [ val, units ] 127 results_dict[k] = new_val 128 return results_dict 129 130 def _GetKeyvals(self, show_all): 131 results_in_chroot = os.path.join(self._chromeos_root, 132 "chroot", "tmp") 133 if not self._temp_dir: 134 self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 135 command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir) 136 self._ce.RunCommand(command, print_to_console=False) 137 138 command = ("python generate_test_report --no-color --csv %s" % 139 (os.path.join("/tmp", os.path.basename(self._temp_dir)))) 140 _, out, _ = self._ce.ChrootRunCommandWOutput( 141 self._chromeos_root, command, print_to_console=False) 142 keyvals_dict = {} 143 tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root, 144 self._temp_dir) 145 for line in out.splitlines(): 146 tokens = re.split("=|,", line) 147 key = tokens[-2] 148 if key.startswith(tmp_dir_in_chroot): 149 key = key[len(tmp_dir_in_chroot) + 1:] 150 value = tokens[-1] 151 keyvals_dict[key] = value 152 153 # Check to see if there is a perf_measurements file and get the 154 # data from it if so. 155 keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict) 156 if self.suite == "telemetry_Crosperf": 157 # For telemtry_Crosperf results, append the units to the return 158 # results, for use in generating the reports. 159 keyvals_dict = self._AppendTelemetryUnits(keyvals_dict, 160 units_dict) 161 return keyvals_dict 162 163 def _GetResultsDir(self): 164 mo = re.search(r"Results placed in (\S+)", self.out) 165 if mo: 166 result = mo.group(1) 167 return result 168 raise Exception("Could not find results directory.") 169 170 def _FindFilesInResultsDir(self, find_args): 171 if not self.results_dir: 172 return None 173 174 command = "find %s %s" % (self.results_dir, 175 find_args) 176 ret, out, _ = self._ce.RunCommandWOutput(command, print_to_console=False) 177 if ret: 178 raise Exception("Could not run find command!") 179 return out 180 181 def _GetPerfDataFiles(self): 182 return self._FindFilesInResultsDir("-name perf.data").splitlines() 183 184 def _GetPerfReportFiles(self): 185 return self._FindFilesInResultsDir("-name perf.data.report").splitlines() 186 187 def _GetDataMeasurementsFiles(self): 188 return self._FindFilesInResultsDir("-name perf_measurements").splitlines() 189 190 def _GeneratePerfReportFiles(self): 191 perf_report_files = [] 192 for perf_data_file in self.perf_data_files: 193 # Generate a perf.report and store it side-by-side with the perf.data 194 # file. 195 chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root, 196 perf_data_file) 197 perf_report_file = "%s.report" % perf_data_file 198 if os.path.exists(perf_report_file): 199 raise Exception("Perf report file already exists: %s" % 200 perf_report_file) 201 chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root, 202 perf_report_file) 203 perf_path = os.path.join (self._chromeos_root, 204 "chroot", 205 "usr/bin/perf") 206 207 perf_file = "/usr/sbin/perf" 208 if os.path.exists(perf_path): 209 perf_file = "/usr/bin/perf" 210 211 # The following is a hack, to use the perf.static binary that 212 # was given to us by Stephane Eranian, until he can figure out 213 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 214 # Get the directory containing the 'crosperf' script. 215 dirname, _ = misc.GetRoot(sys.argv[0]) 216 perf_path = os.path.join (dirname, "..", "perf.static") 217 if os.path.exists(perf_path): 218 # copy the executable into the chroot so that it can be found. 219 src_path = perf_path 220 dst_path = os.path.join (self._chromeos_root, "chroot", 221 "tmp/perf.static") 222 command = "cp %s %s" % (src_path,dst_path) 223 self._ce.RunCommand (command) 224 perf_file = "/tmp/perf.static" 225 226 command = ("%s report " 227 "-n " 228 "--symfs /build/%s " 229 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux " 230 "--kallsyms /build/%s/boot/System.map-* " 231 "-i %s --stdio " 232 "> %s" % 233 (perf_file, 234 self._board, 235 self._board, 236 self._board, 237 chroot_perf_data_file, 238 chroot_perf_report_file)) 239 self._ce.ChrootRunCommand(self._chromeos_root, command) 240 241 # Add a keyval to the dictionary for the events captured. 242 perf_report_files.append( 243 misc.GetOutsideChrootPath(self._chromeos_root, 244 chroot_perf_report_file)) 245 return perf_report_files 246 247 def _GatherPerfResults(self): 248 report_id = 0 249 for perf_report_file in self.perf_report_files: 250 with open(perf_report_file, "r") as f: 251 report_contents = f.read() 252 for group in re.findall(r"Events: (\S+) (\S+)", report_contents): 253 num_events = group[0] 254 event_name = group[1] 255 key = "perf_%s_%s" % (report_id, event_name) 256 value = str(misc.UnitToNumber(num_events)) 257 self.keyvals[key] = value 258 259 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 260 self._board = self.label.board 261 self.out = out 262 self.err = err 263 self.retval = retval 264 self.test_name = test 265 self.suite = suite 266 self.chroot_results_dir = self._GetResultsDir() 267 self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root, 268 self.chroot_results_dir) 269 self.perf_data_files = self._GetPerfDataFiles() 270 # Include all perf.report data in table. 271 self.perf_report_files = self._GeneratePerfReportFiles() 272 # TODO(asharif): Do something similar with perf stat. 273 274 # Grab keyvals from the directory. 275 self._ProcessResults(show_all) 276 277 def _ProcessResults(self, show_all): 278 # Note that this function doesn't know anything about whether there is a 279 # cache hit or miss. It should process results agnostic of the cache hit 280 # state. 281 self.keyvals = self._GetKeyvals(show_all) 282 self.keyvals["retval"] = self.retval 283 # Generate report from all perf.data files. 284 # Now parse all perf report files and include them in keyvals. 285 self._GatherPerfResults() 286 287 def _PopulateFromCacheDir(self, cache_dir, show_all, test, suite): 288 self.test_name = test 289 self.suite = suite 290 # Read in everything from the cache directory. 291 with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: 292 self.out = pickle.load(f) 293 self.err = pickle.load(f) 294 self.retval = pickle.load(f) 295 296 # Untar the tarball to a temporary directory 297 self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root, 298 "chroot", "tmp")) 299 300 command = ("cd %s && tar xf %s" % 301 (self._temp_dir, 302 os.path.join(cache_dir, AUTOTEST_TARBALL))) 303 ret = self._ce.RunCommand(command, print_to_console=False) 304 if ret: 305 raise Exception("Could not untar cached tarball") 306 self.results_dir = self._temp_dir 307 self.perf_data_files = self._GetPerfDataFiles() 308 self.perf_report_files = self._GetPerfReportFiles() 309 self._ProcessResults(show_all) 310 311 def CleanUp(self, rm_chroot_tmp): 312 if rm_chroot_tmp and self.results_dir: 313 dirname, basename = misc.GetRoot(self.results_dir) 314 if basename.find("test_that_results_") != -1: 315 command = "rm -rf %s" % self.results_dir 316 else: 317 command = "rm -rf %s" % dirname 318 self._ce.RunCommand(command) 319 if self._temp_dir: 320 command = "rm -rf %s" % self._temp_dir 321 self._ce.RunCommand(command) 322 323 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 324 # Create the dir if it doesn't exist. 325 temp_dir = tempfile.mkdtemp() 326 327 # Store to the temp directory. 328 with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f: 329 pickle.dump(self.out, f) 330 pickle.dump(self.err, f) 331 pickle.dump(self.retval, f) 332 333 if not test_flag.GetTestMode(): 334 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), "w") as f: 335 f.write("%s\n" % self.label.name) 336 f.write("%s\n" % self.label.chrome_version) 337 f.write("%s\n" % self.machine.checksum_string) 338 for k in key_list: 339 f.write(k) 340 f.write("\n") 341 342 if self.results_dir: 343 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 344 command = ("cd %s && " 345 "tar " 346 "--exclude=var/spool " 347 "--exclude=var/log " 348 "-cjf %s ." % (self.results_dir, tarball)) 349 ret = self._ce.RunCommand(command) 350 if ret: 351 raise Exception("Couldn't store autotest output directory.") 352 # Store machine info. 353 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 354 # this function. 355 with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: 356 f.write(machine_manager.machine_checksum_string[self.label.name]) 357 358 if os.path.exists(cache_dir): 359 command = "rm -rf {0}".format(cache_dir) 360 self._ce.RunCommand(command) 361 362 command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir)) 363 command += "chmod g+x {0} && ".format(temp_dir) 364 command += "mv {0} {1}".format(temp_dir, cache_dir) 365 ret = self._ce.RunCommand(command) 366 if ret: 367 command = "rm -rf {0}".format(temp_dir) 368 self._ce.RunCommand(command) 369 raise Exception("Could not move dir %s to dir %s" % 370 (temp_dir, cache_dir)) 371 372 @classmethod 373 def CreateFromRun(cls, logger, log_level, label, machine, out, err, retval, 374 show_all, test, suite="telemetry_Crosperf"): 375 if suite == "telemetry": 376 result = TelemetryResult(logger, label, log_level, machine) 377 else: 378 result = cls(logger, label, log_level, machine) 379 result._PopulateFromRun(out, err, retval, show_all, test, suite) 380 return result 381 382 @classmethod 383 def CreateFromCacheHit(cls, logger, log_level, label, machine, cache_dir, 384 show_all, test, suite="telemetry_Crosperf"): 385 if suite == "telemetry": 386 result = TelemetryResult(logger, label, log_level, machine) 387 else: 388 result = cls(logger, label, log_level, machine) 389 try: 390 result._PopulateFromCacheDir(cache_dir, show_all, test, suite) 391 392 except Exception as e: 393 logger.LogError("Exception while using cache: %s" % e) 394 return None 395 return result 396 397 398class TelemetryResult(Result): 399 400 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 401 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 402 cmd_exec) 403 404 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 405 self.out = out 406 self.err = err 407 self.retval = retval 408 409 self._ProcessResults() 410 411 def _ProcessResults(self): 412 # The output is: 413 # url,average_commit_time (ms),... 414 # www.google.com,33.4,21.2,... 415 # We need to convert to this format: 416 # {"www.google.com:average_commit_time (ms)": "33.4", 417 # "www.google.com:...": "21.2"} 418 # Added note: Occasionally the output comes back 419 # with "JSON.stringify(window.automation.GetResults())" on 420 # the first line, and then the rest of the output as 421 # described above. 422 423 lines = self.out.splitlines() 424 self.keyvals = {} 425 426 if lines: 427 if lines[0].startswith("JSON.stringify"): 428 lines = lines[1:] 429 430 if not lines: 431 return 432 labels = lines[0].split(",") 433 for line in lines[1:]: 434 fields = line.split(",") 435 if len(fields) != len(labels): 436 continue 437 for i in range(1, len(labels)): 438 key = "%s %s" % (fields[0], labels[i]) 439 value = fields[i] 440 self.keyvals[key] = value 441 self.keyvals["retval"] = self.retval 442 443 def _PopulateFromCacheDir(self, cache_dir): 444 with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: 445 self.out = pickle.load(f) 446 self.err = pickle.load(f) 447 self.retval = pickle.load(f) 448 self._ProcessResults() 449 450 451class CacheConditions(object): 452 # Cache hit only if the result file exists. 453 CACHE_FILE_EXISTS = 0 454 455 # Cache hit if the checksum of cpuinfo and totalmem of 456 # the cached result and the new run match. 457 MACHINES_MATCH = 1 458 459 # Cache hit if the image checksum of the cached result and the new run match. 460 CHECKSUMS_MATCH = 2 461 462 # Cache hit only if the cached result was successful 463 RUN_SUCCEEDED = 3 464 465 # Never a cache hit. 466 FALSE = 4 467 468 # Cache hit if the image path matches the cached image path. 469 IMAGE_PATH_MATCH = 5 470 471 # Cache hit if the uuid of hard disk mataches the cached one 472 473 SAME_MACHINE_MATCH = 6 474 475 476class ResultsCache(object): 477 478 """ This class manages the key of the cached runs without worrying about what 479 is exactly stored (value). The value generation is handled by the Results 480 class. 481 """ 482 CACHE_VERSION = 6 483 484 def Init(self, chromeos_image, chromeos_root, test_name, iteration, 485 test_args, profiler_args, machine_manager, machine, board, 486 cache_conditions, logger_to_use, log_level, label, share_cache, 487 suite, show_all_results, run_local): 488 self.chromeos_image = chromeos_image 489 self.chromeos_root = chromeos_root 490 self.test_name = test_name 491 self.iteration = iteration 492 self.test_args = test_args 493 self.profiler_args = profiler_args 494 self.board = board 495 self.cache_conditions = cache_conditions 496 self.machine_manager = machine_manager 497 self.machine = machine 498 self._logger = logger_to_use 499 self._ce = command_executer.GetCommandExecuter(self._logger, 500 log_level=log_level) 501 self.label = label 502 self.share_cache = share_cache 503 self.suite = suite 504 self.log_level = log_level 505 self.show_all = show_all_results 506 self.run_local = run_local 507 508 def _GetCacheDirForRead(self): 509 matching_dirs = [] 510 for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)): 511 matching_dirs += glob.glob(glob_path) 512 513 if matching_dirs: 514 # Cache file found. 515 return matching_dirs[0] 516 else: 517 return None 518 519 def _GetCacheDirForWrite(self, get_keylist=False): 520 cache_path = self._FormCacheDir(self._GetCacheKeyList(False))[0] 521 if get_keylist: 522 args_str = "%s_%s_%s" % (self.test_args, 523 self.profiler_args, 524 self.run_local) 525 version, image = results_report.ParseChromeosImage( 526 self.label.chromeos_image) 527 keylist = [version, image, self.label.board, 528 self.machine.name, self.test_name, str(self.iteration), 529 args_str] 530 return cache_path, keylist 531 return cache_path 532 533 def _FormCacheDir(self, list_of_strings): 534 cache_key = " ".join(list_of_strings) 535 cache_dir = misc.GetFilenameFromString(cache_key) 536 if self.label.cache_dir: 537 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 538 cache_path = [os.path.join(cache_home, cache_dir)] 539 else: 540 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 541 542 if len(self.share_cache): 543 for path in [x.strip() for x in self.share_cache.split(",")]: 544 if os.path.exists(path): 545 cache_path.append(os.path.join(path, cache_dir)) 546 else: 547 self._logger.LogFatal("Unable to find shared cache: %s" % path) 548 549 return cache_path 550 551 def _GetCacheKeyList(self, read): 552 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 553 machine_checksum = "*" 554 else: 555 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 556 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 557 checksum = "*" 558 elif self.label.image_type == "trybot": 559 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 560 elif self.label.image_type == "official": 561 checksum = "*" 562 else: 563 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 564 565 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 566 image_path_checksum = "*" 567 else: 568 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 569 570 machine_id_checksum = "" 571 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 572 machine_id_checksum = "*" 573 else: 574 if self.machine and self.machine.name in self.label.remote: 575 machine_id_checksum = self.machine.machine_id_checksum 576 else: 577 for machine in self.machine_manager.GetMachines(self.label): 578 if machine.name == self.label.remote[0]: 579 machine_id_checksum = machine.machine_id_checksum 580 break 581 582 temp_test_args = "%s %s %s" % (self.test_args, 583 self.profiler_args, 584 self.run_local) 585 test_args_checksum = hashlib.md5( 586 "".join(temp_test_args)).hexdigest() 587 return (image_path_checksum, 588 self.test_name, str(self.iteration), 589 test_args_checksum, 590 checksum, 591 machine_checksum, 592 machine_id_checksum, 593 str(self.CACHE_VERSION)) 594 595 def ReadResult(self): 596 if CacheConditions.FALSE in self.cache_conditions: 597 cache_dir = self._GetCacheDirForWrite() 598 command = "rm -rf {0}".format(cache_dir) 599 self._ce.RunCommand(command) 600 return None 601 cache_dir = self._GetCacheDirForRead() 602 603 if not cache_dir: 604 return None 605 606 if not os.path.isdir(cache_dir): 607 return None 608 609 if self.log_level == 'verbose': 610 self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir) 611 result = Result.CreateFromCacheHit(self._logger, 612 self.log_level, 613 self.label, 614 self.machine, 615 cache_dir, 616 self.show_all, 617 self.test_name, 618 self.suite) 619 if not result: 620 return None 621 622 if (result.retval == 0 or 623 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 624 return result 625 626 return None 627 628 def StoreResult(self, result): 629 cache_dir, keylist = self._GetCacheDirForWrite(get_keylist=True) 630 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 631 632 633class MockResultsCache(ResultsCache): 634 def Init(self, *args): 635 pass 636 637 def ReadResult(self): 638 return None 639 640 def StoreResult(self, result): 641 pass 642 643 644class MockResult(Result): 645 646 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 647 self.out = out 648 self.err = err 649 self.retval = retval 650