results_cache.py revision bc2d3d13d2e1928217140a76acdf9f9917b1fa30
1#!/usr/bin/python 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Module to deal with result cache.""" 8 9import getpass 10import glob 11import hashlib 12import os 13import pickle 14import re 15import tempfile 16import json 17import sys 18 19from utils import command_executer 20from utils import misc 21 22from image_checksummer import ImageChecksummer 23 24SCRATCH_DIR = os.path.expanduser("~/cros_scratch") 25RESULTS_FILE = "results.txt" 26MACHINE_FILE = "machine.txt" 27AUTOTEST_TARBALL = "autotest.tbz2" 28PERF_RESULTS_FILE = "perf-results.txt" 29 30class Result(object): 31 """ This class manages what exactly is stored inside the cache without knowing 32 what the key of the cache is. For runs with perf, it stores perf.data, 33 perf.report, etc. The key generation is handled by the ResultsCache class. 34 """ 35 36 def __init__(self, logger, label, log_level, cmd_exec=None): 37 self._chromeos_root = label.chromeos_root 38 self._logger = logger 39 self._ce = cmd_exec or command_executer.GetCommandExecuter(self._logger, 40 log_level=log_level) 41 self._temp_dir = None 42 self.label = label 43 self.results_dir = None 44 self.log_level = log_level 45 self.perf_data_files = [] 46 self.perf_report_files = [] 47 48 def _CopyFilesTo(self, dest_dir, files_to_copy): 49 file_index = 0 50 for file_to_copy in files_to_copy: 51 if not os.path.isdir(dest_dir): 52 command = "mkdir -p %s" % dest_dir 53 self._ce.RunCommand(command) 54 dest_file = os.path.join(dest_dir, 55 ("%s.%s" % (os.path.basename(file_to_copy), 56 file_index))) 57 ret = self._ce.CopyFiles(file_to_copy, 58 dest_file, 59 recursive=False) 60 if ret: 61 raise Exception("Could not copy results file: %s" % file_to_copy) 62 63 def CopyResultsTo(self, dest_dir): 64 self._CopyFilesTo(dest_dir, self.perf_data_files) 65 self._CopyFilesTo(dest_dir, self.perf_report_files) 66 if len(self.perf_data_files) or len(self.perf_report_files): 67 self._logger.LogOutput("Perf results files stored in %s." % dest_dir) 68 69 def _GetNewKeyvals(self, keyvals_dict): 70 # Initialize 'units' dictionary. 71 units_dict = {} 72 for k in keyvals_dict: 73 units_dict[k] = "" 74 results_files = self._GetDataMeasurementsFiles() 75 for f in results_files: 76 # Make sure we can find the results file 77 if os.path.exists(f): 78 data_filename = f 79 else: 80 # Otherwise get the base filename and create the correct 81 # path for it. 82 f_dir, f_base = misc.GetRoot(f) 83 data_filename = os.path.join(self._chromeos_root, "chroot/tmp", 84 self._temp_dir, f_base) 85 if os.path.exists(data_filename): 86 with open(data_filename, "r") as data_file: 87 lines = data_file.readlines() 88 for line in lines: 89 tmp_dict = json.loads(line) 90 graph_name = tmp_dict["graph"] 91 graph_str = (graph_name + "__") if graph_name else "" 92 key = graph_str + tmp_dict["description"] 93 keyvals_dict[key] = tmp_dict["value"] 94 units_dict[key] = tmp_dict["units"] 95 96 return keyvals_dict, units_dict 97 98 99 def _AppendTelemetryUnits(self, keyvals_dict, units_dict): 100 """ 101 keyvals_dict is the dictionary of key-value pairs that is used for 102 generating Crosperf reports. 103 104 units_dict is a dictionary of the units for the return values in 105 keyvals_dict. We need to associate the units with the return values, 106 for Telemetry tests, so that we can include the units in the reports. 107 This function takes each value in keyvals_dict, finds the corresponding 108 unit in the units_dict, and replaces the old value with a list of the 109 old value and the units. This later gets properly parsed in the 110 ResultOrganizer class, for generating the reports. 111 112 """ 113 114 results_dict = {} 115 for k in keyvals_dict: 116 # We don't want these lines in our reports; they add no useful data. 117 if k == "" or k == "telemetry_Crosperf": 118 continue 119 val = keyvals_dict[k] 120 units = units_dict[k] 121 new_val = [ val, units ] 122 results_dict[k] = new_val 123 return results_dict 124 125 def _GetKeyvals(self, show_all): 126 results_in_chroot = os.path.join(self._chromeos_root, 127 "chroot", "tmp") 128 if not self._temp_dir: 129 self._temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 130 command = "cp -r {0}/* {1}".format(self.results_dir, self._temp_dir) 131 self._ce.RunCommand(command) 132 133 command = ("python generate_test_report --no-color --csv %s" % 134 (os.path.join("/tmp", os.path.basename(self._temp_dir)))) 135 [_, out, _] = self._ce.ChrootRunCommand(self._chromeos_root, 136 command, 137 return_output=True) 138 keyvals_dict = {} 139 tmp_dir_in_chroot = misc.GetInsideChrootPath(self._chromeos_root, 140 self._temp_dir) 141 for line in out.splitlines(): 142 tokens = re.split("=|,", line) 143 key = tokens[-2] 144 if key.startswith(tmp_dir_in_chroot): 145 key = key[len(tmp_dir_in_chroot) + 1:] 146 value = tokens[-1] 147 keyvals_dict[key] = value 148 149 # Check to see if there is a perf_measurements file and get the 150 # data from it if so. 151 keyvals_dict, units_dict = self._GetNewKeyvals(keyvals_dict) 152 if self.suite == "telemetry_Crosperf": 153 # For telemtry_Crosperf results, append the units to the return 154 # results, for use in generating the reports. 155 keyvals_dict = self._AppendTelemetryUnits(keyvals_dict, 156 units_dict) 157 return keyvals_dict 158 159 def _GetResultsDir(self): 160 mo = re.search(r"Results placed in (\S+)", self.out) 161 if mo: 162 result = mo.group(1) 163 return result 164 raise Exception("Could not find results directory.") 165 166 def _FindFilesInResultsDir(self, find_args): 167 if not self.results_dir: 168 return None 169 170 command = "find %s %s" % (self.results_dir, 171 find_args) 172 ret, out, _ = self._ce.RunCommand(command, return_output=True) 173 if ret: 174 raise Exception("Could not run find command!") 175 return out 176 177 def _GetPerfDataFiles(self): 178 return self._FindFilesInResultsDir("-name perf.data").splitlines() 179 180 def _GetPerfReportFiles(self): 181 return self._FindFilesInResultsDir("-name perf.data.report").splitlines() 182 183 def _GetDataMeasurementsFiles(self): 184 return self._FindFilesInResultsDir("-name perf_measurements").splitlines() 185 186 def _GeneratePerfReportFiles(self): 187 perf_report_files = [] 188 for perf_data_file in self.perf_data_files: 189 # Generate a perf.report and store it side-by-side with the perf.data 190 # file. 191 chroot_perf_data_file = misc.GetInsideChrootPath(self._chromeos_root, 192 perf_data_file) 193 perf_report_file = "%s.report" % perf_data_file 194 if os.path.exists(perf_report_file): 195 raise Exception("Perf report file already exists: %s" % 196 perf_report_file) 197 chroot_perf_report_file = misc.GetInsideChrootPath(self._chromeos_root, 198 perf_report_file) 199 perf_path = os.path.join (self._chromeos_root, 200 "chroot", 201 "usr/bin/perf") 202 203 perf_file = "/usr/sbin/perf" 204 if os.path.exists(perf_path): 205 perf_file = "/usr/bin/perf" 206 207 # The following is a hack, to use the perf.static binary that 208 # was given to us by Stephane Eranian, until he can figure out 209 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 210 # Get the directory containing the 'crosperf' script. 211 dirname, _ = misc.GetRoot(sys.argv[0]) 212 perf_path = os.path.join (dirname, "..", "perf.static") 213 if os.path.exists(perf_path): 214 # copy the executable into the chroot so that it can be found. 215 src_path = perf_path 216 dst_path = os.path.join (self._chromeos_root, "chroot", 217 "tmp/perf.static") 218 command = "cp %s %s" % (src_path,dst_path) 219 self._ce.RunCommand (command) 220 perf_file = "/tmp/perf.static" 221 222 command = ("%s report " 223 "-n " 224 "--symfs /build/%s " 225 "--vmlinux /build/%s/usr/lib/debug/boot/vmlinux " 226 "--kallsyms /build/%s/boot/System.map-* " 227 "-i %s --stdio " 228 "> %s" % 229 (perf_file, 230 self._board, 231 self._board, 232 self._board, 233 chroot_perf_data_file, 234 chroot_perf_report_file)) 235 self._ce.ChrootRunCommand(self._chromeos_root, 236 command) 237 238 # Add a keyval to the dictionary for the events captured. 239 perf_report_files.append( 240 misc.GetOutsideChrootPath(self._chromeos_root, 241 chroot_perf_report_file)) 242 return perf_report_files 243 244 def _GatherPerfResults(self): 245 report_id = 0 246 for perf_report_file in self.perf_report_files: 247 with open(perf_report_file, "r") as f: 248 report_contents = f.read() 249 for group in re.findall(r"Events: (\S+) (\S+)", report_contents): 250 num_events = group[0] 251 event_name = group[1] 252 key = "perf_%s_%s" % (report_id, event_name) 253 value = str(misc.UnitToNumber(num_events)) 254 self.keyvals[key] = value 255 256 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 257 self._board = self.label.board 258 self.out = out 259 self.err = err 260 self.retval = retval 261 self.test_name = test 262 self.suite = suite 263 self.chroot_results_dir = self._GetResultsDir() 264 self.results_dir = misc.GetOutsideChrootPath(self._chromeos_root, 265 self.chroot_results_dir) 266 self.perf_data_files = self._GetPerfDataFiles() 267 # Include all perf.report data in table. 268 self.perf_report_files = self._GeneratePerfReportFiles() 269 # TODO(asharif): Do something similar with perf stat. 270 271 # Grab keyvals from the directory. 272 self._ProcessResults(show_all) 273 274 def _ProcessResults(self, show_all): 275 # Note that this function doesn't know anything about whether there is a 276 # cache hit or miss. It should process results agnostic of the cache hit 277 # state. 278 self.keyvals = self._GetKeyvals(show_all) 279 self.keyvals["retval"] = self.retval 280 # Generate report from all perf.data files. 281 # Now parse all perf report files and include them in keyvals. 282 self._GatherPerfResults() 283 284 def _PopulateFromCacheDir(self, cache_dir, show_all, test, suite): 285 self.test_name = test 286 self.suite = suite 287 # Read in everything from the cache directory. 288 with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: 289 self.out = pickle.load(f) 290 self.err = pickle.load(f) 291 self.retval = pickle.load(f) 292 293 # Untar the tarball to a temporary directory 294 self._temp_dir = tempfile.mkdtemp(dir=os.path.join(self._chromeos_root, 295 "chroot", "tmp")) 296 297 command = ("cd %s && tar xf %s" % 298 (self._temp_dir, 299 os.path.join(cache_dir, AUTOTEST_TARBALL))) 300 ret = self._ce.RunCommand(command) 301 if ret: 302 raise Exception("Could not untar cached tarball") 303 self.results_dir = self._temp_dir 304 self.perf_data_files = self._GetPerfDataFiles() 305 self.perf_report_files = self._GetPerfReportFiles() 306 self._ProcessResults(show_all) 307 308 def CleanUp(self, rm_chroot_tmp): 309 if rm_chroot_tmp and self.results_dir: 310 dirname, basename = misc.GetRoot(self.results_dir) 311 if basename.find("test_that_results_") != -1: 312 command = "rm -rf %s" % self.results_dir 313 else: 314 command = "rm -rf %s" % dirname 315 self._ce.RunCommand(command) 316 if self._temp_dir: 317 command = "rm -rf %s" % self._temp_dir 318 self._ce.RunCommand(command) 319 320 def StoreToCacheDir(self, cache_dir, machine_manager): 321 # Create the dir if it doesn't exist. 322 temp_dir = tempfile.mkdtemp() 323 324 # Store to the temp directory. 325 with open(os.path.join(temp_dir, RESULTS_FILE), "w") as f: 326 pickle.dump(self.out, f) 327 pickle.dump(self.err, f) 328 pickle.dump(self.retval, f) 329 330 if self.results_dir: 331 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 332 command = ("cd %s && " 333 "tar " 334 "--exclude=var/spool " 335 "--exclude=var/log " 336 "-cjf %s ." % (self.results_dir, tarball)) 337 ret = self._ce.RunCommand(command) 338 if ret: 339 raise Exception("Couldn't store autotest output directory.") 340 # Store machine info. 341 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 342 # this function. 343 with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: 344 f.write(machine_manager.machine_checksum_string[self.label.name]) 345 346 if os.path.exists(cache_dir): 347 command = "rm -rf {0}".format(cache_dir) 348 self._ce.RunCommand(command) 349 350 command = "mkdir -p {0} && ".format(os.path.dirname(cache_dir)) 351 command += "chmod g+x {0} && ".format(temp_dir) 352 command += "mv {0} {1}".format(temp_dir, cache_dir) 353 ret = self._ce.RunCommand(command) 354 if ret: 355 command = "rm -rf {0}".format(temp_dir) 356 self._ce.RunCommand(command) 357 raise Exception("Could not move dir %s to dir %s" % 358 (temp_dir, cache_dir)) 359 360 @classmethod 361 def CreateFromRun(cls, logger, log_level, label, out, err, retval, show_all, 362 test, suite="telemetry_Crosperf"): 363 if suite == "telemetry": 364 result = TelemetryResult(logger, label, log_level) 365 else: 366 result = cls(logger, label, log_level) 367 result._PopulateFromRun(out, err, retval, show_all, test, suite) 368 return result 369 370 @classmethod 371 def CreateFromCacheHit(cls, logger, log_level, label, cache_dir, 372 show_all, test, suite="telemetry_Crosperf"): 373 if suite == "telemetry": 374 result = TelemetryResult(logger, label) 375 else: 376 result = cls(logger, label, log_level) 377 try: 378 result._PopulateFromCacheDir(cache_dir, show_all, test, suite) 379 380 except Exception as e: 381 logger.LogError("Exception while using cache: %s" % e) 382 return None 383 return result 384 385 386class TelemetryResult(Result): 387 388 def __init__(self, logger, label, log_level, cmd_exec=None): 389 super(TelemetryResult, self).__init__(logger, label, log_level, cmd_exec) 390 391 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 392 self.out = out 393 self.err = err 394 self.retval = retval 395 396 self._ProcessResults() 397 398 def _ProcessResults(self): 399 # The output is: 400 # url,average_commit_time (ms),... 401 # www.google.com,33.4,21.2,... 402 # We need to convert to this format: 403 # {"www.google.com:average_commit_time (ms)": "33.4", 404 # "www.google.com:...": "21.2"} 405 # Added note: Occasionally the output comes back 406 # with "JSON.stringify(window.automation.GetResults())" on 407 # the first line, and then the rest of the output as 408 # described above. 409 410 lines = self.out.splitlines() 411 self.keyvals = {} 412 413 if lines: 414 if lines[0].startswith("JSON.stringify"): 415 lines = lines[1:] 416 417 if not lines: 418 return 419 labels = lines[0].split(",") 420 for line in lines[1:]: 421 fields = line.split(",") 422 if len(fields) != len(labels): 423 continue 424 for i in range(1, len(labels)): 425 key = "%s %s" % (fields[0], labels[i]) 426 value = fields[i] 427 self.keyvals[key] = value 428 self.keyvals["retval"] = self.retval 429 430 def _PopulateFromCacheDir(self, cache_dir): 431 with open(os.path.join(cache_dir, RESULTS_FILE), "r") as f: 432 self.out = pickle.load(f) 433 self.err = pickle.load(f) 434 self.retval = pickle.load(f) 435 self._ProcessResults() 436 437 438class CacheConditions(object): 439 # Cache hit only if the result file exists. 440 CACHE_FILE_EXISTS = 0 441 442 # Cache hit if the checksum of cpuinfo and totalmem of 443 # the cached result and the new run match. 444 MACHINES_MATCH = 1 445 446 # Cache hit if the image checksum of the cached result and the new run match. 447 CHECKSUMS_MATCH = 2 448 449 # Cache hit only if the cached result was successful 450 RUN_SUCCEEDED = 3 451 452 # Never a cache hit. 453 FALSE = 4 454 455 # Cache hit if the image path matches the cached image path. 456 IMAGE_PATH_MATCH = 5 457 458 # Cache hit if the uuid of hard disk mataches the cached one 459 460 SAME_MACHINE_MATCH = 6 461 462 463class ResultsCache(object): 464 465 """ This class manages the key of the cached runs without worrying about what 466 is exactly stored (value). The value generation is handled by the Results 467 class. 468 """ 469 CACHE_VERSION = 6 470 471 def Init(self, chromeos_image, chromeos_root, test_name, iteration, 472 test_args, profiler_args, machine_manager, board, cache_conditions, 473 logger_to_use, log_level, label, share_cache, suite, 474 show_all_results, run_local): 475 self.chromeos_image = chromeos_image 476 self.chromeos_root = chromeos_root 477 self.test_name = test_name 478 self.iteration = iteration 479 self.test_args = test_args 480 self.profiler_args = profiler_args 481 self.board = board 482 self.cache_conditions = cache_conditions 483 self.machine_manager = machine_manager 484 self._logger = logger_to_use 485 self._ce = command_executer.GetCommandExecuter(self._logger, 486 log_level=log_level) 487 self.label = label 488 self.share_cache = share_cache 489 self.suite = suite 490 self.log_level = log_level 491 self.show_all = show_all_results 492 self.run_local = run_local 493 494 def _GetCacheDirForRead(self): 495 matching_dirs = [] 496 for glob_path in self._FormCacheDir(self._GetCacheKeyList(True)): 497 matching_dirs += glob.glob(glob_path) 498 499 if matching_dirs: 500 # Cache file found. 501 return matching_dirs[0] 502 else: 503 return None 504 505 def _GetCacheDirForWrite(self): 506 return self._FormCacheDir(self._GetCacheKeyList(False))[0] 507 508 def _FormCacheDir(self, list_of_strings): 509 cache_key = " ".join(list_of_strings) 510 cache_dir = misc.GetFilenameFromString(cache_key) 511 if self.label.cache_dir: 512 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 513 cache_path = [os.path.join(cache_home, cache_dir)] 514 else: 515 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 516 517 if len(self.share_cache): 518 for path in [x.strip() for x in self.share_cache.split(",")]: 519 if os.path.exists(path): 520 cache_path.append(os.path.join(path, cache_dir)) 521 else: 522 self._logger.LogFatal("Unable to find shared cache: %s" % path) 523 524 return cache_path 525 526 def _GetCacheKeyList(self, read): 527 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 528 machine_checksum = "*" 529 else: 530 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 531 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 532 checksum = "*" 533 elif self.label.image_type == "trybot": 534 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 535 elif self.label.image_type == "official": 536 checksum = "*" 537 else: 538 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 539 540 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 541 image_path_checksum = "*" 542 else: 543 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 544 545 machine_id_checksum = "" 546 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 547 machine_id_checksum = "*" 548 else: 549 for machine in self.machine_manager.GetMachines(self.label): 550 if machine.name == self.label.remote[0]: 551 machine_id_checksum = machine.machine_id_checksum 552 break 553 554 temp_test_args = "%s %s %s" % (self.test_args, self.profiler_args, self.run_local) 555 test_args_checksum = hashlib.md5( 556 "".join(temp_test_args)).hexdigest() 557 return (image_path_checksum, 558 self.test_name, str(self.iteration), 559 test_args_checksum, 560 checksum, 561 machine_checksum, 562 machine_id_checksum, 563 str(self.CACHE_VERSION)) 564 565 def ReadResult(self): 566 if CacheConditions.FALSE in self.cache_conditions: 567 cache_dir = self._GetCacheDirForWrite() 568 command = "rm -rf {0}".format(cache_dir) 569 self._ce.RunCommand(command) 570 return None 571 cache_dir = self._GetCacheDirForRead() 572 573 if not cache_dir: 574 return None 575 576 if not os.path.isdir(cache_dir): 577 return None 578 579 self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir) 580 result = Result.CreateFromCacheHit(self._logger, 581 self.log_level, 582 self.label, 583 cache_dir, 584 self.show_all, 585 self.test_name, 586 self.suite) 587 if not result: 588 return None 589 590 if (result.retval == 0 or 591 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 592 return result 593 594 return None 595 596 def StoreResult(self, result): 597 cache_dir = self._GetCacheDirForWrite() 598 result.StoreToCacheDir(cache_dir, self.machine_manager) 599 600 601class MockResultsCache(ResultsCache): 602 def Init(self, *args): 603 pass 604 605 def ReadResult(self): 606 return None 607 608 def StoreResult(self, result): 609 pass 610 611 612class MockResult(Result): 613 def _PopulateFromRun(self, out, err, retval, show_all, test, suite): 614 self.out = out 615 self.err = err 616 self.retval = retval 617