results_cache.py revision 9eabe97c2dfaf0c472e3bf43849817c668b0b70a
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Module to deal with result cache.""" 5 6from __future__ import print_function 7 8import glob 9import hashlib 10import os 11import pickle 12import re 13import tempfile 14import json 15import sys 16 17from cros_utils import command_executer 18from cros_utils import misc 19 20from image_checksummer import ImageChecksummer 21 22import results_report 23import test_flag 24 25SCRATCH_DIR = os.path.expanduser('~/cros_scratch') 26RESULTS_FILE = 'results.txt' 27MACHINE_FILE = 'machine.txt' 28AUTOTEST_TARBALL = 'autotest.tbz2' 29PERF_RESULTS_FILE = 'perf-results.txt' 30CACHE_KEYS_FILE = 'cache_keys.txt' 31 32 33class Result(object): 34 """Class for holding the results of a single test run. 35 36 This class manages what exactly is stored inside the cache without knowing 37 what the key of the cache is. For runs with perf, it stores perf.data, 38 perf.report, etc. The key generation is handled by the ResultsCache class. 39 """ 40 41 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 42 self.chromeos_root = label.chromeos_root 43 self._logger = logger 44 self.ce = cmd_exec or command_executer.GetCommandExecuter( 45 self._logger, log_level=log_level) 46 self.temp_dir = None 47 self.label = label 48 self.results_dir = None 49 self.log_level = log_level 50 self.machine = machine 51 self.perf_data_files = [] 52 self.perf_report_files = [] 53 self.results_file = [] 54 self.chrome_version = '' 55 self.err = None 56 self.chroot_results_dir = '' 57 self.test_name = '' 58 self.keyvals = None 59 self.board = None 60 self.suite = None 61 self.retval = None 62 self.out = None 63 64 def CopyFilesTo(self, dest_dir, files_to_copy): 65 file_index = 0 66 for file_to_copy in files_to_copy: 67 if not os.path.isdir(dest_dir): 68 command = 'mkdir -p %s' % dest_dir 69 self.ce.RunCommand(command) 70 dest_file = os.path.join(dest_dir, 71 ('%s.%s' % (os.path.basename(file_to_copy), 72 file_index))) 73 ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 74 if ret: 75 raise IOError('Could not copy results file: %s' % file_to_copy) 76 77 def CopyResultsTo(self, dest_dir): 78 self.CopyFilesTo(dest_dir, self.perf_data_files) 79 self.CopyFilesTo(dest_dir, self.perf_report_files) 80 if len(self.perf_data_files) or len(self.perf_report_files): 81 self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 82 83 def GetNewKeyvals(self, keyvals_dict): 84 # Initialize 'units' dictionary. 85 units_dict = {} 86 for k in keyvals_dict: 87 units_dict[k] = '' 88 results_files = self.GetDataMeasurementsFiles() 89 for f in results_files: 90 # Make sure we can find the results file 91 if os.path.exists(f): 92 data_filename = f 93 else: 94 # Otherwise get the base filename and create the correct 95 # path for it. 96 _, f_base = misc.GetRoot(f) 97 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 98 self.temp_dir, f_base) 99 if data_filename.find('.json') > 0: 100 raw_dict = dict() 101 if os.path.exists(data_filename): 102 with open(data_filename, 'r') as data_file: 103 raw_dict = json.load(data_file) 104 105 if 'charts' in raw_dict: 106 raw_dict = raw_dict['charts'] 107 for k1 in raw_dict: 108 field_dict = raw_dict[k1] 109 for k2 in field_dict: 110 result_dict = field_dict[k2] 111 key = k1 + '__' + k2 112 if 'value' in result_dict: 113 keyvals_dict[key] = result_dict['value'] 114 elif 'values' in result_dict: 115 values = result_dict['values'] 116 if ('type' in result_dict and 117 result_dict['type'] == 'list_of_scalar_values' and values and 118 values != 'null'): 119 keyvals_dict[key] = sum(values) / float(len(values)) 120 else: 121 keyvals_dict[key] = values 122 units_dict[key] = result_dict['units'] 123 else: 124 if os.path.exists(data_filename): 125 with open(data_filename, 'r') as data_file: 126 lines = data_file.readlines() 127 for line in lines: 128 tmp_dict = json.loads(line) 129 graph_name = tmp_dict['graph'] 130 graph_str = (graph_name + '__') if graph_name else '' 131 key = graph_str + tmp_dict['description'] 132 keyvals_dict[key] = tmp_dict['value'] 133 units_dict[key] = tmp_dict['units'] 134 135 return keyvals_dict, units_dict 136 137 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 138 """keyvals_dict is the dict of key-value used to generate Crosperf reports. 139 140 units_dict is a dictionary of the units for the return values in 141 keyvals_dict. We need to associate the units with the return values, 142 for Telemetry tests, so that we can include the units in the reports. 143 This function takes each value in keyvals_dict, finds the corresponding 144 unit in the units_dict, and replaces the old value with a list of the 145 old value and the units. This later gets properly parsed in the 146 ResultOrganizer class, for generating the reports. 147 """ 148 149 results_dict = {} 150 for k in keyvals_dict: 151 # We don't want these lines in our reports; they add no useful data. 152 if k == '' or k == 'telemetry_Crosperf': 153 continue 154 val = keyvals_dict[k] 155 units = units_dict[k] 156 new_val = [val, units] 157 results_dict[k] = new_val 158 return results_dict 159 160 def GetKeyvals(self): 161 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 162 if not self.temp_dir: 163 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 164 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 165 self.ce.RunCommand(command, print_to_console=False) 166 167 command = ('python generate_test_report --no-color --csv %s' % 168 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 169 _, out, _ = self.ce.ChrootRunCommandWOutput( 170 self.chromeos_root, command, print_to_console=False) 171 keyvals_dict = {} 172 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 173 self.temp_dir) 174 for line in out.splitlines(): 175 tokens = re.split('=|,', line) 176 key = tokens[-2] 177 if key.startswith(tmp_dir_in_chroot): 178 key = key[len(tmp_dir_in_chroot) + 1:] 179 value = tokens[-1] 180 keyvals_dict[key] = value 181 182 # Check to see if there is a perf_measurements file and get the 183 # data from it if so. 184 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 185 if self.suite == 'telemetry_Crosperf': 186 # For telemtry_Crosperf results, append the units to the return 187 # results, for use in generating the reports. 188 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 189 return keyvals_dict 190 191 def GetResultsDir(self): 192 mo = re.search(r'Results placed in (\S+)', self.out) 193 if mo: 194 result = mo.group(1) 195 return result 196 raise RuntimeError('Could not find results directory.') 197 198 def FindFilesInResultsDir(self, find_args): 199 if not self.results_dir: 200 return None 201 202 command = 'find %s %s' % (self.results_dir, find_args) 203 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 204 if ret: 205 raise RuntimeError('Could not run find command!') 206 return out 207 208 def GetResultsFile(self): 209 return self.FindFilesInResultsDir('-name results-chart.json').splitlines() 210 211 def GetPerfDataFiles(self): 212 return self.FindFilesInResultsDir('-name perf.data').splitlines() 213 214 def GetPerfReportFiles(self): 215 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 216 217 def GetDataMeasurementsFiles(self): 218 result = self.FindFilesInResultsDir('-name perf_measurements').splitlines() 219 if not result: 220 result = \ 221 self.FindFilesInResultsDir('-name results-chart.json').splitlines() 222 return result 223 224 def GeneratePerfReportFiles(self): 225 perf_report_files = [] 226 for perf_data_file in self.perf_data_files: 227 # Generate a perf.report and store it side-by-side with the perf.data 228 # file. 229 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 230 perf_data_file) 231 perf_report_file = '%s.report' % perf_data_file 232 if os.path.exists(perf_report_file): 233 raise RuntimeError('Perf report file already exists: %s' % 234 perf_report_file) 235 chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, 236 perf_report_file) 237 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 238 239 perf_file = '/usr/sbin/perf' 240 if os.path.exists(perf_path): 241 perf_file = '/usr/bin/perf' 242 243 # The following is a hack, to use the perf.static binary that 244 # was given to us by Stephane Eranian, until he can figure out 245 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 246 # Get the directory containing the 'crosperf' script. 247 dirname, _ = misc.GetRoot(sys.argv[0]) 248 perf_path = os.path.join(dirname, '..', 'perf.static') 249 if os.path.exists(perf_path): 250 # copy the executable into the chroot so that it can be found. 251 src_path = perf_path 252 dst_path = os.path.join(self.chromeos_root, 'chroot', 'tmp/perf.static') 253 command = 'cp %s %s' % (src_path, dst_path) 254 self.ce.RunCommand(command) 255 perf_file = '/tmp/perf.static' 256 257 command = ('%s report ' 258 '-n ' 259 '--symfs /build/%s ' 260 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 261 '--kallsyms /build/%s/boot/System.map-* ' 262 '-i %s --stdio ' 263 '> %s' % (perf_file, self.board, self.board, self.board, 264 chroot_perf_data_file, chroot_perf_report_file)) 265 self.ce.ChrootRunCommand(self.chromeos_root, command) 266 267 # Add a keyval to the dictionary for the events captured. 268 perf_report_files.append( 269 misc.GetOutsideChrootPath(self.chromeos_root, 270 chroot_perf_report_file)) 271 return perf_report_files 272 273 def GatherPerfResults(self): 274 report_id = 0 275 for perf_report_file in self.perf_report_files: 276 with open(perf_report_file, 'r') as f: 277 report_contents = f.read() 278 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 279 num_events = group[0] 280 event_name = group[1] 281 key = 'perf_%s_%s' % (report_id, event_name) 282 value = str(misc.UnitToNumber(num_events)) 283 self.keyvals[key] = value 284 285 def PopulateFromRun(self, out, err, retval, test, suite): 286 self.board = self.label.board 287 self.out = out 288 self.err = err 289 self.retval = retval 290 self.test_name = test 291 self.suite = suite 292 self.chroot_results_dir = self.GetResultsDir() 293 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 294 self.chroot_results_dir) 295 self.results_file = self.GetResultsFile() 296 self.perf_data_files = self.GetPerfDataFiles() 297 # Include all perf.report data in table. 298 self.perf_report_files = self.GeneratePerfReportFiles() 299 # TODO(asharif): Do something similar with perf stat. 300 301 # Grab keyvals from the directory. 302 self.ProcessResults() 303 304 def ProcessJsonResults(self): 305 # Open and parse the json results file generated by telemetry/test_that. 306 if not self.results_file: 307 raise IOError('No results file found.') 308 filename = self.results_file[0] 309 if not filename.endswith('.json'): 310 raise IOError('Attempt to call json on non-json file: %s' % filename) 311 312 if not os.path.exists(filename): 313 return {} 314 315 keyvals = {} 316 with open(filename, 'r') as f: 317 raw_dict = json.load(f) 318 if 'charts' in raw_dict: 319 raw_dict = raw_dict['charts'] 320 for k, field_dict in raw_dict.iteritems(): 321 for item in field_dict: 322 keyname = k + '__' + item 323 value_dict = field_dict[item] 324 if 'value' in value_dict: 325 result = value_dict['value'] 326 elif 'values' in value_dict: 327 values = value_dict['values'] 328 if not values: 329 continue 330 if ('type' in value_dict and 331 value_dict['type'] == 'list_of_scalar_values' and 332 values != 'null'): 333 result = sum(values) / float(len(values)) 334 else: 335 result = values 336 units = value_dict['units'] 337 new_value = [result, units] 338 keyvals[keyname] = new_value 339 return keyvals 340 341 def ProcessResults(self, use_cache=False): 342 # Note that this function doesn't know anything about whether there is a 343 # cache hit or miss. It should process results agnostic of the cache hit 344 # state. 345 if self.results_file and self.results_file[0].find( 346 'results-chart.json') != -1: 347 self.keyvals = self.ProcessJsonResults() 348 else: 349 if not use_cache: 350 print('\n ** WARNING **: Had to use deprecated output-method to ' 351 'collect results.\n') 352 self.keyvals = self.GetKeyvals() 353 self.keyvals['retval'] = self.retval 354 # Generate report from all perf.data files. 355 # Now parse all perf report files and include them in keyvals. 356 self.GatherPerfResults() 357 358 def GetChromeVersionFromCache(self, cache_dir): 359 # Read chrome_version from keys file, if present. 360 chrome_version = '' 361 keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) 362 if os.path.exists(keys_file): 363 with open(keys_file, 'r') as f: 364 lines = f.readlines() 365 for l in lines: 366 if l.startswith('Google Chrome '): 367 chrome_version = l 368 if chrome_version.endswith('\n'): 369 chrome_version = chrome_version[:-1] 370 break 371 return chrome_version 372 373 def PopulateFromCacheDir(self, cache_dir, test, suite): 374 self.test_name = test 375 self.suite = suite 376 # Read in everything from the cache directory. 377 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 378 self.out = pickle.load(f) 379 self.err = pickle.load(f) 380 self.retval = pickle.load(f) 381 382 # Untar the tarball to a temporary directory 383 self.temp_dir = tempfile.mkdtemp( 384 dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) 385 386 command = ('cd %s && tar xf %s' % 387 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 388 ret = self.ce.RunCommand(command, print_to_console=False) 389 if ret: 390 raise RuntimeError('Could not untar cached tarball') 391 self.results_dir = self.temp_dir 392 self.results_file = self.GetDataMeasurementsFiles() 393 self.perf_data_files = self.GetPerfDataFiles() 394 self.perf_report_files = self.GetPerfReportFiles() 395 self.chrome_version = self.GetChromeVersionFromCache(cache_dir) 396 self.ProcessResults(use_cache=True) 397 398 def CleanUp(self, rm_chroot_tmp): 399 if rm_chroot_tmp and self.results_dir: 400 dirname, basename = misc.GetRoot(self.results_dir) 401 if basename.find('test_that_results_') != -1: 402 command = 'rm -rf %s' % self.results_dir 403 else: 404 command = 'rm -rf %s' % dirname 405 self.ce.RunCommand(command) 406 if self.temp_dir: 407 command = 'rm -rf %s' % self.temp_dir 408 self.ce.RunCommand(command) 409 410 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 411 # Create the dir if it doesn't exist. 412 temp_dir = tempfile.mkdtemp() 413 414 # Store to the temp directory. 415 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 416 pickle.dump(self.out, f) 417 pickle.dump(self.err, f) 418 pickle.dump(self.retval, f) 419 420 if not test_flag.GetTestMode(): 421 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 422 f.write('%s\n' % self.label.name) 423 f.write('%s\n' % self.label.chrome_version) 424 f.write('%s\n' % self.machine.checksum_string) 425 for k in key_list: 426 f.write(k) 427 f.write('\n') 428 429 if self.results_dir: 430 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 431 command = ('cd %s && ' 432 'tar ' 433 '--exclude=var/spool ' 434 '--exclude=var/log ' 435 '-cjf %s .' % (self.results_dir, tarball)) 436 ret = self.ce.RunCommand(command) 437 if ret: 438 raise RuntimeError("Couldn't store autotest output directory.") 439 # Store machine info. 440 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 441 # this function. 442 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 443 f.write(machine_manager.machine_checksum_string[self.label.name]) 444 445 if os.path.exists(cache_dir): 446 command = 'rm -rf {0}'.format(cache_dir) 447 self.ce.RunCommand(command) 448 449 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 450 command += 'chmod g+x {0} && '.format(temp_dir) 451 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 452 ret = self.ce.RunCommand(command) 453 if ret: 454 command = 'rm -rf {0}'.format(temp_dir) 455 self.ce.RunCommand(command) 456 raise RuntimeError('Could not move dir %s to dir %s' % 457 (temp_dir, cache_dir)) 458 459 @classmethod 460 def CreateFromRun(cls, 461 logger, 462 log_level, 463 label, 464 machine, 465 out, 466 err, 467 retval, 468 test, 469 suite='telemetry_Crosperf'): 470 if suite == 'telemetry': 471 result = TelemetryResult(logger, label, log_level, machine) 472 else: 473 result = cls(logger, label, log_level, machine) 474 result.PopulateFromRun(out, err, retval, test, suite) 475 return result 476 477 @classmethod 478 def CreateFromCacheHit(cls, 479 logger, 480 log_level, 481 label, 482 machine, 483 cache_dir, 484 test, 485 suite='telemetry_Crosperf'): 486 if suite == 'telemetry': 487 result = TelemetryResult(logger, label, log_level, machine) 488 else: 489 result = cls(logger, label, log_level, machine) 490 try: 491 result.PopulateFromCacheDir(cache_dir, test, suite) 492 493 except RuntimeError as e: 494 logger.LogError('Exception while using cache: %s' % e) 495 return None 496 return result 497 498 499class TelemetryResult(Result): 500 """Class to hold the results of a single Telemetry run.""" 501 502 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 503 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 504 cmd_exec) 505 506 def PopulateFromRun(self, out, err, retval, test, suite): 507 self.out = out 508 self.err = err 509 self.retval = retval 510 511 self.ProcessResults() 512 513 # pylint: disable=arguments-differ 514 def ProcessResults(self): 515 # The output is: 516 # url,average_commit_time (ms),... 517 # www.google.com,33.4,21.2,... 518 # We need to convert to this format: 519 # {"www.google.com:average_commit_time (ms)": "33.4", 520 # "www.google.com:...": "21.2"} 521 # Added note: Occasionally the output comes back 522 # with "JSON.stringify(window.automation.GetResults())" on 523 # the first line, and then the rest of the output as 524 # described above. 525 526 lines = self.out.splitlines() 527 self.keyvals = {} 528 529 if lines: 530 if lines[0].startswith('JSON.stringify'): 531 lines = lines[1:] 532 533 if not lines: 534 return 535 labels = lines[0].split(',') 536 for line in lines[1:]: 537 fields = line.split(',') 538 if len(fields) != len(labels): 539 continue 540 for i in xrange(1, len(labels)): 541 key = '%s %s' % (fields[0], labels[i]) 542 value = fields[i] 543 self.keyvals[key] = value 544 self.keyvals['retval'] = self.retval 545 546 def PopulateFromCacheDir(self, cache_dir, test, suite): 547 self.test_name = test 548 self.suite = suite 549 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 550 self.out = pickle.load(f) 551 self.err = pickle.load(f) 552 self.retval = pickle.load(f) 553 554 self.chrome_version = \ 555 super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) 556 self.ProcessResults() 557 558 559class CacheConditions(object): 560 """Various Cache condition values, for export.""" 561 562 # Cache hit only if the result file exists. 563 CACHE_FILE_EXISTS = 0 564 565 # Cache hit if the checksum of cpuinfo and totalmem of 566 # the cached result and the new run match. 567 MACHINES_MATCH = 1 568 569 # Cache hit if the image checksum of the cached result and the new run match. 570 CHECKSUMS_MATCH = 2 571 572 # Cache hit only if the cached result was successful 573 RUN_SUCCEEDED = 3 574 575 # Never a cache hit. 576 FALSE = 4 577 578 # Cache hit if the image path matches the cached image path. 579 IMAGE_PATH_MATCH = 5 580 581 # Cache hit if the uuid of hard disk mataches the cached one 582 583 SAME_MACHINE_MATCH = 6 584 585 586class ResultsCache(object): 587 """Class to handle the cache for storing/retrieving test run results. 588 589 This class manages the key of the cached runs without worrying about what 590 is exactly stored (value). The value generation is handled by the Results 591 class. 592 """ 593 CACHE_VERSION = 6 594 595 def __init__(self): 596 # Proper initialization happens in the Init function below. 597 self.chromeos_image = None 598 self.chromeos_root = None 599 self.test_name = None 600 self.iteration = None 601 self.test_args = None 602 self.profiler_args = None 603 self.board = None 604 self.cache_conditions = None 605 self.machine_manager = None 606 self.machine = None 607 self._logger = None 608 self.ce = None 609 self.label = None 610 self.share_cache = None 611 self.suite = None 612 self.log_level = None 613 self.show_all = None 614 self.run_local = None 615 616 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 617 profiler_args, machine_manager, machine, board, cache_conditions, 618 logger_to_use, log_level, label, share_cache, suite, 619 show_all_results, run_local): 620 self.chromeos_image = chromeos_image 621 self.chromeos_root = chromeos_root 622 self.test_name = test_name 623 self.iteration = iteration 624 self.test_args = test_args 625 self.profiler_args = profiler_args 626 self.board = board 627 self.cache_conditions = cache_conditions 628 self.machine_manager = machine_manager 629 self.machine = machine 630 self._logger = logger_to_use 631 self.ce = command_executer.GetCommandExecuter( 632 self._logger, log_level=log_level) 633 self.label = label 634 self.share_cache = share_cache 635 self.suite = suite 636 self.log_level = log_level 637 self.show_all = show_all_results 638 self.run_local = run_local 639 640 def GetCacheDirForRead(self): 641 matching_dirs = [] 642 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 643 matching_dirs += glob.glob(glob_path) 644 645 if matching_dirs: 646 # Cache file found. 647 return matching_dirs[0] 648 return None 649 650 def GetCacheDirForWrite(self, get_keylist=False): 651 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 652 if get_keylist: 653 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 654 self.run_local) 655 version, image = results_report.ParseChromeosImage( 656 self.label.chromeos_image) 657 keylist = [ 658 version, image, self.label.board, self.machine.name, self.test_name, 659 str(self.iteration), args_str 660 ] 661 return cache_path, keylist 662 return cache_path 663 664 def FormCacheDir(self, list_of_strings): 665 cache_key = ' '.join(list_of_strings) 666 cache_dir = misc.GetFilenameFromString(cache_key) 667 if self.label.cache_dir: 668 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 669 cache_path = [os.path.join(cache_home, cache_dir)] 670 else: 671 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 672 673 if len(self.share_cache): 674 for path in [x.strip() for x in self.share_cache.split(',')]: 675 if os.path.exists(path): 676 cache_path.append(os.path.join(path, cache_dir)) 677 else: 678 self._logger.LogFatal('Unable to find shared cache: %s' % path) 679 680 return cache_path 681 682 def GetCacheKeyList(self, read): 683 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 684 machine_checksum = '*' 685 else: 686 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 687 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 688 checksum = '*' 689 elif self.label.image_type == 'trybot': 690 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 691 elif self.label.image_type == 'official': 692 checksum = '*' 693 else: 694 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 695 696 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 697 image_path_checksum = '*' 698 else: 699 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 700 701 machine_id_checksum = '' 702 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 703 machine_id_checksum = '*' 704 else: 705 if self.machine and self.machine.name in self.label.remote: 706 machine_id_checksum = self.machine.machine_id_checksum 707 else: 708 for machine in self.machine_manager.GetMachines(self.label): 709 if machine.name == self.label.remote[0]: 710 machine_id_checksum = machine.machine_id_checksum 711 break 712 713 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 714 self.run_local) 715 test_args_checksum = hashlib.md5(temp_test_args).hexdigest() 716 return (image_path_checksum, self.test_name, str(self.iteration), 717 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 718 str(self.CACHE_VERSION)) 719 720 def ReadResult(self): 721 if CacheConditions.FALSE in self.cache_conditions: 722 cache_dir = self.GetCacheDirForWrite() 723 command = 'rm -rf %s' % (cache_dir,) 724 self.ce.RunCommand(command) 725 return None 726 cache_dir = self.GetCacheDirForRead() 727 728 if not cache_dir: 729 return None 730 731 if not os.path.isdir(cache_dir): 732 return None 733 734 if self.log_level == 'verbose': 735 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 736 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 737 self.machine, cache_dir, self.test_name, 738 self.suite) 739 if not result: 740 return None 741 742 if (result.retval == 0 or 743 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 744 return result 745 746 return None 747 748 def StoreResult(self, result): 749 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 750 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 751 752 753class MockResultsCache(ResultsCache): 754 """Class for mock testing, corresponding to ResultsCache class.""" 755 756 def Init(self, *args): 757 pass 758 759 def ReadResult(self): 760 return None 761 762 def StoreResult(self, result): 763 pass 764 765 766class MockResult(Result): 767 """Class for mock testing, corresponding to Result class.""" 768 769 def PopulateFromRun(self, out, err, retval, test, suite): 770 self.out = out 771 self.err = err 772 self.retval = retval 773