results_cache.py revision eb9fce674ff90a6de04827bfe1ef6e07a99c8f61
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Module to deal with result cache.""" 5 6from __future__ import print_function 7 8import glob 9import hashlib 10import os 11import pickle 12import re 13import tempfile 14import json 15import sys 16 17from cros_utils import command_executer 18from cros_utils import misc 19 20from image_checksummer import ImageChecksummer 21 22import results_report 23import test_flag 24 25SCRATCH_DIR = os.path.expanduser('~/cros_scratch') 26RESULTS_FILE = 'results.txt' 27MACHINE_FILE = 'machine.txt' 28AUTOTEST_TARBALL = 'autotest.tbz2' 29PERF_RESULTS_FILE = 'perf-results.txt' 30CACHE_KEYS_FILE = 'cache_keys.txt' 31 32 33class Result(object): 34 """Class for holding the results of a single test run. 35 36 This class manages what exactly is stored inside the cache without knowing 37 what the key of the cache is. For runs with perf, it stores perf.data, 38 perf.report, etc. The key generation is handled by the ResultsCache class. 39 """ 40 41 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 42 self.chromeos_root = label.chromeos_root 43 self._logger = logger 44 self.ce = cmd_exec or command_executer.GetCommandExecuter( 45 self._logger, 46 log_level=log_level) 47 self.temp_dir = None 48 self.label = label 49 self.results_dir = None 50 self.log_level = log_level 51 self.machine = machine 52 self.perf_data_files = [] 53 self.perf_report_files = [] 54 self.results_file = [] 55 self.chrome_version = '' 56 self.err = None 57 self.chroot_results_dir = '' 58 self.test_name = '' 59 self.keyvals = None 60 self.board = None 61 self.suite = None 62 self.retval = None 63 self.out = None 64 65 def CopyFilesTo(self, dest_dir, files_to_copy): 66 file_index = 0 67 for file_to_copy in files_to_copy: 68 if not os.path.isdir(dest_dir): 69 command = 'mkdir -p %s' % dest_dir 70 self.ce.RunCommand(command) 71 dest_file = os.path.join(dest_dir, 72 ('%s.%s' % (os.path.basename(file_to_copy), 73 file_index))) 74 ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 75 if ret: 76 raise IOError('Could not copy results file: %s' % file_to_copy) 77 78 def CopyResultsTo(self, dest_dir): 79 self.CopyFilesTo(dest_dir, self.perf_data_files) 80 self.CopyFilesTo(dest_dir, self.perf_report_files) 81 if len(self.perf_data_files) or len(self.perf_report_files): 82 self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 83 84 def GetNewKeyvals(self, keyvals_dict): 85 # Initialize 'units' dictionary. 86 units_dict = {} 87 for k in keyvals_dict: 88 units_dict[k] = '' 89 results_files = self.GetDataMeasurementsFiles() 90 for f in results_files: 91 # Make sure we can find the results file 92 if os.path.exists(f): 93 data_filename = f 94 else: 95 # Otherwise get the base filename and create the correct 96 # path for it. 97 _, f_base = misc.GetRoot(f) 98 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 99 self.temp_dir, f_base) 100 if data_filename.find('.json') > 0: 101 raw_dict = dict() 102 if os.path.exists(data_filename): 103 with open(data_filename, 'r') as data_file: 104 raw_dict = json.load(data_file) 105 106 if 'charts' in raw_dict: 107 raw_dict = raw_dict['charts'] 108 for k1 in raw_dict: 109 field_dict = raw_dict[k1] 110 for k2 in field_dict: 111 result_dict = field_dict[k2] 112 key = k1 + '__' + k2 113 if 'value' in result_dict: 114 keyvals_dict[key] = result_dict['value'] 115 elif 'values' in result_dict: 116 keyvals_dict[key] = result_dict['values'] 117 units_dict[key] = result_dict['units'] 118 else: 119 if os.path.exists(data_filename): 120 with open(data_filename, 'r') as data_file: 121 lines = data_file.readlines() 122 for line in lines: 123 tmp_dict = json.loads(line) 124 graph_name = tmp_dict['graph'] 125 graph_str = (graph_name + '__') if graph_name else '' 126 key = graph_str + tmp_dict['description'] 127 keyvals_dict[key] = tmp_dict['value'] 128 units_dict[key] = tmp_dict['units'] 129 130 return keyvals_dict, units_dict 131 132 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 133 """keyvals_dict is the dict of key-value used to generate Crosperf reports. 134 135 units_dict is a dictionary of the units for the return values in 136 keyvals_dict. We need to associate the units with the return values, 137 for Telemetry tests, so that we can include the units in the reports. 138 This function takes each value in keyvals_dict, finds the corresponding 139 unit in the units_dict, and replaces the old value with a list of the 140 old value and the units. This later gets properly parsed in the 141 ResultOrganizer class, for generating the reports. 142 """ 143 144 results_dict = {} 145 for k in keyvals_dict: 146 # We don't want these lines in our reports; they add no useful data. 147 if k == '' or k == 'telemetry_Crosperf': 148 continue 149 val = keyvals_dict[k] 150 units = units_dict[k] 151 new_val = [val, units] 152 results_dict[k] = new_val 153 return results_dict 154 155 def GetKeyvals(self): 156 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 157 if not self.temp_dir: 158 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 159 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 160 self.ce.RunCommand(command, print_to_console=False) 161 162 command = ('python generate_test_report --no-color --csv %s' % 163 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 164 _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root, 165 command, 166 print_to_console=False) 167 keyvals_dict = {} 168 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 169 self.temp_dir) 170 for line in out.splitlines(): 171 tokens = re.split('=|,', line) 172 key = tokens[-2] 173 if key.startswith(tmp_dir_in_chroot): 174 key = key[len(tmp_dir_in_chroot) + 1:] 175 value = tokens[-1] 176 keyvals_dict[key] = value 177 178 # Check to see if there is a perf_measurements file and get the 179 # data from it if so. 180 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 181 if self.suite == 'telemetry_Crosperf': 182 # For telemtry_Crosperf results, append the units to the return 183 # results, for use in generating the reports. 184 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 185 return keyvals_dict 186 187 def GetResultsDir(self): 188 mo = re.search(r'Results placed in (\S+)', self.out) 189 if mo: 190 result = mo.group(1) 191 return result 192 raise RuntimeError('Could not find results directory.') 193 194 def FindFilesInResultsDir(self, find_args): 195 if not self.results_dir: 196 return None 197 198 command = 'find %s %s' % (self.results_dir, find_args) 199 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 200 if ret: 201 raise RuntimeError('Could not run find command!') 202 return out 203 204 def GetResultsFile(self): 205 return self.FindFilesInResultsDir('-name results-chart.json').splitlines() 206 207 def GetPerfDataFiles(self): 208 return self.FindFilesInResultsDir('-name perf.data').splitlines() 209 210 def GetPerfReportFiles(self): 211 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 212 213 def GetDataMeasurementsFiles(self): 214 result = self.FindFilesInResultsDir('-name perf_measurements').splitlines() 215 if not result: 216 result = \ 217 self.FindFilesInResultsDir('-name results-chart.json').splitlines() 218 return result 219 220 def GeneratePerfReportFiles(self): 221 perf_report_files = [] 222 for perf_data_file in self.perf_data_files: 223 # Generate a perf.report and store it side-by-side with the perf.data 224 # file. 225 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 226 perf_data_file) 227 perf_report_file = '%s.report' % perf_data_file 228 if os.path.exists(perf_report_file): 229 raise RuntimeError('Perf report file already exists: %s' % 230 perf_report_file) 231 chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, 232 perf_report_file) 233 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 234 235 perf_file = '/usr/sbin/perf' 236 if os.path.exists(perf_path): 237 perf_file = '/usr/bin/perf' 238 239 # The following is a hack, to use the perf.static binary that 240 # was given to us by Stephane Eranian, until he can figure out 241 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 242 # Get the directory containing the 'crosperf' script. 243 dirname, _ = misc.GetRoot(sys.argv[0]) 244 perf_path = os.path.join(dirname, '..', 'perf.static') 245 if os.path.exists(perf_path): 246 # copy the executable into the chroot so that it can be found. 247 src_path = perf_path 248 dst_path = os.path.join(self.chromeos_root, 'chroot', 249 'tmp/perf.static') 250 command = 'cp %s %s' % (src_path, dst_path) 251 self.ce.RunCommand(command) 252 perf_file = '/tmp/perf.static' 253 254 command = ('%s report ' 255 '-n ' 256 '--symfs /build/%s ' 257 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 258 '--kallsyms /build/%s/boot/System.map-* ' 259 '-i %s --stdio ' 260 '> %s' % (perf_file, self.board, self.board, self.board, 261 chroot_perf_data_file, chroot_perf_report_file)) 262 self.ce.ChrootRunCommand(self.chromeos_root, command) 263 264 # Add a keyval to the dictionary for the events captured. 265 perf_report_files.append(misc.GetOutsideChrootPath( 266 self.chromeos_root, chroot_perf_report_file)) 267 return perf_report_files 268 269 def GatherPerfResults(self): 270 report_id = 0 271 for perf_report_file in self.perf_report_files: 272 with open(perf_report_file, 'r') as f: 273 report_contents = f.read() 274 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 275 num_events = group[0] 276 event_name = group[1] 277 key = 'perf_%s_%s' % (report_id, event_name) 278 value = str(misc.UnitToNumber(num_events)) 279 self.keyvals[key] = value 280 281 def PopulateFromRun(self, out, err, retval, test, suite): 282 self.board = self.label.board 283 self.out = out 284 self.err = err 285 self.retval = retval 286 self.test_name = test 287 self.suite = suite 288 self.chroot_results_dir = self.GetResultsDir() 289 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 290 self.chroot_results_dir) 291 self.results_file = self.GetResultsFile() 292 self.perf_data_files = self.GetPerfDataFiles() 293 # Include all perf.report data in table. 294 self.perf_report_files = self.GeneratePerfReportFiles() 295 # TODO(asharif): Do something similar with perf stat. 296 297 # Grab keyvals from the directory. 298 self.ProcessResults() 299 300 def ProcessJsonResults(self): 301 # Open and parse the json results file generated by telemetry/test_that. 302 if not self.results_file: 303 raise IOError('No results file found.') 304 filename = self.results_file[0] 305 if not filename.endswith('.json'): 306 raise IOError('Attempt to call json on non-json file: %s' % filename) 307 308 if not os.path.exists(filename): 309 return {} 310 311 keyvals = {} 312 with open(filename, 'r') as f: 313 raw_dict = json.load(f) 314 if 'charts' in raw_dict: 315 raw_dict = raw_dict['charts'] 316 for k, field_dict in raw_dict.iteritems(): 317 for item in field_dict: 318 keyname = k + "__" + item 319 value_dict = field_dict[item] 320 if 'value' in value_dict: 321 result = value_dict['value'] 322 elif 'values' in value_dict: 323 if not value_dict['values']: 324 continue 325 result = value_dict['values'] 326 units = value_dict['units'] 327 new_value = [result, units] 328 keyvals[keyname] = new_value 329 return keyvals 330 331 def ProcessResults(self, use_cache=False): 332 # Note that this function doesn't know anything about whether there is a 333 # cache hit or miss. It should process results agnostic of the cache hit 334 # state. 335 if self.results_file and self.results_file[0].find('results_chart.json'): 336 self.keyvals = self.ProcessJsonResults() 337 else: 338 if not use_cache: 339 print('\n ** WARNING **: Had to use deprecated output-method to ' 340 'collect results.\n') 341 self.keyvals = self.GetKeyvals() 342 self.keyvals['retval'] = self.retval 343 # Generate report from all perf.data files. 344 # Now parse all perf report files and include them in keyvals. 345 self.GatherPerfResults() 346 347 def GetChromeVersionFromCache(self, cache_dir): 348 # Read chrome_version from keys file, if present. 349 chrome_version = '' 350 keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) 351 if os.path.exists(keys_file): 352 with open(keys_file, 'r') as f: 353 lines = f.readlines() 354 for l in lines: 355 if l.startswith('Google Chrome '): 356 chrome_version = l 357 if chrome_version.endswith('\n'): 358 chrome_version = chrome_version[:-1] 359 break 360 return chrome_version 361 362 def PopulateFromCacheDir(self, cache_dir, test, suite): 363 self.test_name = test 364 self.suite = suite 365 # Read in everything from the cache directory. 366 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 367 self.out = pickle.load(f) 368 self.err = pickle.load(f) 369 self.retval = pickle.load(f) 370 371 # Untar the tarball to a temporary directory 372 self.temp_dir = tempfile.mkdtemp( 373 dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) 374 375 command = ('cd %s && tar xf %s' % 376 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 377 ret = self.ce.RunCommand(command, print_to_console=False) 378 if ret: 379 raise RuntimeError('Could not untar cached tarball') 380 self.results_dir = self.temp_dir 381 self.perf_data_files = self.GetPerfDataFiles() 382 self.perf_report_files = self.GetPerfReportFiles() 383 self.chrome_version = self.GetChromeVersionFromCache(cache_dir) 384 self.ProcessResults(use_cache=True) 385 386 def CleanUp(self, rm_chroot_tmp): 387 if rm_chroot_tmp and self.results_dir: 388 dirname, basename = misc.GetRoot(self.results_dir) 389 if basename.find('test_that_results_') != -1: 390 command = 'rm -rf %s' % self.results_dir 391 else: 392 command = 'rm -rf %s' % dirname 393 self.ce.RunCommand(command) 394 if self.temp_dir: 395 command = 'rm -rf %s' % self.temp_dir 396 self.ce.RunCommand(command) 397 398 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 399 # Create the dir if it doesn't exist. 400 temp_dir = tempfile.mkdtemp() 401 402 # Store to the temp directory. 403 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 404 pickle.dump(self.out, f) 405 pickle.dump(self.err, f) 406 pickle.dump(self.retval, f) 407 408 if not test_flag.GetTestMode(): 409 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 410 f.write('%s\n' % self.label.name) 411 f.write('%s\n' % self.label.chrome_version) 412 f.write('%s\n' % self.machine.checksum_string) 413 for k in key_list: 414 f.write(k) 415 f.write('\n') 416 417 if self.results_dir: 418 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 419 command = ('cd %s && ' 420 'tar ' 421 '--exclude=var/spool ' 422 '--exclude=var/log ' 423 '-cjf %s .' % (self.results_dir, tarball)) 424 ret = self.ce.RunCommand(command) 425 if ret: 426 raise RuntimeError("Couldn't store autotest output directory.") 427 # Store machine info. 428 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 429 # this function. 430 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 431 f.write(machine_manager.machine_checksum_string[self.label.name]) 432 433 if os.path.exists(cache_dir): 434 command = 'rm -rf {0}'.format(cache_dir) 435 self.ce.RunCommand(command) 436 437 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 438 command += 'chmod g+x {0} && '.format(temp_dir) 439 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 440 ret = self.ce.RunCommand(command) 441 if ret: 442 command = 'rm -rf {0}'.format(temp_dir) 443 self.ce.RunCommand(command) 444 raise RuntimeError('Could not move dir %s to dir %s' % 445 (temp_dir, cache_dir)) 446 447 @classmethod 448 def CreateFromRun(cls, 449 logger, 450 log_level, 451 label, 452 machine, 453 out, 454 err, 455 retval, 456 test, 457 suite='telemetry_Crosperf'): 458 if suite == 'telemetry': 459 result = TelemetryResult(logger, label, log_level, machine) 460 else: 461 result = cls(logger, label, log_level, machine) 462 result.PopulateFromRun(out, err, retval, test, suite) 463 return result 464 465 @classmethod 466 def CreateFromCacheHit(cls, 467 logger, 468 log_level, 469 label, 470 machine, 471 cache_dir, 472 test, 473 suite='telemetry_Crosperf'): 474 if suite == 'telemetry': 475 result = TelemetryResult(logger, label, log_level, machine) 476 else: 477 result = cls(logger, label, log_level, machine) 478 try: 479 result.PopulateFromCacheDir(cache_dir, test, suite) 480 481 except RuntimeError as e: 482 logger.LogError('Exception while using cache: %s' % e) 483 return None 484 return result 485 486 487class TelemetryResult(Result): 488 """Class to hold the results of a single Telemetry run.""" 489 490 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 491 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 492 cmd_exec) 493 494 def PopulateFromRun(self, out, err, retval, test, suite): 495 self.out = out 496 self.err = err 497 self.retval = retval 498 499 self.ProcessResults() 500 501 def ProcessResults(self): 502 # The output is: 503 # url,average_commit_time (ms),... 504 # www.google.com,33.4,21.2,... 505 # We need to convert to this format: 506 # {"www.google.com:average_commit_time (ms)": "33.4", 507 # "www.google.com:...": "21.2"} 508 # Added note: Occasionally the output comes back 509 # with "JSON.stringify(window.automation.GetResults())" on 510 # the first line, and then the rest of the output as 511 # described above. 512 513 lines = self.out.splitlines() 514 self.keyvals = {} 515 516 if lines: 517 if lines[0].startswith('JSON.stringify'): 518 lines = lines[1:] 519 520 if not lines: 521 return 522 labels = lines[0].split(',') 523 for line in lines[1:]: 524 fields = line.split(',') 525 if len(fields) != len(labels): 526 continue 527 for i in xrange(1, len(labels)): 528 key = '%s %s' % (fields[0], labels[i]) 529 value = fields[i] 530 self.keyvals[key] = value 531 self.keyvals['retval'] = self.retval 532 533 def PopulateFromCacheDir(self, cache_dir, test, suite): 534 self.test_name = test 535 self.suite = suite 536 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 537 self.out = pickle.load(f) 538 self.err = pickle.load(f) 539 self.retval = pickle.load(f) 540 541 self.chrome_version = \ 542 super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) 543 self.ProcessResults() 544 545 546class CacheConditions(object): 547 """Various Cache condition values, for export.""" 548 549 # Cache hit only if the result file exists. 550 CACHE_FILE_EXISTS = 0 551 552 # Cache hit if the checksum of cpuinfo and totalmem of 553 # the cached result and the new run match. 554 MACHINES_MATCH = 1 555 556 # Cache hit if the image checksum of the cached result and the new run match. 557 CHECKSUMS_MATCH = 2 558 559 # Cache hit only if the cached result was successful 560 RUN_SUCCEEDED = 3 561 562 # Never a cache hit. 563 FALSE = 4 564 565 # Cache hit if the image path matches the cached image path. 566 IMAGE_PATH_MATCH = 5 567 568 # Cache hit if the uuid of hard disk mataches the cached one 569 570 SAME_MACHINE_MATCH = 6 571 572 573class ResultsCache(object): 574 """Class to handle the cache for storing/retrieving test run results. 575 576 This class manages the key of the cached runs without worrying about what 577 is exactly stored (value). The value generation is handled by the Results 578 class. 579 """ 580 CACHE_VERSION = 6 581 582 def __init__(self): 583 # Proper initialization happens in the Init function below. 584 self.chromeos_image = None 585 self.chromeos_root = None 586 self.test_name = None 587 self.iteration = None 588 self.test_args = None 589 self.profiler_args = None 590 self.board = None 591 self.cache_conditions = None 592 self.machine_manager = None 593 self.machine = None 594 self._logger = None 595 self.ce = None 596 self.label = None 597 self.share_cache = None 598 self.suite = None 599 self.log_level = None 600 self.show_all = None 601 self.run_local = None 602 603 604 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 605 profiler_args, machine_manager, machine, board, cache_conditions, 606 logger_to_use, log_level, label, share_cache, suite, 607 show_all_results, run_local): 608 self.chromeos_image = chromeos_image 609 self.chromeos_root = chromeos_root 610 self.test_name = test_name 611 self.iteration = iteration 612 self.test_args = test_args 613 self.profiler_args = profiler_args 614 self.board = board 615 self.cache_conditions = cache_conditions 616 self.machine_manager = machine_manager 617 self.machine = machine 618 self._logger = logger_to_use 619 self.ce = command_executer.GetCommandExecuter(self._logger, 620 log_level=log_level) 621 self.label = label 622 self.share_cache = share_cache 623 self.suite = suite 624 self.log_level = log_level 625 self.show_all = show_all_results 626 self.run_local = run_local 627 628 def GetCacheDirForRead(self): 629 matching_dirs = [] 630 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 631 matching_dirs += glob.glob(glob_path) 632 633 if matching_dirs: 634 # Cache file found. 635 return matching_dirs[0] 636 return None 637 638 def GetCacheDirForWrite(self, get_keylist=False): 639 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 640 if get_keylist: 641 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 642 self.run_local) 643 version, image = results_report.ParseChromeosImage( 644 self.label.chromeos_image) 645 keylist = [version, image, self.label.board, self.machine.name, 646 self.test_name, str(self.iteration), args_str] 647 return cache_path, keylist 648 return cache_path 649 650 def FormCacheDir(self, list_of_strings): 651 cache_key = ' '.join(list_of_strings) 652 cache_dir = misc.GetFilenameFromString(cache_key) 653 if self.label.cache_dir: 654 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 655 cache_path = [os.path.join(cache_home, cache_dir)] 656 else: 657 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 658 659 if len(self.share_cache): 660 for path in [x.strip() for x in self.share_cache.split(',')]: 661 if os.path.exists(path): 662 cache_path.append(os.path.join(path, cache_dir)) 663 else: 664 self._logger.LogFatal('Unable to find shared cache: %s' % path) 665 666 return cache_path 667 668 def GetCacheKeyList(self, read): 669 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 670 machine_checksum = '*' 671 else: 672 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 673 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 674 checksum = '*' 675 elif self.label.image_type == 'trybot': 676 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 677 elif self.label.image_type == 'official': 678 checksum = '*' 679 else: 680 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 681 682 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 683 image_path_checksum = '*' 684 else: 685 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 686 687 machine_id_checksum = '' 688 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 689 machine_id_checksum = '*' 690 else: 691 if self.machine and self.machine.name in self.label.remote: 692 machine_id_checksum = self.machine.machine_id_checksum 693 else: 694 for machine in self.machine_manager.GetMachines(self.label): 695 if machine.name == self.label.remote[0]: 696 machine_id_checksum = machine.machine_id_checksum 697 break 698 699 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 700 self.run_local) 701 test_args_checksum = hashlib.md5(temp_test_args).hexdigest() 702 return (image_path_checksum, self.test_name, str(self.iteration), 703 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 704 str(self.CACHE_VERSION)) 705 706 def ReadResult(self): 707 if CacheConditions.FALSE in self.cache_conditions: 708 cache_dir = self.GetCacheDirForWrite() 709 command = 'rm -rf %s' % (cache_dir, ) 710 self.ce.RunCommand(command) 711 return None 712 cache_dir = self.GetCacheDirForRead() 713 714 if not cache_dir: 715 return None 716 717 if not os.path.isdir(cache_dir): 718 return None 719 720 if self.log_level == 'verbose': 721 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 722 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 723 self.machine, cache_dir, 724 self.test_name, self.suite) 725 if not result: 726 return None 727 728 if (result.retval == 0 or 729 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 730 return result 731 732 return None 733 734 def StoreResult(self, result): 735 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 736 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 737 738 739class MockResultsCache(ResultsCache): 740 """Class for mock testing, corresponding to ResultsCache class.""" 741 742 def Init(self, *args): 743 pass 744 745 def ReadResult(self): 746 return None 747 748 def StoreResult(self, result): 749 pass 750 751 752class MockResult(Result): 753 """Class for mock testing, corresponding to Result class.""" 754 755 def PopulateFromRun(self, out, err, retval, test, suite): 756 self.out = out 757 self.err = err 758 self.retval = retval 759