results_cache.py revision 655dfa3bd83f6abbc7fe7cc0fa14ad38f379dcb6
1 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Module to deal with result cache.""" 6 7import getpass 8import glob 9import hashlib 10import os 11import pickle 12import re 13import tempfile 14import json 15import sys 16 17from cros_utils import command_executer 18from cros_utils import misc 19 20from image_checksummer import ImageChecksummer 21 22import results_report 23import test_flag 24 25SCRATCH_DIR = os.path.expanduser('~/cros_scratch') 26RESULTS_FILE = 'results.txt' 27MACHINE_FILE = 'machine.txt' 28AUTOTEST_TARBALL = 'autotest.tbz2' 29PERF_RESULTS_FILE = 'perf-results.txt' 30CACHE_KEYS_FILE = 'cache_keys.txt' 31 32 33class Result(object): 34 """ This class manages what exactly is stored inside the cache without knowing 35 what the key of the cache is. For runs with perf, it stores perf.data, 36 perf.report, etc. The key generation is handled by the ResultsCache class. 37 """ 38 39 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 40 self.chromeos_root = label.chromeos_root 41 self._logger = logger 42 self.ce = cmd_exec or command_executer.GetCommandExecuter( 43 self._logger, 44 log_level=log_level) 45 self.temp_dir = None 46 self.label = label 47 self.results_dir = None 48 self.log_level = log_level 49 self.machine = machine 50 self.perf_data_files = [] 51 self.perf_report_files = [] 52 53 def CopyFilesTo(self, dest_dir, files_to_copy): 54 file_index = 0 55 for file_to_copy in files_to_copy: 56 if not os.path.isdir(dest_dir): 57 command = 'mkdir -p %s' % dest_dir 58 self.ce.RunCommand(command) 59 dest_file = os.path.join(dest_dir, 60 ('%s.%s' % (os.path.basename(file_to_copy), 61 file_index))) 62 ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 63 if ret: 64 raise Exception('Could not copy results file: %s' % file_to_copy) 65 66 def CopyResultsTo(self, dest_dir): 67 self.CopyFilesTo(dest_dir, self.perf_data_files) 68 self.CopyFilesTo(dest_dir, self.perf_report_files) 69 if len(self.perf_data_files) or len(self.perf_report_files): 70 self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 71 72 def GetNewKeyvals(self, keyvals_dict): 73 # Initialize 'units' dictionary. 74 units_dict = {} 75 for k in keyvals_dict: 76 units_dict[k] = '' 77 results_files = self.GetDataMeasurementsFiles() 78 for f in results_files: 79 # Make sure we can find the results file 80 if os.path.exists(f): 81 data_filename = f 82 else: 83 # Otherwise get the base filename and create the correct 84 # path for it. 85 f_dir, f_base = misc.GetRoot(f) 86 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 87 self.temp_dir, f_base) 88 if os.path.exists(data_filename): 89 with open(data_filename, 'r') as data_file: 90 lines = data_file.readlines() 91 for line in lines: 92 tmp_dict = json.loads(line) 93 graph_name = tmp_dict['graph'] 94 graph_str = (graph_name + '__') if graph_name else '' 95 key = graph_str + tmp_dict['description'] 96 keyvals_dict[key] = tmp_dict['value'] 97 units_dict[key] = tmp_dict['units'] 98 99 return keyvals_dict, units_dict 100 101 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 102 """keyvals_dict is the dictionary of key-value pairs that is used for generating Crosperf reports. 103 104 units_dict is a dictionary of the units for the return values in 105 keyvals_dict. We need to associate the units with the return values, 106 for Telemetry tests, so that we can include the units in the reports. 107 This function takes each value in keyvals_dict, finds the corresponding 108 unit in the units_dict, and replaces the old value with a list of the 109 old value and the units. This later gets properly parsed in the 110 ResultOrganizer class, for generating the reports. 111 112 """ 113 114 results_dict = {} 115 for k in keyvals_dict: 116 # We don't want these lines in our reports; they add no useful data. 117 if k == '' or k == 'telemetry_Crosperf': 118 continue 119 val = keyvals_dict[k] 120 units = units_dict[k] 121 new_val = [val, units] 122 results_dict[k] = new_val 123 return results_dict 124 125 def GetKeyvals(self, show_all): 126 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 127 if not self.temp_dir: 128 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 129 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 130 self.ce.RunCommand(command, print_to_console=False) 131 132 command = ('python generate_test_report --no-color --csv %s' % 133 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 134 _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root, 135 command, 136 print_to_console=False) 137 keyvals_dict = {} 138 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 139 self.temp_dir) 140 for line in out.splitlines(): 141 tokens = re.split('=|,', line) 142 key = tokens[-2] 143 if key.startswith(tmp_dir_in_chroot): 144 key = key[len(tmp_dir_in_chroot) + 1:] 145 value = tokens[-1] 146 keyvals_dict[key] = value 147 148 # Check to see if there is a perf_measurements file and get the 149 # data from it if so. 150 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 151 if self.suite == 'telemetry_Crosperf': 152 # For telemtry_Crosperf results, append the units to the return 153 # results, for use in generating the reports. 154 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 155 return keyvals_dict 156 157 def GetResultsDir(self): 158 mo = re.search(r'Results placed in (\S+)', self.out) 159 if mo: 160 result = mo.group(1) 161 return result 162 raise Exception('Could not find results directory.') 163 164 def FindFilesInResultsDir(self, find_args): 165 if not self.results_dir: 166 return None 167 168 command = 'find %s %s' % (self.results_dir, find_args) 169 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 170 if ret: 171 raise Exception('Could not run find command!') 172 return out 173 174 def GetPerfDataFiles(self): 175 return self.FindFilesInResultsDir('-name perf.data').splitlines() 176 177 def GetPerfReportFiles(self): 178 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 179 180 def GetDataMeasurementsFiles(self): 181 return self.FindFilesInResultsDir('-name perf_measurements').splitlines() 182 183 def GeneratePerfReportFiles(self): 184 perf_report_files = [] 185 for perf_data_file in self.perf_data_files: 186 # Generate a perf.report and store it side-by-side with the perf.data 187 # file. 188 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 189 perf_data_file) 190 perf_report_file = '%s.report' % perf_data_file 191 if os.path.exists(perf_report_file): 192 raise Exception('Perf report file already exists: %s' % 193 perf_report_file) 194 chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, 195 perf_report_file) 196 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 197 198 perf_file = '/usr/sbin/perf' 199 if os.path.exists(perf_path): 200 perf_file = '/usr/bin/perf' 201 202 # The following is a hack, to use the perf.static binary that 203 # was given to us by Stephane Eranian, until he can figure out 204 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 205 # Get the directory containing the 'crosperf' script. 206 dirname, _ = misc.GetRoot(sys.argv[0]) 207 perf_path = os.path.join(dirname, '..', 'perf.static') 208 if os.path.exists(perf_path): 209 # copy the executable into the chroot so that it can be found. 210 src_path = perf_path 211 dst_path = os.path.join(self.chromeos_root, 'chroot', 212 'tmp/perf.static') 213 command = 'cp %s %s' % (src_path, dst_path) 214 self.ce.RunCommand(command) 215 perf_file = '/tmp/perf.static' 216 217 command = ('%s report ' 218 '-n ' 219 '--symfs /build/%s ' 220 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 221 '--kallsyms /build/%s/boot/System.map-* ' 222 '-i %s --stdio ' 223 '> %s' % (perf_file, self.board, self.board, self.board, 224 chroot_perf_data_file, chroot_perf_report_file)) 225 self.ce.ChrootRunCommand(self.chromeos_root, command) 226 227 # Add a keyval to the dictionary for the events captured. 228 perf_report_files.append(misc.GetOutsideChrootPath( 229 self.chromeos_root, chroot_perf_report_file)) 230 return perf_report_files 231 232 def GatherPerfResults(self): 233 report_id = 0 234 for perf_report_file in self.perf_report_files: 235 with open(perf_report_file, 'r') as f: 236 report_contents = f.read() 237 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 238 num_events = group[0] 239 event_name = group[1] 240 key = 'perf_%s_%s' % (report_id, event_name) 241 value = str(misc.UnitToNumber(num_events)) 242 self.keyvals[key] = value 243 244 def PopulateFromRun(self, out, err, retval, show_all, test, suite): 245 self.board = self.label.board 246 self.out = out 247 self.err = err 248 self.retval = retval 249 self.test_name = test 250 self.suite = suite 251 self.chroot_results_dir = self.GetResultsDir() 252 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 253 self.chroot_results_dir) 254 self.perf_data_files = self.GetPerfDataFiles() 255 # Include all perf.report data in table. 256 self.perf_report_files = self.GeneratePerfReportFiles() 257 # TODO(asharif): Do something similar with perf stat. 258 259 # Grab keyvals from the directory. 260 self.ProcessResults(show_all) 261 262 def ProcessResults(self, show_all): 263 # Note that this function doesn't know anything about whether there is a 264 # cache hit or miss. It should process results agnostic of the cache hit 265 # state. 266 self.keyvals = self.GetKeyvals(show_all) 267 self.keyvals['retval'] = self.retval 268 # Generate report from all perf.data files. 269 # Now parse all perf report files and include them in keyvals. 270 self.GatherPerfResults() 271 272 def PopulateFromCacheDir(self, cache_dir, show_all, test, suite): 273 self.test_name = test 274 self.suite = suite 275 # Read in everything from the cache directory. 276 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 277 self.out = pickle.load(f) 278 self.err = pickle.load(f) 279 self.retval = pickle.load(f) 280 281 # Untar the tarball to a temporary directory 282 self.temp_dir = tempfile.mkdtemp( 283 dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) 284 285 command = ('cd %s && tar xf %s' % 286 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 287 ret = self.ce.RunCommand(command, print_to_console=False) 288 if ret: 289 raise Exception('Could not untar cached tarball') 290 self.results_dir = self.temp_dir 291 self.perf_data_files = self.GetPerfDataFiles() 292 self.perf_report_files = self.GetPerfReportFiles() 293 self.ProcessResults(show_all) 294 295 def CleanUp(self, rm_chroot_tmp): 296 if rm_chroot_tmp and self.results_dir: 297 dirname, basename = misc.GetRoot(self.results_dir) 298 if basename.find('test_that_results_') != -1: 299 command = 'rm -rf %s' % self.results_dir 300 else: 301 command = 'rm -rf %s' % dirname 302 self.ce.RunCommand(command) 303 if self.temp_dir: 304 command = 'rm -rf %s' % self.temp_dir 305 self.ce.RunCommand(command) 306 307 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 308 # Create the dir if it doesn't exist. 309 temp_dir = tempfile.mkdtemp() 310 311 # Store to the temp directory. 312 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 313 pickle.dump(self.out, f) 314 pickle.dump(self.err, f) 315 pickle.dump(self.retval, f) 316 317 if not test_flag.GetTestMode(): 318 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 319 f.write('%s\n' % self.label.name) 320 f.write('%s\n' % self.label.chrome_version) 321 f.write('%s\n' % self.machine.checksum_string) 322 for k in key_list: 323 f.write(k) 324 f.write('\n') 325 326 if self.results_dir: 327 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 328 command = ('cd %s && ' 329 'tar ' 330 '--exclude=var/spool ' 331 '--exclude=var/log ' 332 '-cjf %s .' % (self.results_dir, tarball)) 333 ret = self.ce.RunCommand(command) 334 if ret: 335 raise Exception("Couldn't store autotest output directory.") 336 # Store machine info. 337 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 338 # this function. 339 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 340 f.write(machine_manager.machine_checksum_string[self.label.name]) 341 342 if os.path.exists(cache_dir): 343 command = 'rm -rf {0}'.format(cache_dir) 344 self.ce.RunCommand(command) 345 346 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 347 command += 'chmod g+x {0} && '.format(temp_dir) 348 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 349 ret = self.ce.RunCommand(command) 350 if ret: 351 command = 'rm -rf {0}'.format(temp_dir) 352 self.ce.RunCommand(command) 353 raise Exception('Could not move dir %s to dir %s' % (temp_dir, cache_dir)) 354 355 @classmethod 356 def CreateFromRun(cls, 357 logger, 358 log_level, 359 label, 360 machine, 361 out, 362 err, 363 retval, 364 show_all, 365 test, 366 suite='telemetry_Crosperf'): 367 if suite == 'telemetry': 368 result = TelemetryResult(logger, label, log_level, machine) 369 else: 370 result = cls(logger, label, log_level, machine) 371 result.PopulateFromRun(out, err, retval, show_all, test, suite) 372 return result 373 374 @classmethod 375 def CreateFromCacheHit(cls, 376 logger, 377 log_level, 378 label, 379 machine, 380 cache_dir, 381 show_all, 382 test, 383 suite='telemetry_Crosperf'): 384 if suite == 'telemetry': 385 result = TelemetryResult(logger, label, log_level, machine) 386 else: 387 result = cls(logger, label, log_level, machine) 388 try: 389 result.PopulateFromCacheDir(cache_dir, show_all, test, suite) 390 391 except Exception as e: 392 logger.LogError('Exception while using cache: %s' % e) 393 return None 394 return result 395 396 397class TelemetryResult(Result): 398 399 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 400 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 401 cmd_exec) 402 403 def PopulateFromRun(self, out, err, retval, show_all, test, suite): 404 self.out = out 405 self.err = err 406 self.retval = retval 407 408 self.ProcessResults() 409 410 def ProcessResults(self): 411 # The output is: 412 # url,average_commit_time (ms),... 413 # www.google.com,33.4,21.2,... 414 # We need to convert to this format: 415 # {"www.google.com:average_commit_time (ms)": "33.4", 416 # "www.google.com:...": "21.2"} 417 # Added note: Occasionally the output comes back 418 # with "JSON.stringify(window.automation.GetResults())" on 419 # the first line, and then the rest of the output as 420 # described above. 421 422 lines = self.out.splitlines() 423 self.keyvals = {} 424 425 if lines: 426 if lines[0].startswith('JSON.stringify'): 427 lines = lines[1:] 428 429 if not lines: 430 return 431 labels = lines[0].split(',') 432 for line in lines[1:]: 433 fields = line.split(',') 434 if len(fields) != len(labels): 435 continue 436 for i in range(1, len(labels)): 437 key = '%s %s' % (fields[0], labels[i]) 438 value = fields[i] 439 self.keyvals[key] = value 440 self.keyvals['retval'] = self.retval 441 442 def PopulateFromCacheDir(self, cache_dir): 443 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 444 self.out = pickle.load(f) 445 self.err = pickle.load(f) 446 self.retval = pickle.load(f) 447 self.ProcessResults() 448 449 450class CacheConditions(object): 451 # Cache hit only if the result file exists. 452 CACHE_FILE_EXISTS = 0 453 454 # Cache hit if the checksum of cpuinfo and totalmem of 455 # the cached result and the new run match. 456 MACHINES_MATCH = 1 457 458 # Cache hit if the image checksum of the cached result and the new run match. 459 CHECKSUMS_MATCH = 2 460 461 # Cache hit only if the cached result was successful 462 RUN_SUCCEEDED = 3 463 464 # Never a cache hit. 465 FALSE = 4 466 467 # Cache hit if the image path matches the cached image path. 468 IMAGE_PATH_MATCH = 5 469 470 # Cache hit if the uuid of hard disk mataches the cached one 471 472 SAME_MACHINE_MATCH = 6 473 474 475class ResultsCache(object): 476 """ This class manages the key of the cached runs without worrying about what 477 is exactly stored (value). The value generation is handled by the Results 478 class. 479 """ 480 CACHE_VERSION = 6 481 482 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 483 profiler_args, machine_manager, machine, board, cache_conditions, 484 logger_to_use, log_level, label, share_cache, suite, 485 show_all_results, run_local): 486 self.chromeos_image = chromeos_image 487 self.chromeos_root = chromeos_root 488 self.test_name = test_name 489 self.iteration = iteration 490 self.test_args = test_args 491 self.profiler_args = profiler_args 492 self.board = board 493 self.cache_conditions = cache_conditions 494 self.machine_manager = machine_manager 495 self.machine = machine 496 self._logger = logger_to_use 497 self.ce = command_executer.GetCommandExecuter(self._logger, 498 log_level=log_level) 499 self.label = label 500 self.share_cache = share_cache 501 self.suite = suite 502 self.log_level = log_level 503 self.show_all = show_all_results 504 self.run_local = run_local 505 506 def GetCacheDirForRead(self): 507 matching_dirs = [] 508 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 509 matching_dirs += glob.glob(glob_path) 510 511 if matching_dirs: 512 # Cache file found. 513 return matching_dirs[0] 514 else: 515 return None 516 517 def GetCacheDirForWrite(self, get_keylist=False): 518 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 519 if get_keylist: 520 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 521 self.run_local) 522 version, image = results_report.ParseChromeosImage( 523 self.label.chromeos_image) 524 keylist = [version, image, self.label.board, self.machine.name, 525 self.test_name, str(self.iteration), args_str] 526 return cache_path, keylist 527 return cache_path 528 529 def FormCacheDir(self, list_of_strings): 530 cache_key = ' '.join(list_of_strings) 531 cache_dir = misc.GetFilenameFromString(cache_key) 532 if self.label.cache_dir: 533 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 534 cache_path = [os.path.join(cache_home, cache_dir)] 535 else: 536 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 537 538 if len(self.share_cache): 539 for path in [x.strip() for x in self.share_cache.split(',')]: 540 if os.path.exists(path): 541 cache_path.append(os.path.join(path, cache_dir)) 542 else: 543 self._logger.LogFatal('Unable to find shared cache: %s' % path) 544 545 return cache_path 546 547 def GetCacheKeyList(self, read): 548 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 549 machine_checksum = '*' 550 else: 551 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 552 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 553 checksum = '*' 554 elif self.label.image_type == 'trybot': 555 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 556 elif self.label.image_type == 'official': 557 checksum = '*' 558 else: 559 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 560 561 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 562 image_path_checksum = '*' 563 else: 564 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 565 566 machine_id_checksum = '' 567 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 568 machine_id_checksum = '*' 569 else: 570 if self.machine and self.machine.name in self.label.remote: 571 machine_id_checksum = self.machine.machine_id_checksum 572 else: 573 for machine in self.machine_manager.GetMachines(self.label): 574 if machine.name == self.label.remote[0]: 575 machine_id_checksum = machine.machine_id_checksum 576 break 577 578 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 579 self.run_local) 580 test_args_checksum = hashlib.md5(''.join(temp_test_args)).hexdigest() 581 return (image_path_checksum, self.test_name, str(self.iteration), 582 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 583 str(self.CACHE_VERSION)) 584 585 def ReadResult(self): 586 if CacheConditions.FALSE in self.cache_conditions: 587 cache_dir = self.GetCacheDirForWrite() 588 command = 'rm -rf {0}'.format(cache_dir) 589 self.ce.RunCommand(command) 590 return None 591 cache_dir = self.GetCacheDirForRead() 592 593 if not cache_dir: 594 return None 595 596 if not os.path.isdir(cache_dir): 597 return None 598 599 if self.log_level == 'verbose': 600 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 601 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 602 self.machine, cache_dir, self.show_all, 603 self.test_name, self.suite) 604 if not result: 605 return None 606 607 if (result.retval == 0 or 608 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 609 return result 610 611 return None 612 613 def StoreResult(self, result): 614 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 615 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 616 617 618class MockResultsCache(ResultsCache): 619 620 def Init(self, *args): 621 pass 622 623 def ReadResult(self): 624 return None 625 626 def StoreResult(self, result): 627 pass 628 629 630class MockResult(Result): 631 632 def PopulateFromRun(self, out, err, retval, show_all, test, suite): 633 self.out = out 634 self.err = err 635 self.retval = retval 636