results_cache.py revision 25d32b4add9a0d0ac4ec682062e45baea97bff04
1 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Module to deal with result cache.""" 6 7import getpass 8import glob 9import hashlib 10import os 11import pickle 12import re 13import tempfile 14import json 15import sys 16 17from cros_utils import command_executer 18from cros_utils import misc 19 20from image_checksummer import ImageChecksummer 21 22import results_report 23import test_flag 24 25SCRATCH_DIR = os.path.expanduser('~/cros_scratch') 26RESULTS_FILE = 'results.txt' 27MACHINE_FILE = 'machine.txt' 28AUTOTEST_TARBALL = 'autotest.tbz2' 29PERF_RESULTS_FILE = 'perf-results.txt' 30CACHE_KEYS_FILE = 'cache_keys.txt' 31 32 33class Result(object): 34 """ This class manages what exactly is stored inside the cache without knowing 35 what the key of the cache is. For runs with perf, it stores perf.data, 36 perf.report, etc. The key generation is handled by the ResultsCache class. 37 """ 38 39 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 40 self.chromeos_root = label.chromeos_root 41 self._logger = logger 42 self.ce = cmd_exec or command_executer.GetCommandExecuter( 43 self._logger, 44 log_level=log_level) 45 self.temp_dir = None 46 self.label = label 47 self.results_dir = None 48 self.log_level = log_level 49 self.machine = machine 50 self.perf_data_files = [] 51 self.perf_report_files = [] 52 self.chrome_version = '' 53 54 def CopyFilesTo(self, dest_dir, files_to_copy): 55 file_index = 0 56 for file_to_copy in files_to_copy: 57 if not os.path.isdir(dest_dir): 58 command = 'mkdir -p %s' % dest_dir 59 self.ce.RunCommand(command) 60 dest_file = os.path.join(dest_dir, 61 ('%s.%s' % (os.path.basename(file_to_copy), 62 file_index))) 63 ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 64 if ret: 65 raise Exception('Could not copy results file: %s' % file_to_copy) 66 67 def CopyResultsTo(self, dest_dir): 68 self.CopyFilesTo(dest_dir, self.perf_data_files) 69 self.CopyFilesTo(dest_dir, self.perf_report_files) 70 if len(self.perf_data_files) or len(self.perf_report_files): 71 self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 72 73 def GetNewKeyvals(self, keyvals_dict): 74 # Initialize 'units' dictionary. 75 units_dict = {} 76 for k in keyvals_dict: 77 units_dict[k] = '' 78 results_files = self.GetDataMeasurementsFiles() 79 for f in results_files: 80 # Make sure we can find the results file 81 if os.path.exists(f): 82 data_filename = f 83 else: 84 # Otherwise get the base filename and create the correct 85 # path for it. 86 f_dir, f_base = misc.GetRoot(f) 87 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 88 self.temp_dir, f_base) 89 if os.path.exists(data_filename): 90 with open(data_filename, 'r') as data_file: 91 lines = data_file.readlines() 92 for line in lines: 93 tmp_dict = json.loads(line) 94 graph_name = tmp_dict['graph'] 95 graph_str = (graph_name + '__') if graph_name else '' 96 key = graph_str + tmp_dict['description'] 97 keyvals_dict[key] = tmp_dict['value'] 98 units_dict[key] = tmp_dict['units'] 99 100 return keyvals_dict, units_dict 101 102 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 103 """keyvals_dict is the dictionary of key-value pairs that is used for generating Crosperf reports. 104 105 units_dict is a dictionary of the units for the return values in 106 keyvals_dict. We need to associate the units with the return values, 107 for Telemetry tests, so that we can include the units in the reports. 108 This function takes each value in keyvals_dict, finds the corresponding 109 unit in the units_dict, and replaces the old value with a list of the 110 old value and the units. This later gets properly parsed in the 111 ResultOrganizer class, for generating the reports. 112 113 """ 114 115 results_dict = {} 116 for k in keyvals_dict: 117 # We don't want these lines in our reports; they add no useful data. 118 if k == '' or k == 'telemetry_Crosperf': 119 continue 120 val = keyvals_dict[k] 121 units = units_dict[k] 122 new_val = [val, units] 123 results_dict[k] = new_val 124 return results_dict 125 126 def GetKeyvals(self, show_all): 127 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 128 if not self.temp_dir: 129 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 130 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 131 self.ce.RunCommand(command, print_to_console=False) 132 133 command = ('python generate_test_report --no-color --csv %s' % 134 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 135 _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root, 136 command, 137 print_to_console=False) 138 keyvals_dict = {} 139 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 140 self.temp_dir) 141 for line in out.splitlines(): 142 tokens = re.split('=|,', line) 143 key = tokens[-2] 144 if key.startswith(tmp_dir_in_chroot): 145 key = key[len(tmp_dir_in_chroot) + 1:] 146 value = tokens[-1] 147 keyvals_dict[key] = value 148 149 # Check to see if there is a perf_measurements file and get the 150 # data from it if so. 151 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 152 if self.suite == 'telemetry_Crosperf': 153 # For telemtry_Crosperf results, append the units to the return 154 # results, for use in generating the reports. 155 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 156 return keyvals_dict 157 158 def GetResultsDir(self): 159 mo = re.search(r'Results placed in (\S+)', self.out) 160 if mo: 161 result = mo.group(1) 162 return result 163 raise Exception('Could not find results directory.') 164 165 def FindFilesInResultsDir(self, find_args): 166 if not self.results_dir: 167 return None 168 169 command = 'find %s %s' % (self.results_dir, find_args) 170 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 171 if ret: 172 raise Exception('Could not run find command!') 173 return out 174 175 def GetPerfDataFiles(self): 176 return self.FindFilesInResultsDir('-name perf.data').splitlines() 177 178 def GetPerfReportFiles(self): 179 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 180 181 def GetDataMeasurementsFiles(self): 182 return self.FindFilesInResultsDir('-name perf_measurements').splitlines() 183 184 def GeneratePerfReportFiles(self): 185 perf_report_files = [] 186 for perf_data_file in self.perf_data_files: 187 # Generate a perf.report and store it side-by-side with the perf.data 188 # file. 189 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 190 perf_data_file) 191 perf_report_file = '%s.report' % perf_data_file 192 if os.path.exists(perf_report_file): 193 raise Exception('Perf report file already exists: %s' % 194 perf_report_file) 195 chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, 196 perf_report_file) 197 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 198 199 perf_file = '/usr/sbin/perf' 200 if os.path.exists(perf_path): 201 perf_file = '/usr/bin/perf' 202 203 # The following is a hack, to use the perf.static binary that 204 # was given to us by Stephane Eranian, until he can figure out 205 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 206 # Get the directory containing the 'crosperf' script. 207 dirname, _ = misc.GetRoot(sys.argv[0]) 208 perf_path = os.path.join(dirname, '..', 'perf.static') 209 if os.path.exists(perf_path): 210 # copy the executable into the chroot so that it can be found. 211 src_path = perf_path 212 dst_path = os.path.join(self.chromeos_root, 'chroot', 213 'tmp/perf.static') 214 command = 'cp %s %s' % (src_path, dst_path) 215 self.ce.RunCommand(command) 216 perf_file = '/tmp/perf.static' 217 218 command = ('%s report ' 219 '-n ' 220 '--symfs /build/%s ' 221 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 222 '--kallsyms /build/%s/boot/System.map-* ' 223 '-i %s --stdio ' 224 '> %s' % (perf_file, self.board, self.board, self.board, 225 chroot_perf_data_file, chroot_perf_report_file)) 226 self.ce.ChrootRunCommand(self.chromeos_root, command) 227 228 # Add a keyval to the dictionary for the events captured. 229 perf_report_files.append(misc.GetOutsideChrootPath( 230 self.chromeos_root, chroot_perf_report_file)) 231 return perf_report_files 232 233 def GatherPerfResults(self): 234 report_id = 0 235 for perf_report_file in self.perf_report_files: 236 with open(perf_report_file, 'r') as f: 237 report_contents = f.read() 238 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 239 num_events = group[0] 240 event_name = group[1] 241 key = 'perf_%s_%s' % (report_id, event_name) 242 value = str(misc.UnitToNumber(num_events)) 243 self.keyvals[key] = value 244 245 def PopulateFromRun(self, out, err, retval, show_all, test, suite): 246 self.board = self.label.board 247 self.out = out 248 self.err = err 249 self.retval = retval 250 self.test_name = test 251 self.suite = suite 252 self.chroot_results_dir = self.GetResultsDir() 253 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 254 self.chroot_results_dir) 255 self.perf_data_files = self.GetPerfDataFiles() 256 # Include all perf.report data in table. 257 self.perf_report_files = self.GeneratePerfReportFiles() 258 # TODO(asharif): Do something similar with perf stat. 259 260 # Grab keyvals from the directory. 261 self.ProcessResults(show_all) 262 263 def ProcessResults(self, show_all): 264 # Note that this function doesn't know anything about whether there is a 265 # cache hit or miss. It should process results agnostic of the cache hit 266 # state. 267 self.keyvals = self.GetKeyvals(show_all) 268 self.keyvals['retval'] = self.retval 269 # Generate report from all perf.data files. 270 # Now parse all perf report files and include them in keyvals. 271 self.GatherPerfResults() 272 273 def GetChromeVersionFromCache(self, cache_dir): 274 # Read chrome_version from keys file, if present. 275 chrome_version = '' 276 keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) 277 if os.path.exists(keys_file): 278 with open(keys_file, 'r') as f: 279 lines = f.readlines() 280 for l in lines: 281 if l.find('Google Chrome ') == 0: 282 chrome_version = l 283 if chrome_version[-1] == '\n': 284 chrome_version = chrome_version[:-1] 285 break 286 return chrome_version 287 288 def PopulateFromCacheDir(self, cache_dir, show_all, test, suite): 289 self.test_name = test 290 self.suite = suite 291 # Read in everything from the cache directory. 292 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 293 self.out = pickle.load(f) 294 self.err = pickle.load(f) 295 self.retval = pickle.load(f) 296 297 # Untar the tarball to a temporary directory 298 self.temp_dir = tempfile.mkdtemp( 299 dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) 300 301 command = ('cd %s && tar xf %s' % 302 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 303 ret = self.ce.RunCommand(command, print_to_console=False) 304 if ret: 305 raise Exception('Could not untar cached tarball') 306 self.results_dir = self.temp_dir 307 self.perf_data_files = self.GetPerfDataFiles() 308 self.perf_report_files = self.GetPerfReportFiles() 309 self.chrome_version = self.GetChromeVersionFromCache(cache_dir) 310 self.ProcessResults(show_all) 311 312 def CleanUp(self, rm_chroot_tmp): 313 if rm_chroot_tmp and self.results_dir: 314 dirname, basename = misc.GetRoot(self.results_dir) 315 if basename.find('test_that_results_') != -1: 316 command = 'rm -rf %s' % self.results_dir 317 else: 318 command = 'rm -rf %s' % dirname 319 self.ce.RunCommand(command) 320 if self.temp_dir: 321 command = 'rm -rf %s' % self.temp_dir 322 self.ce.RunCommand(command) 323 324 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 325 # Create the dir if it doesn't exist. 326 temp_dir = tempfile.mkdtemp() 327 328 # Store to the temp directory. 329 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 330 pickle.dump(self.out, f) 331 pickle.dump(self.err, f) 332 pickle.dump(self.retval, f) 333 334 if not test_flag.GetTestMode(): 335 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 336 f.write('%s\n' % self.label.name) 337 f.write('%s\n' % self.label.chrome_version) 338 f.write('%s\n' % self.machine.checksum_string) 339 for k in key_list: 340 f.write(k) 341 f.write('\n') 342 343 if self.results_dir: 344 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 345 command = ('cd %s && ' 346 'tar ' 347 '--exclude=var/spool ' 348 '--exclude=var/log ' 349 '-cjf %s .' % (self.results_dir, tarball)) 350 ret = self.ce.RunCommand(command) 351 if ret: 352 raise Exception("Couldn't store autotest output directory.") 353 # Store machine info. 354 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 355 # this function. 356 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 357 f.write(machine_manager.machine_checksum_string[self.label.name]) 358 359 if os.path.exists(cache_dir): 360 command = 'rm -rf {0}'.format(cache_dir) 361 self.ce.RunCommand(command) 362 363 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 364 command += 'chmod g+x {0} && '.format(temp_dir) 365 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 366 ret = self.ce.RunCommand(command) 367 if ret: 368 command = 'rm -rf {0}'.format(temp_dir) 369 self.ce.RunCommand(command) 370 raise Exception('Could not move dir %s to dir %s' % (temp_dir, cache_dir)) 371 372 @classmethod 373 def CreateFromRun(cls, 374 logger, 375 log_level, 376 label, 377 machine, 378 out, 379 err, 380 retval, 381 show_all, 382 test, 383 suite='telemetry_Crosperf'): 384 if suite == 'telemetry': 385 result = TelemetryResult(logger, label, log_level, machine) 386 else: 387 result = cls(logger, label, log_level, machine) 388 result.PopulateFromRun(out, err, retval, show_all, test, suite) 389 return result 390 391 @classmethod 392 def CreateFromCacheHit(cls, 393 logger, 394 log_level, 395 label, 396 machine, 397 cache_dir, 398 show_all, 399 test, 400 suite='telemetry_Crosperf'): 401 if suite == 'telemetry': 402 result = TelemetryResult(logger, label, log_level, machine) 403 else: 404 result = cls(logger, label, log_level, machine) 405 try: 406 result.PopulateFromCacheDir(cache_dir, show_all, test, suite) 407 408 except Exception as e: 409 logger.LogError('Exception while using cache: %s' % e) 410 return None 411 return result 412 413 414class TelemetryResult(Result): 415 416 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 417 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 418 cmd_exec) 419 420 def PopulateFromRun(self, out, err, retval, show_all, test, suite): 421 self.out = out 422 self.err = err 423 self.retval = retval 424 425 self.ProcessResults() 426 427 def ProcessResults(self): 428 # The output is: 429 # url,average_commit_time (ms),... 430 # www.google.com,33.4,21.2,... 431 # We need to convert to this format: 432 # {"www.google.com:average_commit_time (ms)": "33.4", 433 # "www.google.com:...": "21.2"} 434 # Added note: Occasionally the output comes back 435 # with "JSON.stringify(window.automation.GetResults())" on 436 # the first line, and then the rest of the output as 437 # described above. 438 439 lines = self.out.splitlines() 440 self.keyvals = {} 441 442 if lines: 443 if lines[0].startswith('JSON.stringify'): 444 lines = lines[1:] 445 446 if not lines: 447 return 448 labels = lines[0].split(',') 449 for line in lines[1:]: 450 fields = line.split(',') 451 if len(fields) != len(labels): 452 continue 453 for i in range(1, len(labels)): 454 key = '%s %s' % (fields[0], labels[i]) 455 value = fields[i] 456 self.keyvals[key] = value 457 self.keyvals['retval'] = self.retval 458 459 def PopulateFromCacheDir(self, cache_dir): 460 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 461 self.out = pickle.load(f) 462 self.err = pickle.load(f) 463 self.retval = pickle.load(f) 464 465 self.chrome_version = \ 466 super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) 467 self.ProcessResults() 468 469 470class CacheConditions(object): 471 # Cache hit only if the result file exists. 472 CACHE_FILE_EXISTS = 0 473 474 # Cache hit if the checksum of cpuinfo and totalmem of 475 # the cached result and the new run match. 476 MACHINES_MATCH = 1 477 478 # Cache hit if the image checksum of the cached result and the new run match. 479 CHECKSUMS_MATCH = 2 480 481 # Cache hit only if the cached result was successful 482 RUN_SUCCEEDED = 3 483 484 # Never a cache hit. 485 FALSE = 4 486 487 # Cache hit if the image path matches the cached image path. 488 IMAGE_PATH_MATCH = 5 489 490 # Cache hit if the uuid of hard disk mataches the cached one 491 492 SAME_MACHINE_MATCH = 6 493 494 495class ResultsCache(object): 496 """ This class manages the key of the cached runs without worrying about what 497 is exactly stored (value). The value generation is handled by the Results 498 class. 499 """ 500 CACHE_VERSION = 6 501 502 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 503 profiler_args, machine_manager, machine, board, cache_conditions, 504 logger_to_use, log_level, label, share_cache, suite, 505 show_all_results, run_local): 506 self.chromeos_image = chromeos_image 507 self.chromeos_root = chromeos_root 508 self.test_name = test_name 509 self.iteration = iteration 510 self.test_args = test_args 511 self.profiler_args = profiler_args 512 self.board = board 513 self.cache_conditions = cache_conditions 514 self.machine_manager = machine_manager 515 self.machine = machine 516 self._logger = logger_to_use 517 self.ce = command_executer.GetCommandExecuter(self._logger, 518 log_level=log_level) 519 self.label = label 520 self.share_cache = share_cache 521 self.suite = suite 522 self.log_level = log_level 523 self.show_all = show_all_results 524 self.run_local = run_local 525 526 def GetCacheDirForRead(self): 527 matching_dirs = [] 528 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 529 matching_dirs += glob.glob(glob_path) 530 531 if matching_dirs: 532 # Cache file found. 533 return matching_dirs[0] 534 else: 535 return None 536 537 def GetCacheDirForWrite(self, get_keylist=False): 538 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 539 if get_keylist: 540 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 541 self.run_local) 542 version, image = results_report.ParseChromeosImage( 543 self.label.chromeos_image) 544 keylist = [version, image, self.label.board, self.machine.name, 545 self.test_name, str(self.iteration), args_str] 546 return cache_path, keylist 547 return cache_path 548 549 def FormCacheDir(self, list_of_strings): 550 cache_key = ' '.join(list_of_strings) 551 cache_dir = misc.GetFilenameFromString(cache_key) 552 if self.label.cache_dir: 553 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 554 cache_path = [os.path.join(cache_home, cache_dir)] 555 else: 556 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 557 558 if len(self.share_cache): 559 for path in [x.strip() for x in self.share_cache.split(',')]: 560 if os.path.exists(path): 561 cache_path.append(os.path.join(path, cache_dir)) 562 else: 563 self._logger.LogFatal('Unable to find shared cache: %s' % path) 564 565 return cache_path 566 567 def GetCacheKeyList(self, read): 568 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 569 machine_checksum = '*' 570 else: 571 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 572 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 573 checksum = '*' 574 elif self.label.image_type == 'trybot': 575 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 576 elif self.label.image_type == 'official': 577 checksum = '*' 578 else: 579 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 580 581 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 582 image_path_checksum = '*' 583 else: 584 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 585 586 machine_id_checksum = '' 587 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 588 machine_id_checksum = '*' 589 else: 590 if self.machine and self.machine.name in self.label.remote: 591 machine_id_checksum = self.machine.machine_id_checksum 592 else: 593 for machine in self.machine_manager.GetMachines(self.label): 594 if machine.name == self.label.remote[0]: 595 machine_id_checksum = machine.machine_id_checksum 596 break 597 598 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 599 self.run_local) 600 test_args_checksum = hashlib.md5(''.join(temp_test_args)).hexdigest() 601 return (image_path_checksum, self.test_name, str(self.iteration), 602 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 603 str(self.CACHE_VERSION)) 604 605 def ReadResult(self): 606 if CacheConditions.FALSE in self.cache_conditions: 607 cache_dir = self.GetCacheDirForWrite() 608 command = 'rm -rf {0}'.format(cache_dir) 609 self.ce.RunCommand(command) 610 return None 611 cache_dir = self.GetCacheDirForRead() 612 613 if not cache_dir: 614 return None 615 616 if not os.path.isdir(cache_dir): 617 return None 618 619 if self.log_level == 'verbose': 620 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 621 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 622 self.machine, cache_dir, self.show_all, 623 self.test_name, self.suite) 624 if not result: 625 return None 626 627 if (result.retval == 0 or 628 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 629 return result 630 631 return None 632 633 def StoreResult(self, result): 634 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 635 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 636 637 638class MockResultsCache(ResultsCache): 639 640 def Init(self, *args): 641 pass 642 643 def ReadResult(self): 644 return None 645 646 def StoreResult(self, result): 647 pass 648 649 650class MockResult(Result): 651 652 def PopulateFromRun(self, out, err, retval, show_all, test, suite): 653 self.out = out 654 self.err = err 655 self.retval = retval 656