results_cache.py revision 882e707d7e4f63c8e64fa554e77303c4800258c7
12fc823f5794737391e231c1dce6c2b0793213e53mmentovai 22fc823f5794737391e231c1dce6c2b0793213e53mmentovai# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 32fc823f5794737391e231c1dce6c2b0793213e53mmentovai# Use of this source code is governed by a BSD-style license that can be 42fc823f5794737391e231c1dce6c2b0793213e53mmentovai# found in the LICENSE file. 52fc823f5794737391e231c1dce6c2b0793213e53mmentovai"""Module to deal with result cache.""" 62fc823f5794737391e231c1dce6c2b0793213e53mmentovai 72fc823f5794737391e231c1dce6c2b0793213e53mmentovaifrom __future__ import print_function 82fc823f5794737391e231c1dce6c2b0793213e53mmentovai 92fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport glob 102fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport hashlib 112fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport os 122fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport pickle 132fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport re 142fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport tempfile 152fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport json 162fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport sys 172fc823f5794737391e231c1dce6c2b0793213e53mmentovai 182fc823f5794737391e231c1dce6c2b0793213e53mmentovaifrom cros_utils import command_executer 192fc823f5794737391e231c1dce6c2b0793213e53mmentovaifrom cros_utils import misc 202fc823f5794737391e231c1dce6c2b0793213e53mmentovai 212fc823f5794737391e231c1dce6c2b0793213e53mmentovaifrom image_checksummer import ImageChecksummer 222fc823f5794737391e231c1dce6c2b0793213e53mmentovai 232fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport results_report 242fc823f5794737391e231c1dce6c2b0793213e53mmentovaiimport test_flag 252fc823f5794737391e231c1dce6c2b0793213e53mmentovai 262fc823f5794737391e231c1dce6c2b0793213e53mmentovaiSCRATCH_DIR = os.path.expanduser('~/cros_scratch') 272fc823f5794737391e231c1dce6c2b0793213e53mmentovaiRESULTS_FILE = 'results.txt' 282fc823f5794737391e231c1dce6c2b0793213e53mmentovaiMACHINE_FILE = 'machine.txt' 292fc823f5794737391e231c1dce6c2b0793213e53mmentovaiAUTOTEST_TARBALL = 'autotest.tbz2' 302fc823f5794737391e231c1dce6c2b0793213e53mmentovaiPERF_RESULTS_FILE = 'perf-results.txt' 312fc823f5794737391e231c1dce6c2b0793213e53mmentovaiCACHE_KEYS_FILE = 'cache_keys.txt' 322fc823f5794737391e231c1dce6c2b0793213e53mmentovai 332fc823f5794737391e231c1dce6c2b0793213e53mmentovai 342fc823f5794737391e231c1dce6c2b0793213e53mmentovaiclass Result(object): 352fc823f5794737391e231c1dce6c2b0793213e53mmentovai """Class for holding the results of a single test run. 362fc823f5794737391e231c1dce6c2b0793213e53mmentovai 372fc823f5794737391e231c1dce6c2b0793213e53mmentovai This class manages what exactly is stored inside the cache without knowing 382fc823f5794737391e231c1dce6c2b0793213e53mmentovai what the key of the cache is. For runs with perf, it stores perf.data, 392fc823f5794737391e231c1dce6c2b0793213e53mmentovai perf.report, etc. The key generation is handled by the ResultsCache class. 402fc823f5794737391e231c1dce6c2b0793213e53mmentovai """ 412fc823f5794737391e231c1dce6c2b0793213e53mmentovai 422fc823f5794737391e231c1dce6c2b0793213e53mmentovai def __init__(self, logger, label, log_level, machine, cmd_exec=None): 43e5dc60822e5938fea2ae892ccddb906641ba174emmentovai self.chromeos_root = label.chromeos_root 442fc823f5794737391e231c1dce6c2b0793213e53mmentovai self._logger = logger 4508730fc9a639e5b962f9a803ae8f5e91630e9484SiyangXie@gmail.com self.ce = cmd_exec or command_executer.GetCommandExecuter( 4608730fc9a639e5b962f9a803ae8f5e91630e9484SiyangXie@gmail.com self._logger, 4708730fc9a639e5b962f9a803ae8f5e91630e9484SiyangXie@gmail.com log_level=log_level) 482fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.temp_dir = None 492fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.label = label 502fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.results_dir = None 512fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.log_level = log_level 522fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.machine = machine 532fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.perf_data_files = [] 542fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.perf_report_files = [] 552fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.results_file = [] 562fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.chrome_version = '' 572fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.err = None 582fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.chroot_results_dir = '' 5965571f17edb82d122b5f6dc741bd7d4b9e315e1bmmentovai self.test_name = '' 6065571f17edb82d122b5f6dc741bd7d4b9e315e1bmmentovai self.keyvals = None 612fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.board = None 622fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.suite = None 632fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.retval = None 642fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.out = None 652fc823f5794737391e231c1dce6c2b0793213e53mmentovai 662fc823f5794737391e231c1dce6c2b0793213e53mmentovai def CopyFilesTo(self, dest_dir, files_to_copy): 672fc823f5794737391e231c1dce6c2b0793213e53mmentovai file_index = 0 682fc823f5794737391e231c1dce6c2b0793213e53mmentovai for file_to_copy in files_to_copy: 692fc823f5794737391e231c1dce6c2b0793213e53mmentovai if not os.path.isdir(dest_dir): 702fc823f5794737391e231c1dce6c2b0793213e53mmentovai command = 'mkdir -p %s' % dest_dir 7108730fc9a639e5b962f9a803ae8f5e91630e9484SiyangXie@gmail.com self.ce.RunCommand(command) 7241f998fe5a0630506d6d2a1bae78b1be179fe850SiyangXie@gmail.com dest_file = os.path.join(dest_dir, 7308730fc9a639e5b962f9a803ae8f5e91630e9484SiyangXie@gmail.com ('%s.%s' % (os.path.basename(file_to_copy), 742fc823f5794737391e231c1dce6c2b0793213e53mmentovai file_index))) 752fc823f5794737391e231c1dce6c2b0793213e53mmentovai ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 762fc823f5794737391e231c1dce6c2b0793213e53mmentovai if ret: 772fc823f5794737391e231c1dce6c2b0793213e53mmentovai raise IOError('Could not copy results file: %s' % file_to_copy) 782fc823f5794737391e231c1dce6c2b0793213e53mmentovai 792fc823f5794737391e231c1dce6c2b0793213e53mmentovai def CopyResultsTo(self, dest_dir): 802fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.CopyFilesTo(dest_dir, self.perf_data_files) 812fc823f5794737391e231c1dce6c2b0793213e53mmentovai self.CopyFilesTo(dest_dir, self.perf_report_files) 822fc823f5794737391e231c1dce6c2b0793213e53mmentovai if len(self.perf_data_files) or len(self.perf_report_files): 83e5dc60822e5938fea2ae892ccddb906641ba174emmentovai self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 842fc823f5794737391e231c1dce6c2b0793213e53mmentovai 852fc823f5794737391e231c1dce6c2b0793213e53mmentovai def GetNewKeyvals(self, keyvals_dict): 86 # Initialize 'units' dictionary. 87 units_dict = {} 88 for k in keyvals_dict: 89 units_dict[k] = '' 90 results_files = self.GetDataMeasurementsFiles() 91 for f in results_files: 92 # Make sure we can find the results file 93 if os.path.exists(f): 94 data_filename = f 95 else: 96 # Otherwise get the base filename and create the correct 97 # path for it. 98 _, f_base = misc.GetRoot(f) 99 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 100 self.temp_dir, f_base) 101 if data_filename.find('.json') > 0: 102 raw_dict = dict() 103 if os.path.exists(data_filename): 104 with open(data_filename, 'r') as data_file: 105 raw_dict = json.load(data_file) 106 107 for k1 in raw_dict: 108 field_dict = raw_dict[k1] 109 for k2 in field_dict: 110 result_dict = field_dict[k2] 111 key = k1 + '__' + k2 112 if 'value' in result_dict: 113 keyvals_dict[key] = result_dict['value'] 114 elif 'values' in result_dict: 115 keyvals_dict[key] = result_dict['values'] 116 units_dict[key] = result_dict['units'] 117 else: 118 if os.path.exists(data_filename): 119 with open(data_filename, 'r') as data_file: 120 lines = data_file.readlines() 121 for line in lines: 122 tmp_dict = json.loads(line) 123 graph_name = tmp_dict['graph'] 124 graph_str = (graph_name + '__') if graph_name else '' 125 key = graph_str + tmp_dict['description'] 126 keyvals_dict[key] = tmp_dict['value'] 127 units_dict[key] = tmp_dict['units'] 128 129 return keyvals_dict, units_dict 130 131 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 132 """keyvals_dict is the dict of key-value used to generate Crosperf reports. 133 134 units_dict is a dictionary of the units for the return values in 135 keyvals_dict. We need to associate the units with the return values, 136 for Telemetry tests, so that we can include the units in the reports. 137 This function takes each value in keyvals_dict, finds the corresponding 138 unit in the units_dict, and replaces the old value with a list of the 139 old value and the units. This later gets properly parsed in the 140 ResultOrganizer class, for generating the reports. 141 """ 142 143 results_dict = {} 144 for k in keyvals_dict: 145 # We don't want these lines in our reports; they add no useful data. 146 if k == '' or k == 'telemetry_Crosperf': 147 continue 148 val = keyvals_dict[k] 149 units = units_dict[k] 150 new_val = [val, units] 151 results_dict[k] = new_val 152 return results_dict 153 154 def GetKeyvals(self): 155 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 156 if not self.temp_dir: 157 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 158 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 159 self.ce.RunCommand(command, print_to_console=False) 160 161 command = ('python generate_test_report --no-color --csv %s' % 162 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 163 _, out, _ = self.ce.ChrootRunCommandWOutput(self.chromeos_root, 164 command, 165 print_to_console=False) 166 keyvals_dict = {} 167 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 168 self.temp_dir) 169 for line in out.splitlines(): 170 tokens = re.split('=|,', line) 171 key = tokens[-2] 172 if key.startswith(tmp_dir_in_chroot): 173 key = key[len(tmp_dir_in_chroot) + 1:] 174 value = tokens[-1] 175 keyvals_dict[key] = value 176 177 # Check to see if there is a perf_measurements file and get the 178 # data from it if so. 179 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 180 if self.suite == 'telemetry_Crosperf': 181 # For telemtry_Crosperf results, append the units to the return 182 # results, for use in generating the reports. 183 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 184 return keyvals_dict 185 186 def GetResultsDir(self): 187 mo = re.search(r'Results placed in (\S+)', self.out) 188 if mo: 189 result = mo.group(1) 190 return result 191 raise RuntimeError('Could not find results directory.') 192 193 def FindFilesInResultsDir(self, find_args): 194 if not self.results_dir: 195 return None 196 197 command = 'find %s %s' % (self.results_dir, find_args) 198 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 199 if ret: 200 raise RuntimeError('Could not run find command!') 201 return out 202 203 def GetResultsFile(self): 204 return self.FindFilesInResultsDir('-name results-chart.json').splitlines() 205 206 def GetPerfDataFiles(self): 207 return self.FindFilesInResultsDir('-name perf.data').splitlines() 208 209 def GetPerfReportFiles(self): 210 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 211 212 def GetDataMeasurementsFiles(self): 213 result = self.FindFilesInResultsDir('-name perf_measurements').splitlines() 214 if not result: 215 result = \ 216 self.FindFilesInResultsDir('-name results-chart.json').splitlines() 217 return result 218 219 def GeneratePerfReportFiles(self): 220 perf_report_files = [] 221 for perf_data_file in self.perf_data_files: 222 # Generate a perf.report and store it side-by-side with the perf.data 223 # file. 224 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 225 perf_data_file) 226 perf_report_file = '%s.report' % perf_data_file 227 if os.path.exists(perf_report_file): 228 raise RuntimeError('Perf report file already exists: %s' % 229 perf_report_file) 230 chroot_perf_report_file = misc.GetInsideChrootPath(self.chromeos_root, 231 perf_report_file) 232 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 233 234 perf_file = '/usr/sbin/perf' 235 if os.path.exists(perf_path): 236 perf_file = '/usr/bin/perf' 237 238 # The following is a hack, to use the perf.static binary that 239 # was given to us by Stephane Eranian, until he can figure out 240 # why "normal" perf cannot properly symbolize ChromeOS perf.data files. 241 # Get the directory containing the 'crosperf' script. 242 dirname, _ = misc.GetRoot(sys.argv[0]) 243 perf_path = os.path.join(dirname, '..', 'perf.static') 244 if os.path.exists(perf_path): 245 # copy the executable into the chroot so that it can be found. 246 src_path = perf_path 247 dst_path = os.path.join(self.chromeos_root, 'chroot', 248 'tmp/perf.static') 249 command = 'cp %s %s' % (src_path, dst_path) 250 self.ce.RunCommand(command) 251 perf_file = '/tmp/perf.static' 252 253 command = ('%s report ' 254 '-n ' 255 '--symfs /build/%s ' 256 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 257 '--kallsyms /build/%s/boot/System.map-* ' 258 '-i %s --stdio ' 259 '> %s' % (perf_file, self.board, self.board, self.board, 260 chroot_perf_data_file, chroot_perf_report_file)) 261 self.ce.ChrootRunCommand(self.chromeos_root, command) 262 263 # Add a keyval to the dictionary for the events captured. 264 perf_report_files.append(misc.GetOutsideChrootPath( 265 self.chromeos_root, chroot_perf_report_file)) 266 return perf_report_files 267 268 def GatherPerfResults(self): 269 report_id = 0 270 for perf_report_file in self.perf_report_files: 271 with open(perf_report_file, 'r') as f: 272 report_contents = f.read() 273 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 274 num_events = group[0] 275 event_name = group[1] 276 key = 'perf_%s_%s' % (report_id, event_name) 277 value = str(misc.UnitToNumber(num_events)) 278 self.keyvals[key] = value 279 280 def PopulateFromRun(self, out, err, retval, test, suite): 281 self.board = self.label.board 282 self.out = out 283 self.err = err 284 self.retval = retval 285 self.test_name = test 286 self.suite = suite 287 self.chroot_results_dir = self.GetResultsDir() 288 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 289 self.chroot_results_dir) 290 self.results_file = self.GetResultsFile() 291 self.perf_data_files = self.GetPerfDataFiles() 292 # Include all perf.report data in table. 293 self.perf_report_files = self.GeneratePerfReportFiles() 294 # TODO(asharif): Do something similar with perf stat. 295 296 # Grab keyvals from the directory. 297 self.ProcessResults() 298 299 def ProcessJsonResults(self): 300 # Open and parse the json results file generated by telemetry/test_that. 301 if not self.results_file: 302 raise IOError('No results file found.') 303 filename = self.results_file[0] 304 if not filename.endswith('.json'): 305 raise IOError('Attempt to call json on non-json file: %s' % filename) 306 307 if not os.path.exists(filename): 308 return {} 309 310 keyvals = {} 311 with open(filename, 'r') as f: 312 raw_dict = json.load(f) 313 if 'charts' in raw_dict: 314 raw_dict = raw_dict['charts'] 315 for k, field_dict in raw_dict.iteritems(): 316 for item in field_dict: 317 keyname = k + "__" + item 318 value_dict = field_dict[item] 319 if 'value' in value_dict: 320 result = value_dict['value'] 321 elif 'values' in value_dict: 322 if not value_dict['values']: 323 continue 324 result = value_dict['values'] 325 units = value_dict['units'] 326 new_value = [result, units] 327 keyvals[keyname] = new_value 328 return keyvals 329 330 def ProcessResults(self): 331 # Note that this function doesn't know anything about whether there is a 332 # cache hit or miss. It should process results agnostic of the cache hit 333 # state. 334 if self.results_file and self.results_file[0].find('results_chart.json'): 335 self.keyvals = self.ProcessJsonResults() 336 else: 337 print('\n ** WARNING **: Had to use deprecated output-method to ' 338 'collect results.\n') 339 self.keyvals = self.GetKeyvals() 340 self.keyvals['retval'] = self.retval 341 # Generate report from all perf.data files. 342 # Now parse all perf report files and include them in keyvals. 343 self.GatherPerfResults() 344 345 def GetChromeVersionFromCache(self, cache_dir): 346 # Read chrome_version from keys file, if present. 347 chrome_version = '' 348 keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) 349 if os.path.exists(keys_file): 350 with open(keys_file, 'r') as f: 351 lines = f.readlines() 352 for l in lines: 353 if l.startswith('Google Chrome '): 354 chrome_version = l 355 if chrome_version.endswith('\n'): 356 chrome_version = chrome_version[:-1] 357 break 358 return chrome_version 359 360 def PopulateFromCacheDir(self, cache_dir, test, suite): 361 self.test_name = test 362 self.suite = suite 363 # Read in everything from the cache directory. 364 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 365 self.out = pickle.load(f) 366 self.err = pickle.load(f) 367 self.retval = pickle.load(f) 368 369 # Untar the tarball to a temporary directory 370 self.temp_dir = tempfile.mkdtemp( 371 dir=os.path.join(self.chromeos_root, 'chroot', 'tmp')) 372 373 command = ('cd %s && tar xf %s' % 374 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 375 ret = self.ce.RunCommand(command, print_to_console=False) 376 if ret: 377 raise RuntimeError('Could not untar cached tarball') 378 self.results_dir = self.temp_dir 379 self.perf_data_files = self.GetPerfDataFiles() 380 self.perf_report_files = self.GetPerfReportFiles() 381 self.chrome_version = self.GetChromeVersionFromCache(cache_dir) 382 self.ProcessResults() 383 384 def CleanUp(self, rm_chroot_tmp): 385 if rm_chroot_tmp and self.results_dir: 386 dirname, basename = misc.GetRoot(self.results_dir) 387 if basename.find('test_that_results_') != -1: 388 command = 'rm -rf %s' % self.results_dir 389 else: 390 command = 'rm -rf %s' % dirname 391 self.ce.RunCommand(command) 392 if self.temp_dir: 393 command = 'rm -rf %s' % self.temp_dir 394 self.ce.RunCommand(command) 395 396 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 397 # Create the dir if it doesn't exist. 398 temp_dir = tempfile.mkdtemp() 399 400 # Store to the temp directory. 401 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 402 pickle.dump(self.out, f) 403 pickle.dump(self.err, f) 404 pickle.dump(self.retval, f) 405 406 if not test_flag.GetTestMode(): 407 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 408 f.write('%s\n' % self.label.name) 409 f.write('%s\n' % self.label.chrome_version) 410 f.write('%s\n' % self.machine.checksum_string) 411 for k in key_list: 412 f.write(k) 413 f.write('\n') 414 415 if self.results_dir: 416 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 417 command = ('cd %s && ' 418 'tar ' 419 '--exclude=var/spool ' 420 '--exclude=var/log ' 421 '-cjf %s .' % (self.results_dir, tarball)) 422 ret = self.ce.RunCommand(command) 423 if ret: 424 raise RuntimeError("Couldn't store autotest output directory.") 425 # Store machine info. 426 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 427 # this function. 428 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 429 f.write(machine_manager.machine_checksum_string[self.label.name]) 430 431 if os.path.exists(cache_dir): 432 command = 'rm -rf {0}'.format(cache_dir) 433 self.ce.RunCommand(command) 434 435 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 436 command += 'chmod g+x {0} && '.format(temp_dir) 437 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 438 ret = self.ce.RunCommand(command) 439 if ret: 440 command = 'rm -rf {0}'.format(temp_dir) 441 self.ce.RunCommand(command) 442 raise RuntimeError('Could not move dir %s to dir %s' % 443 (temp_dir, cache_dir)) 444 445 @classmethod 446 def CreateFromRun(cls, 447 logger, 448 log_level, 449 label, 450 machine, 451 out, 452 err, 453 retval, 454 test, 455 suite='telemetry_Crosperf'): 456 if suite == 'telemetry': 457 result = TelemetryResult(logger, label, log_level, machine) 458 else: 459 result = cls(logger, label, log_level, machine) 460 result.PopulateFromRun(out, err, retval, test, suite) 461 return result 462 463 @classmethod 464 def CreateFromCacheHit(cls, 465 logger, 466 log_level, 467 label, 468 machine, 469 cache_dir, 470 test, 471 suite='telemetry_Crosperf'): 472 if suite == 'telemetry': 473 result = TelemetryResult(logger, label, log_level, machine) 474 else: 475 result = cls(logger, label, log_level, machine) 476 try: 477 result.PopulateFromCacheDir(cache_dir, test, suite) 478 479 except RuntimeError as e: 480 logger.LogError('Exception while using cache: %s' % e) 481 return None 482 return result 483 484 485class TelemetryResult(Result): 486 """Class to hold the results of a single Telemetry run.""" 487 488 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 489 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 490 cmd_exec) 491 492 def PopulateFromRun(self, out, err, retval, test, suite): 493 self.out = out 494 self.err = err 495 self.retval = retval 496 497 self.ProcessResults() 498 499 def ProcessResults(self): 500 # The output is: 501 # url,average_commit_time (ms),... 502 # www.google.com,33.4,21.2,... 503 # We need to convert to this format: 504 # {"www.google.com:average_commit_time (ms)": "33.4", 505 # "www.google.com:...": "21.2"} 506 # Added note: Occasionally the output comes back 507 # with "JSON.stringify(window.automation.GetResults())" on 508 # the first line, and then the rest of the output as 509 # described above. 510 511 lines = self.out.splitlines() 512 self.keyvals = {} 513 514 if lines: 515 if lines[0].startswith('JSON.stringify'): 516 lines = lines[1:] 517 518 if not lines: 519 return 520 labels = lines[0].split(',') 521 for line in lines[1:]: 522 fields = line.split(',') 523 if len(fields) != len(labels): 524 continue 525 for i in xrange(1, len(labels)): 526 key = '%s %s' % (fields[0], labels[i]) 527 value = fields[i] 528 self.keyvals[key] = value 529 self.keyvals['retval'] = self.retval 530 531 def PopulateFromCacheDir(self, cache_dir, test, suite): 532 self.test_name = test 533 self.suite = suite 534 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 535 self.out = pickle.load(f) 536 self.err = pickle.load(f) 537 self.retval = pickle.load(f) 538 539 self.chrome_version = \ 540 super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) 541 self.ProcessResults() 542 543 544class CacheConditions(object): 545 """Various Cache condition values, for export.""" 546 547 # Cache hit only if the result file exists. 548 CACHE_FILE_EXISTS = 0 549 550 # Cache hit if the checksum of cpuinfo and totalmem of 551 # the cached result and the new run match. 552 MACHINES_MATCH = 1 553 554 # Cache hit if the image checksum of the cached result and the new run match. 555 CHECKSUMS_MATCH = 2 556 557 # Cache hit only if the cached result was successful 558 RUN_SUCCEEDED = 3 559 560 # Never a cache hit. 561 FALSE = 4 562 563 # Cache hit if the image path matches the cached image path. 564 IMAGE_PATH_MATCH = 5 565 566 # Cache hit if the uuid of hard disk mataches the cached one 567 568 SAME_MACHINE_MATCH = 6 569 570 571class ResultsCache(object): 572 """Class to handle the cache for storing/retrieving test run results. 573 574 This class manages the key of the cached runs without worrying about what 575 is exactly stored (value). The value generation is handled by the Results 576 class. 577 """ 578 CACHE_VERSION = 6 579 580 def __init__(self): 581 # Proper initialization happens in the Init function below. 582 self.chromeos_image = None 583 self.chromeos_root = None 584 self.test_name = None 585 self.iteration = None 586 self.test_args = None 587 self.profiler_args = None 588 self.board = None 589 self.cache_conditions = None 590 self.machine_manager = None 591 self.machine = None 592 self._logger = None 593 self.ce = None 594 self.label = None 595 self.share_cache = None 596 self.suite = None 597 self.log_level = None 598 self.show_all = None 599 self.run_local = None 600 601 602 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 603 profiler_args, machine_manager, machine, board, cache_conditions, 604 logger_to_use, log_level, label, share_cache, suite, 605 show_all_results, run_local): 606 self.chromeos_image = chromeos_image 607 self.chromeos_root = chromeos_root 608 self.test_name = test_name 609 self.iteration = iteration 610 self.test_args = test_args 611 self.profiler_args = profiler_args 612 self.board = board 613 self.cache_conditions = cache_conditions 614 self.machine_manager = machine_manager 615 self.machine = machine 616 self._logger = logger_to_use 617 self.ce = command_executer.GetCommandExecuter(self._logger, 618 log_level=log_level) 619 self.label = label 620 self.share_cache = share_cache 621 self.suite = suite 622 self.log_level = log_level 623 self.show_all = show_all_results 624 self.run_local = run_local 625 626 def GetCacheDirForRead(self): 627 matching_dirs = [] 628 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 629 matching_dirs += glob.glob(glob_path) 630 631 if matching_dirs: 632 # Cache file found. 633 return matching_dirs[0] 634 return None 635 636 def GetCacheDirForWrite(self, get_keylist=False): 637 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 638 if get_keylist: 639 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 640 self.run_local) 641 version, image = results_report.ParseChromeosImage( 642 self.label.chromeos_image) 643 keylist = [version, image, self.label.board, self.machine.name, 644 self.test_name, str(self.iteration), args_str] 645 return cache_path, keylist 646 return cache_path 647 648 def FormCacheDir(self, list_of_strings): 649 cache_key = ' '.join(list_of_strings) 650 cache_dir = misc.GetFilenameFromString(cache_key) 651 if self.label.cache_dir: 652 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 653 cache_path = [os.path.join(cache_home, cache_dir)] 654 else: 655 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 656 657 if len(self.share_cache): 658 for path in [x.strip() for x in self.share_cache.split(',')]: 659 if os.path.exists(path): 660 cache_path.append(os.path.join(path, cache_dir)) 661 else: 662 self._logger.LogFatal('Unable to find shared cache: %s' % path) 663 664 return cache_path 665 666 def GetCacheKeyList(self, read): 667 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 668 machine_checksum = '*' 669 else: 670 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 671 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 672 checksum = '*' 673 elif self.label.image_type == 'trybot': 674 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 675 elif self.label.image_type == 'official': 676 checksum = '*' 677 else: 678 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 679 680 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 681 image_path_checksum = '*' 682 else: 683 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 684 685 machine_id_checksum = '' 686 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 687 machine_id_checksum = '*' 688 else: 689 if self.machine and self.machine.name in self.label.remote: 690 machine_id_checksum = self.machine.machine_id_checksum 691 else: 692 for machine in self.machine_manager.GetMachines(self.label): 693 if machine.name == self.label.remote[0]: 694 machine_id_checksum = machine.machine_id_checksum 695 break 696 697 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 698 self.run_local) 699 test_args_checksum = hashlib.md5(temp_test_args).hexdigest() 700 return (image_path_checksum, self.test_name, str(self.iteration), 701 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 702 str(self.CACHE_VERSION)) 703 704 def ReadResult(self): 705 if CacheConditions.FALSE in self.cache_conditions: 706 cache_dir = self.GetCacheDirForWrite() 707 command = 'rm -rf %s' % (cache_dir, ) 708 self.ce.RunCommand(command) 709 return None 710 cache_dir = self.GetCacheDirForRead() 711 712 if not cache_dir: 713 return None 714 715 if not os.path.isdir(cache_dir): 716 return None 717 718 if self.log_level == 'verbose': 719 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 720 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 721 self.machine, cache_dir, 722 self.test_name, self.suite) 723 if not result: 724 return None 725 726 if (result.retval == 0 or 727 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 728 return result 729 730 return None 731 732 def StoreResult(self, result): 733 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 734 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 735 736 737class MockResultsCache(ResultsCache): 738 """Class for mock testing, corresponding to ResultsCache class.""" 739 740 def Init(self, *args): 741 pass 742 743 def ReadResult(self): 744 return None 745 746 def StoreResult(self, result): 747 pass 748 749 750class MockResult(Result): 751 """Class for mock testing, corresponding to Result class.""" 752 753 def PopulateFromRun(self, out, err, retval, test, suite): 754 self.out = out 755 self.err = err 756 self.retval = retval 757