104dc5dc8547dbfbe524cf35ac39537346ad749bbYunlian Jiang# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 204dc5dc8547dbfbe524cf35ac39537346ad749bbYunlian Jiang# Use of this source code is governed by a BSD-style license that can be 304dc5dc8547dbfbe524cf35ac39537346ad749bbYunlian Jiang# found in the LICENSE file. 44467f004e7f0854963bec90daff1879fbd9d2fecAhmad Sharif"""Parse data from benchmark_runs for tabulator.""" 589d263c7cf9773129cbe8e8858ad21ea539a2ba0Luis Lozano 689d263c7cf9773129cbe8e8858ad21ea539a2ba0Luis Lozanofrom __future__ import print_function 789d263c7cf9773129cbe8e8858ad21ea539a2ba0Luis Lozano 8afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVimport errno 944a44befd1f500b9a227ebfd849702efce83ef6acmticeimport json 1044a44befd1f500b9a227ebfd849702efce83ef6acmticeimport os 114467f004e7f0854963bec90daff1879fbd9d2fecAhmad Sharifimport re 1244a44befd1f500b9a227ebfd849702efce83ef6acmticeimport sys 13f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif 141974c70a4e0b161a18a843229fc1af8344e11600Yunlian Jiangfrom cros_utils import misc 1544a44befd1f500b9a227ebfd849702efce83ef6acmtice 16afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json' 17afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}') 18f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano 19f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif 20afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _AdjustIteration(benchmarks, max_dup, bench): 21afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Adjust the interation numbers if they have keys like ABCD{i}.""" 22afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for benchmark in benchmarks: 23afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if benchmark.name != bench or benchmark.iteration_adjusted: 24afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV continue 25afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV benchmark.iteration_adjusted = True 26afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV benchmark.iterations *= (max_dup + 1) 27afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 28afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 29afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _GetMaxDup(data): 30afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Find the maximum i inside ABCD{i}. 31afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 32afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV data should be a [[[Key]]], where Key is a string that may look like 33afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV ABCD{i}. 34afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """ 35afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV max_dup = 0 36afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for label in data: 37afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for run in label: 38afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for key in run: 39afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV match = _DUP_KEY_REGEX.match(key) 40afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if match: 41afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV max_dup = max(max_dup, int(match.group(2))) 42afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return max_dup 43afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 44afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 45afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _Repeat(func, times): 46afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Returns the result of running func() n times.""" 47afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return [func() for _ in xrange(times)] 48afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 49afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 50afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _GetNonDupLabel(max_dup, runs): 51afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Create new list for the runs of the same label. 52afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 53afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV Specifically, this will split out keys like foo{0}, foo{1} from one run into 54afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV their own runs. For example, given a run like: 55afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5} 56afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 57afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV You'll get: 58afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}] 59afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 60afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV Hands back the lists of transformed runs, all concatenated together. 61afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """ 62afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV new_runs = [] 63afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for run in runs: 64afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV new_run = {} 65afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV added_runs = _Repeat(dict, max_dup) 66afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for key, value in run.iteritems(): 67afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV match = _DUP_KEY_REGEX.match(key) 68afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if not match: 69afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV new_run[key] = value 70afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV else: 71afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV new_key, index_str = match.groups() 72afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV added_runs[int(index_str)-1][new_key] = str(value) 73afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV new_runs.append(new_run) 74afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV new_runs += added_runs 75afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return new_runs 76afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 77afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 78afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _DuplicatePass(result, benchmarks): 79afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Properly expands keys like `foo{1}` in `result`.""" 80afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for bench, data in result.iteritems(): 81afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV max_dup = _GetMaxDup(data) 82afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # If there's nothing to expand, there's nothing to do. 83afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if not max_dup: 84afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV continue 85afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for i, runs in enumerate(data): 86afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV data[i] = _GetNonDupLabel(max_dup, runs) 87afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV _AdjustIteration(benchmarks, max_dup, bench) 88afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 89afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 90afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _ReadSummaryFile(filename): 91afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Reads the summary file at filename.""" 92afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV dirname, _ = misc.GetRoot(filename) 93afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE) 94afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV try: 95afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # Slurp the summary file into a dictionary. The keys in the dictionary are 96afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # the benchmark names. The value for a key is a list containing the names 97afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # of all the result fields that should be returned in a 'default' report. 98afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV with open(fullname) as in_file: 99afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return json.load(in_file) 100afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV except IOError as e: 101afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # ENOENT means "no such file or directory" 102afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if e.errno == errno.ENOENT: 103afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return {} 104afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV raise 105afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 106afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 107afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef _MakeOrganizeResultOutline(benchmark_runs, labels): 108afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """Creates the "outline" of the OrganizeResults result for a set of runs. 109afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 110afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV Report generation returns lists of different sizes, depending on the input 111afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV data. Depending on the order in which we iterate through said input data, we 112afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV may populate the Nth index of a list, then the N-1st, then the N+Mth, ... 113afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 114afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't 115afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV have to worry about resizing while computing results. 116afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV """ 117afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # Count how many iterations exist for each benchmark run. 118afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # We can't simply count up, since we may be given an incomplete set of 119afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3]) 120afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV iteration_count = {} 121afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for run in benchmark_runs: 122afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV name = run.benchmark.name 123afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV old_iterations = iteration_count.get(name, -1) 124afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # N.B. run.iteration starts at 1, not 0. 125afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV iteration_count[name] = max(old_iterations, run.iteration) 126afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 127afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # Result structure: {benchmark_name: [[{key: val}]]} 128afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV result = {} 129afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for run in benchmark_runs: 130afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV name = run.benchmark.name 131afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV num_iterations = iteration_count[name] 132afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # default param makes cros lint be quiet about defining num_iterations in a 133afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # loop. 134afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV make_dicts = lambda n=num_iterations: _Repeat(dict, n) 135afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV result[name] = _Repeat(make_dicts, len(labels)) 136afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return result 137afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 138afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IVdef OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False): 139f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif """Create a dict from benchmark_runs. 140f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif 141f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif The structure of the output dict is as follows: 142f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif {"benchmark_1":[ 143f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}] 144f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif #one label 145f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif [] 146f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif #the other label 147f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif ] 148f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif "benchmark_2": 149f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif [ 150f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif ]}. 151f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif """ 152afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV result = _MakeOrganizeResultOutline(benchmark_runs, labels) 153afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV label_names = [label.name for label in labels] 154afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV label_indices = {name: i for i, name in enumerate(label_names)} 155afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV summary_file = _ReadSummaryFile(sys.argv[0]) 156afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if benchmarks is None: 157afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV benchmarks = [] 158f395c26437cbdabc2960447fba89b226f4409e82Ahmad Sharif 159afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for benchmark_run in benchmark_runs: 160afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if not benchmark_run.result: 161afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV continue 162afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV benchmark = benchmark_run.benchmark 163afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV label_index = label_indices[benchmark_run.label.name] 164afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV cur_label_list = result[benchmark.name][label_index] 165afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV cur_dict = cur_label_list[benchmark_run.iteration - 1] 166afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV 167afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV show_all_results = json_report or benchmark.show_all_results 168afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if not show_all_results: 169afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV summary_list = summary_file.get(benchmark.test_name) 170afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if summary_list: 171afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV summary_list.append('retval') 1729b852cfd9a602c5f80c8e621c696b796ce5177fdCaroline Tice else: 173afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # Did not find test_name in json file; show everything. 174afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV show_all_results = True 175afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV for test_key in benchmark_run.result.keyvals: 176afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if show_all_results or test_key in summary_list: 17789d263c7cf9773129cbe8e8858ad21ea539a2ba0Luis Lozano cur_dict[test_key] = benchmark_run.result.keyvals[test_key] 178afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # Occasionally Telemetry tests will not fail but they will not return a 179afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # result, either. Look for those cases, and force them to be a fail. 180afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # (This can happen if, for example, the test has been disabled.) 181afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if len(cur_dict) == 1 and cur_dict['retval'] == 0: 182afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV cur_dict['retval'] = 1 183afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV # TODO: This output should be sent via logger. 184afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV print("WARNING: Test '%s' appears to have succeeded but returned" 185afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV ' no results.' % benchmark.name, 186afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV file=sys.stderr) 187afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV if json_report and benchmark_run.machine: 188afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV cur_dict['machine'] = benchmark_run.machine.name 189afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV cur_dict['machine_checksum'] = benchmark_run.machine.checksum 190afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV cur_dict['machine_string'] = benchmark_run.machine.checksum_string 191afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV _DuplicatePass(result, benchmarks) 192afb8cc77e82c35faedfe541d097fc01fd1d7ca3dGeorge Burgess IV return result 193