report.py revision 8720abc26202a8f25dce60d4bdefed5dfeb76b4f
10f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi#!/usr/bin/python 20f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 30f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport argparse 40f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport fnmatch as fnm 50f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport json 60f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport math 70f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport numpy as np 80f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport os 90f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport re 100f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport sys 110f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 120f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasifrom collections import defaultdict 130f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasifrom colors import TestColors 14909185f2974e895696b155de92b168b84ea05266Patrick Bellasifrom results import Results 150f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 160f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi# Configure logging 170f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasiimport logging 180f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasireload(logging) 190f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasilogging.basicConfig( 200f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi format='%(asctime)-9s %(levelname)-8s: %(message)s', 21d4aed708779077ec4055e064602df8bef56c7418Patrick Bellasi level=logging.DEBUG, 22d4aed708779077ec4055e064602df8bef56c7418Patrick Bellasi # level=logging.INFO, 230f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi datefmt='%I:%M:%S') 240f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 250f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi# By default compare all the possible combinations 26e3bebfc47781208a03169507b5bae5f22621579bPatrick BellasiDEFAULT_COMPARE = [(r'base_', r'test_')] 270f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 2897f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasiclass Report(object): 290f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 300f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 31a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi def __init__(self, results_dir, compare=None, formats=['relative']): 3297f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi self.results_json = results_dir + '/results.json' 3397f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi self.results = {} 3497f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi 3597f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi self.compare = [] 3697f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi 3797f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi # Parse results (if required) 3897f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi if not os.path.isfile(self.results_json): 3997f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi Results(results_dir) 4097f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi 4197f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi # Load results from file (if already parsed) 4297f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi logging.info('%14s - Load results from [%s]...', 4397f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi 'Results', self.results_json) 4497f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi with open(self.results_json) as infile: 4597f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi self.results = json.load(infile) 460f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 470f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # Setup configuration comparisons 480f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi if compare is None: 490f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi compare = DEFAULT_COMPARE 500f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.warning('%14s - Comparing all the possible combination', 510f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 'Results') 520f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi for (base_rexp, test_rexp) in compare: 53e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi logging.info('Configured regexps for comparisions (bases , tests): (%s, %s)', 54e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi base_rexp, test_rexp) 550f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi base_rexp = re.compile(base_rexp, re.DOTALL) 560f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi test_rexp = re.compile(test_rexp, re.DOTALL) 570f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi self.compare.append((base_rexp, test_rexp)) 580f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 59cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi # Report all supported workload classes 6021756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi self.__rtapp_report(formats) 6121756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi self.__default_report(formats) 62cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi 63cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi ############################### REPORT RTAPP ############################### 64cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi 6521756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi def __rtapp_report(self, formats): 66cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi 67cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi if 'rtapp' not in self.results.keys(): 68cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi logging.debug('%14s - No RTApp workloads to report', 'ReportRTApp') 69cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi return 70cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi 71cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi logging.debug('%14s - Reporting RTApp workloads', 'ReportRTApp') 72cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi 73cc2343c456bfabf56a06f2c93b07df2802913b7bPatrick Bellasi # Setup lables depending on requested report 7421756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi if 'absolute' in formats: 750f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi nrg_lable = 'Energy Indexes (Absolute)' 760f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi prf_lable = 'Performance Indexes (Absolute)' 770f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.info('') 780f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.info('%14s - Absolute comparisions:', 'Report') 790f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi print '' 800f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi else: 810f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi nrg_lable = 'Energy Indexes (Relative)' 820f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi prf_lable = 'Performance Indexes (Relative)' 830f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.info('') 840f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.info('%14s - Relative comparisions:', 'Report') 850f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi print '' 860f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 8797f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi # Dump headers 88faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi print '{:9s} {:20s} |'\ 890f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi ' {:33s} | {:54s} |'\ 900f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi .format('Test Id', 'Comparision', 910f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi nrg_lable, prf_lable) 92faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi print '{:9s} {:20s} |'\ 930f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi ' {:>10s} {:>10s} {:>10s} |'\ 940f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ 950f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi .format('', '', 960f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 'LITTLE', 'big', 'Total', 970f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 'PerfIndex', 'NegSlacks', 'EDP1', 'EDP2', 'EDP3') 980f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 990f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # For each test 10097f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi _results = self.results['rtapp'] 10197f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi for tid in sorted(_results.keys()): 1020f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi new_test = True 1030f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # For each configuration... 104e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi for base_idx in sorted(_results[tid].keys()): 1050f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # Which matches at least on base regexp 1060f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi for (base_rexp, test_rexp) in self.compare: 1070f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi if not base_rexp.match(base_idx): 1080f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi continue 1090f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # Look for a configuration which matches the test regexp 110e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi for test_idx in sorted(_results[tid].keys()): 1110f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi if test_idx == base_idx: 1120f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi continue 1130f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi if new_test: 114faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi print '{:-<33s}+{:-<35s}+{:-<56s}+'\ 11597f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi .format('','', '') 116a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi self.__rtapp_reference(tid, base_idx) 1170f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi new_test = False 118e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi if test_rexp.match(test_idx) == None: 1190f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi continue 12021756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi self.__rtapp_compare(tid, base_idx, test_idx, formats) 1210f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 1220f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi print '' 1230f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 124a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi def __rtapp_reference(self, tid, base_idx): 125a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi _results = self.results['rtapp'] 126a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 127a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi logging.debug('Test %s: compare against [%s] base', 128a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi tid, base_idx) 129a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_line = '{0:8s}: {1:22s} | '.format(tid, base_idx) 130a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 131a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi # Dump all energy metrics 132a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi for cpus in ['LITTLE', 'big', 'Total']: 133a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_base = _results[tid][base_idx]['energy'][cpus]['avg'] 134a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi # Dump absolute values 135a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_line += ' {0:10.3f}'.format(res_base) 136a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_line += ' |' 137a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 138a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi # If available, dump also performance results 139a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi if 'performance' not in _results[tid][base_idx].keys(): 140a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi print res_line 141a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi return 142a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 143a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: 144a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_base = _results[tid][base_idx]['performance'][pidx]['avg'] 145a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 146a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi logging.debug('idx: %s, base: %s', pidx, res_base) 147a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 1488720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi if pidx in ['perf_avg']: 1498720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi res_line += ' {0:s}'.format(TestColors.rate(res_base)) 1508720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi continue 1518720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi if pidx in ['slack_pct']: 1528720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi res_line += ' {0:s}'.format( 1538720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi TestColors.rate(res_base, positive_is_good = False)) 1548720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi continue 155a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi if 'edp' in pidx: 156a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_line += ' {0:10.2e}'.format(res_base) 1578720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi continue 158a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_line += ' |' 159a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi print res_line 160a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi 16121756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi def __rtapp_compare(self, tid, base_idx, test_idx, formats): 16297f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi _results = self.results['rtapp'] 16397f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi 1640f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.debug('Test %s: compare %s with %s', 1650f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi tid, base_idx, test_idx) 166a0c6af17b243d7e84a68449eb497fd91d59f1e6fPatrick Bellasi res_line = '{0:8s}: {1:20s} | '.format(tid, test_idx) 1670f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 1680f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # Dump all energy metrics 1690f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi for cpus in ['LITTLE', 'big', 'Total']: 17097f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi res_base = _results[tid][base_idx]['energy'][cpus]['avg'] 17197f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi res_test = _results[tid][test_idx]['energy'][cpus]['avg'] 1720f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi speedup_cnt = res_test - res_base 17321756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi if 'absolute' in formats: 1740f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi res_line += ' {0:10.2f}'.format(speedup_cnt) 1750f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi else: 1760f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi speedup_pct = 100.0 * speedup_cnt / res_base 1770f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi res_line += ' {0:s}'\ 1780f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi .format(TestColors.rate( 1790f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi speedup_pct, 1800f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi positive_is_good = False)) 1810f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi res_line += ' |' 1820f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 1830f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi # If available, dump also performance results 18497f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi if 'performance' not in _results[tid][base_idx].keys(): 1850f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi print res_line 1860f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi return 1870f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 1880f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: 18997f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi res_base = _results[tid][base_idx]['performance'][pidx]['avg'] 19097f727443b83ef54e3bcdcd167e76cc4e0a59ad1Patrick Bellasi res_test = _results[tid][test_idx]['performance'][pidx]['avg'] 1910f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 1920f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi logging.debug('idx: %s, base: %s, test: %s', 1930f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi pidx, res_base, res_test) 1940f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 1958720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi if pidx in ['perf_avg']: 1968720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi res_line += ' {0:s}'.format(TestColors.rate(res_test)) 1978720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi continue 1988720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi 1998720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi if pidx in ['slack_pct']: 2008720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi res_line += ' {0:s}'.format( 2018720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi TestColors.rate(res_test, positive_is_good = False)) 2028720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi continue 2030f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 2048720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi # Compute difference base-vs-test 2058720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi if 'edp' in pidx: 2068720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi speedup_cnt = res_base - res_test 2078720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi if 'absolute': 2080f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi res_line += ' {0:10.2e}'.format(speedup_cnt) 2090f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi else: 2108720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) 2118720abc26202a8f25dce60d4bdefed5dfeb76b4fPatrick Bellasi 2120f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi res_line += ' |' 2130f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi print res_line 2140f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi 215f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi ############################### REPORT DEFAULT ############################# 216f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 21721756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi def __default_report(self, formats): 218f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 219f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Build list of workload types which can be rendered using the default parser 220f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi wtypes = [] 221f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi for supported_wtype in DEFAULT_WTYPES: 222f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if supported_wtype in self.results.keys(): 223f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi wtypes.append(supported_wtype) 224f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 225f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if len(wtypes) == 0: 226f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.debug('%14s - No Default workloads to report', 'ReportDefault') 227f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi return 228f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 229f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.debug('%14s - Reporting Default workloads', 'ReportDefault') 230f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 231f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Setup lables depending on requested report 23221756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi if 'absolute' in formats: 233f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi nrg_lable = 'Energy Indexes (Absolute)' 234f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi prf_lable = 'Performance Indexes (Absolute)' 235f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.info('') 236f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.info('%14s - Absolute comparisions:', 'Report') 237f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi print '' 238f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi else: 239f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi nrg_lable = 'Energy Indexes (Relative)' 240f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi prf_lable = 'Performance Indexes (Relative)' 241f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.info('') 242f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.info('%14s - Relative comparisions:', 'Report') 243f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi print '' 244f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 245f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Dump headers 246faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi print '{:9s} {:20s} |'\ 247f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi ' {:33s} | {:54s} |'\ 248f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi .format('Test Id', 'Comparision', 249f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi nrg_lable, prf_lable) 250faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi print '{:9s} {:20s} |'\ 251f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi ' {:>10s} {:>10s} {:>10s} |'\ 252f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ 253f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi .format('', '', 254f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 'LITTLE', 'big', 'Total', 255f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 'Perf', 'CTime', 'EDP1', 'EDP2', 'EDP3') 256f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 257f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # For each default test 258f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi for wtype in wtypes: 259f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi _results = self.results[wtype] 260f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi for tid in sorted(_results.keys()): 261f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi new_test = True 262f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # For each configuration... 263e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi for base_idx in sorted(_results[tid].keys()): 264f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Which matches at least on base regexp 265f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi for (base_rexp, test_rexp) in self.compare: 266f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if not base_rexp.match(base_idx): 267f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi continue 268f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Look for a configuration which matches the test regexp 269e3bebfc47781208a03169507b5bae5f22621579bPatrick Bellasi for test_idx in sorted(_results[tid].keys()): 270f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if test_idx == base_idx: 271f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi continue 272f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if new_test: 273faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi print '{:-<33s}+{:-<35s}+{:-<56s}+'\ 274f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi .format('','', '') 275f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi new_test = False 276f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if not test_rexp.match(test_idx): 277f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi continue 27821756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi self.__default_compare(wtype, tid, base_idx, test_idx, formats) 279f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 280f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi print '' 281f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 28221756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi def __default_compare(self, wtype, tid, base_idx, test_idx, formats): 283f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi _results = self.results[wtype] 284f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 285f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.debug('Test %s: compare %s with %s', 286f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi tid, base_idx, test_idx) 287f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_comp = '{0:s} vs {1:s}'.format(test_idx, base_idx) 288faf58b49b3246516c719d3bfdb000e22e4c6a431Patrick Bellasi res_line = '{0:8s}: {1:22s} | '.format(tid, res_comp) 289f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 290f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Dump all energy metrics 291f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi for cpus in ['LITTLE', 'big', 'Total']: 292f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 293f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # If either base of test have a 0 MAX energy, this measn that 294f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # energy has not been collected 295f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi base_max = _results[tid][base_idx]['energy'][cpus]['max'] 296f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi test_max = _results[tid][test_idx]['energy'][cpus]['max'] 297f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if base_max == 0 or test_max == 0: 298f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' {0:10s}'.format('NA') 299f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi continue 300f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 301f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Otherwise, report energy values 302f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_base = _results[tid][base_idx]['energy'][cpus]['avg'] 303f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_test = _results[tid][test_idx]['energy'][cpus]['avg'] 304f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 305f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_cnt = res_test - res_base 30621756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi if 'absolute' in formats: 307f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' {0:10.2f}'.format(speedup_cnt) 308f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi else: 309f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_pct = 100.0 * speedup_cnt / res_base 310f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' {0:s}'\ 311f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi .format(TestColors.rate( 312f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_pct, 313f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi positive_is_good = False)) 314f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' |' 315f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 316f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # If available, dump also performance results 317f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if 'performance' not in _results[tid][base_idx].keys(): 318f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi print res_line 319f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi return 320f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 321f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi for pidx in ['perf_avg', 'ctime_avg', 'edp1', 'edp2', 'edp3']: 322f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_base = _results[tid][base_idx]['performance'][pidx]['avg'] 323f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_test = _results[tid][test_idx]['performance'][pidx]['avg'] 324f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 325f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi logging.debug('idx: %s, base: %s, test: %s', 326f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi pidx, res_base, res_test) 327f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 328f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Compute difference base-vs-test 329f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_cnt = 0 330f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if res_base != 0: 331f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if pidx in ['perf_avg']: 332f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_cnt = res_test - res_base 333f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi else: 334f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_cnt = res_base - res_test 335f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 336f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # Compute speedup if required 337f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_pct = 0 33821756b552403589da5720eb04b85c9bfee83fd22Patrick Bellasi if 'absolute' in formats: 339f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if 'edp' in pidx: 340f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' {0:10.2e}'.format(speedup_cnt) 341f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi else: 342f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' {0:10.2f}'.format(speedup_cnt) 343f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi else: 344f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if res_base != 0: 345f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi if pidx in ['perf_avg']: 346f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi # speedup_pct = 100.0 * speedup_cnt / res_base 347f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_pct = speedup_cnt 348f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi else: 349f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi speedup_pct = 100.0 * speedup_cnt / res_base 350f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) 351f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi res_line += ' |' 352f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi print res_line 353f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 354f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi# List of workload types which can be parsed using the default test parser 355f97104f5c525f13f8375625062b0b7311a516d59Patrick BellasiDEFAULT_WTYPES = ['perf_bench_messaging', 'perf_bench_pipe'] 356f97104f5c525f13f8375625062b0b7311a516d59Patrick Bellasi 3570f8ac68fcd4e514ae6ee32a6496bd239fb9c03aPatrick Bellasi#vim :set tabstop=4 shiftwidth=4 expandtab 358