xref: /external/lisa/libs/utils/report.py

# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


import argparse
import fnmatch as fnm
import json
import math
import numpy as np
import os
import re
import sys
import logging

from collections import defaultdict
from colors import TestColors
from results import Results


# By default compare all the possible combinations
DEFAULT_COMPARE = [(r'base_', r'test_')]

class Report(object):


    def __init__(self, results_dir, compare=None, formats=['relative']):
        self.results_json = results_dir + '/results.json'
        self.results = {}

        self.compare = []

        # Setup logging
        self._log = logging.getLogger('Report')

        # Parse results (if required)
        if not os.path.isfile(self.results_json):
            Results(results_dir)

        # Load results from file (if already parsed)
        self._log.info('Load results from [%s]...',
                       self.results_json)
        with open(self.results_json) as infile:
           self.results = json.load(infile)

        # Setup configuration comparisons
        if compare is None:
            compare = DEFAULT_COMPARE
            self._log.warning('Comparing all the possible combination')
        for (base_rexp, test_rexp) in compare:
            self._log.info('Configured regexps for comparisions '
                           '(bases , tests): (%s, %s)',
                           base_rexp, test_rexp)
            base_rexp = re.compile(base_rexp, re.DOTALL)
            test_rexp = re.compile(test_rexp, re.DOTALL)
            self.compare.append((base_rexp, test_rexp))

        # Report all supported workload classes
        self.__rtapp_report(formats)
        self.__default_report(formats)

    ############################### REPORT RTAPP ###############################

    def __rtapp_report(self, formats):

        if 'rtapp' not in self.results.keys():
            self._log.debug('No RTApp workloads to report')
            return

        self._log.debug('Reporting RTApp workloads')

        # Setup lables depending on requested report
        if 'absolute' in formats:
            nrg_lable = 'Energy Indexes (Absolute)'
            prf_lable = 'Performance Indexes (Absolute)'
            self._log.info('')
            self._log.info('Absolute comparisions:')
            print ''
        else:
            nrg_lable = 'Energy Indexes (Relative)'
            prf_lable = 'Performance Indexes (Relative)'
            self._log.info('')
            self._log.info('Relative comparisions:')
            print ''

        # Dump headers
        print '{:13s}   {:20s} |'\
                ' {:33s} | {:54s} |'\
                .format('Test Id', 'Comparision',
                        nrg_lable, prf_lable)
        print '{:13s}   {:20s} |'\
                ' {:>10s} {:>10s} {:>10s}  |'\
                ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\
                .format('', '',
                        'LITTLE', 'big', 'Total',
                        'PerfIndex', 'NegSlacks', 'EDP1', 'EDP2', 'EDP3')

        # For each test
        _results = self.results['rtapp']
        for tid in sorted(_results.keys()):
            new_test = True
            # For each configuration...
            for base_idx in sorted(_results[tid].keys()):
                # Which matches at least on base regexp
                for (base_rexp, test_rexp) in self.compare:
                    if not base_rexp.match(base_idx):
                        continue
                    # Look for a configuration which matches the test regexp
                    for test_idx in sorted(_results[tid].keys()):
                        if test_idx == base_idx:
                            continue
                        if new_test:
                            print '{:-<37s}+{:-<35s}+{:-<56s}+'\
                                    .format('','', '')
                            self.__rtapp_reference(tid, base_idx)
                            new_test = False
                        if test_rexp.match(test_idx) == None:
                            continue
                        self.__rtapp_compare(tid, base_idx, test_idx, formats)

        print ''

    def __rtapp_reference(self, tid, base_idx):
        _results = self.results['rtapp']

        self._log.debug('Test %s: compare against [%s] base',
                        tid, base_idx)
        res_line = '{0:12s}: {1:22s} | '.format(tid, base_idx)

        # Dump all energy metrics
        for cpus in ['LITTLE', 'big', 'Total']:
            res_base = _results[tid][base_idx]['energy'][cpus]['avg']
            # Dump absolute values
            res_line += ' {0:10.3f}'.format(res_base)
        res_line += ' |'

        # If available, dump also performance results
        if 'performance' not in _results[tid][base_idx].keys():
            print res_line
            return

        for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']:
            res_base = _results[tid][base_idx]['performance'][pidx]['avg']

            self._log.debug('idx: %s, base: %s', pidx, res_base)

            if pidx in ['perf_avg']:
                res_line += ' {0:s}'.format(TestColors.rate(res_base))
                continue
            if pidx in ['slack_pct']:
                res_line += ' {0:s}'.format(
                        TestColors.rate(res_base, positive_is_good = False))
                continue
            if 'edp' in pidx:
                res_line += ' {0:10.2e}'.format(res_base)
                continue
        res_line += ' |'
        print res_line

    def __rtapp_compare(self, tid, base_idx, test_idx, formats):
        _results = self.results['rtapp']

        self._log.debug('Test %s: compare %s with %s',
                        tid, base_idx, test_idx)
        res_line = '{0:12s}:   {1:20s} | '.format(tid, test_idx)

        # Dump all energy metrics
        for cpus in ['LITTLE', 'big', 'Total']:
            res_base = _results[tid][base_idx]['energy'][cpus]['avg']
            res_test = _results[tid][test_idx]['energy'][cpus]['avg']
            speedup_cnt =  res_test - res_base
            if 'absolute' in formats:
                res_line += ' {0:10.2f}'.format(speedup_cnt)
            else:
                speedup_pct = 0
                if res_base != 0:
                    speedup_pct =  100.0 * speedup_cnt / res_base
                res_line += ' {0:s}'\
                        .format(TestColors.rate(
                            speedup_pct,
                            positive_is_good = False))
        res_line += ' |'

        # If available, dump also performance results
        if 'performance' not in _results[tid][base_idx].keys():
            print res_line
            return

        for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']:
            res_base = _results[tid][base_idx]['performance'][pidx]['avg']
            res_test = _results[tid][test_idx]['performance'][pidx]['avg']

            self._log.debug('idx: %s, base: %s, test: %s',
                            pidx, res_base, res_test)

            if pidx in ['perf_avg']:
                res_line += ' {0:s}'.format(TestColors.rate(res_test))
                continue

            if pidx in ['slack_pct']:
                res_line += ' {0:s}'.format(
                        TestColors.rate(res_test, positive_is_good = False))
                continue

            # Compute difference base-vs-test
            if 'edp' in pidx:
                speedup_cnt = res_base - res_test
                if 'absolute':
                    res_line += ' {0:10.2e}'.format(speedup_cnt)
                else:
                    res_line += ' {0:s}'.format(TestColors.rate(speedup_pct))

        res_line += ' |'
        print res_line

    ############################### REPORT DEFAULT #############################

    def __default_report(self, formats):

        # Build list of workload types which can be rendered using the default parser
        wtypes = []
        for supported_wtype in DEFAULT_WTYPES:
            if supported_wtype in self.results.keys():
                wtypes.append(supported_wtype)

        if len(wtypes) == 0:
            self._log.debug('No Default workloads to report')
            return

        self._log.debug('Reporting Default workloads')

        # Setup lables depending on requested report
        if 'absolute' in formats:
            nrg_lable = 'Energy Indexes (Absolute)'
            prf_lable = 'Performance Indexes (Absolute)'
            self._log.info('')
            self._log.info('Absolute comparisions:')
            print ''
        else:
            nrg_lable = 'Energy Indexes (Relative)'
            prf_lable = 'Performance Indexes (Relative)'
            self._log.info('')
            self._log.info('Relative comparisions:')
            print ''

        # Dump headers
        print '{:9s}   {:20s} |'\
                ' {:33s} | {:54s} |'\
                .format('Test Id', 'Comparision',
                        nrg_lable, prf_lable)
        print '{:9s}   {:20s} |'\
                ' {:>10s} {:>10s} {:>10s}  |'\
                ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\
                .format('', '',
                        'LITTLE', 'big', 'Total',
                        'Perf', 'CTime', 'EDP1', 'EDP2', 'EDP3')

        # For each default test
        for wtype in wtypes:
            _results = self.results[wtype]
            for tid in sorted(_results.keys()):
                new_test = True
                # For each configuration...
                for base_idx in sorted(_results[tid].keys()):
                    # Which matches at least on base regexp
                    for (base_rexp, test_rexp) in self.compare:
                        if not base_rexp.match(base_idx):
                            continue
                        # Look for a configuration which matches the test regexp
                        for test_idx in sorted(_results[tid].keys()):
                            if test_idx == base_idx:
                                continue
                            if new_test:
                                print '{:-<37s}+{:-<35s}+{:-<56s}+'\
                                        .format('','', '')
                                new_test = False
                            if not test_rexp.match(test_idx):
                                continue
                            self.__default_compare(wtype, tid, base_idx, test_idx, formats)

        print ''

    def __default_compare(self, wtype, tid, base_idx, test_idx, formats):
        _results = self.results[wtype]

        self._log.debug('Test %s: compare %s with %s',
                        tid, base_idx, test_idx)
        res_comp = '{0:s} vs {1:s}'.format(test_idx, base_idx)
        res_line = '{0:8s}: {1:22s} | '.format(tid, res_comp)

        # Dump all energy metrics
        for cpus in ['LITTLE', 'big', 'Total']:

            # If either base of test have a 0 MAX energy, this measn that
            # energy has not been collected
            base_max = _results[tid][base_idx]['energy'][cpus]['max']
            test_max = _results[tid][test_idx]['energy'][cpus]['max']
            if base_max == 0 or test_max == 0:
                res_line += ' {0:10s}'.format('NA')
                continue

            # Otherwise, report energy values
            res_base = _results[tid][base_idx]['energy'][cpus]['avg']
            res_test = _results[tid][test_idx]['energy'][cpus]['avg']

            speedup_cnt =  res_test - res_base
            if 'absolute' in formats:
                res_line += ' {0:10.2f}'.format(speedup_cnt)
            else:
                speedup_pct =  100.0 * speedup_cnt / res_base
                res_line += ' {0:s}'\
                        .format(TestColors.rate(
                            speedup_pct,
                            positive_is_good = False))
        res_line += ' |'

        # If available, dump also performance results
        if 'performance' not in _results[tid][base_idx].keys():
            print res_line
            return

        for pidx in ['perf_avg', 'ctime_avg', 'edp1', 'edp2', 'edp3']:
            res_base = _results[tid][base_idx]['performance'][pidx]['avg']
            res_test = _results[tid][test_idx]['performance'][pidx]['avg']

            self._log.debug('idx: %s, base: %s, test: %s',
                            pidx, res_base, res_test)

            # Compute difference base-vs-test
            speedup_cnt = 0
            if res_base != 0:
                if pidx in ['perf_avg']:
                    speedup_cnt =  res_test - res_base
                else:
                    speedup_cnt =  res_base - res_test

            # Compute speedup if required
            speedup_pct = 0
            if 'absolute' in formats:
                if 'edp' in pidx:
                    res_line += ' {0:10.2e}'.format(speedup_cnt)
                else:
                    res_line += ' {0:10.2f}'.format(speedup_cnt)
            else:
                if res_base != 0:
                    if pidx in ['perf_avg']:
                        # speedup_pct =  100.0 * speedup_cnt / res_base
                        speedup_pct =  speedup_cnt
                    else:
                        speedup_pct =  100.0 * speedup_cnt / res_base
                res_line += ' {0:s}'.format(TestColors.rate(speedup_pct))
        res_line += ' |'
        print res_line

# List of workload types which can be parsed using the default test parser
DEFAULT_WTYPES = ['perf_bench_messaging', 'perf_bench_pipe']

#vim :set tabstop=4 shiftwidth=4 expandtab