xref: /external/autotest/client/site_tests/firmware_TouchMTB/firmware_summary.py

# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A module providing the summary for multiple test results.

This firmware_summary module is used to collect the test results of
multiple rounds from the logs generated by different firmware versions.
The test results of the various validators of every gesture are displayed.
In addition, the test results of every validator across all gestures are
also summarized.

Usage:
$ python firmware_summary log_directory


A typical summary output looks like

Test Summary (by gesture)            :  fw_2.41   fw_2.42     count
---------------------------------------------------------------------
one_finger_tracking
  CountTrackingIDValidator           :     1.00      0.90        12
  LinearityBothEndsValidator         :     0.97      0.89        12
  LinearityMiddleValidator           :     1.00      1.00        12
  NoGapValidator                     :     0.74      0.24        12
  NoReversedMotionBothEndsValidator  :     0.68      0.34        12
  NoReversedMotionMiddleValidator    :     1.00      1.00        12
  ReportRateValidator                :     1.00      1.00        12
one_finger_to_edge
  CountTrackingIDValidator           :     1.00      1.00         4
  LinearityBothEndsValidator         :     0.88      0.89         4
  LinearityMiddleValidator           :     1.00      1.00         4
  NoGapValidator                     :     0.50      0.00         4
  NoReversedMotionMiddleValidator    :     1.00      1.00         4
  RangeValidator                     :     1.00      1.00         4

  ...


Test Summary (by validator)          :   fw_2.4  fw_2.4.a     count
---------------------------------------------------------------------
  CountPacketsValidator              :     1.00      0.82         6
  CountTrackingIDValidator           :     0.92      0.88        84

  ...

"""


import getopt
import os
import sys

import firmware_log
import test_conf as conf

from collections import defaultdict

from common_util import print_and_exit
from firmware_constants import OPTIONS
from test_conf import (log_root_dir, merged_validators, segment_weights,
                       validator_weights)
from validators import BaseValidator, get_parent_validators


class OptionsDisplayMetrics:
    """The options of displaying metrics."""
    # Defining the options of displaying metrics
    HIDE_SOME_METRICS_STATS = '0'
    DISPLAY_ALL_METRICS_STATS = '1'
    DISPLAY_ALL_METRICS_WITH_RAW_VALUES = '2'
    DISPLAY_METRICS_OPTIONS = [HIDE_SOME_METRICS_STATS,
                               DISPLAY_ALL_METRICS_STATS,
                               DISPLAY_ALL_METRICS_WITH_RAW_VALUES]
    DISPLAY_METRICS_DEFAULT = DISPLAY_ALL_METRICS_WITH_RAW_VALUES

    def __init__(self, option):
        """Initialize with the level value.

        @param option: the option of display metrics
        """
        if option not in self.DISPLAY_METRICS_OPTIONS:
            option = self.DISPLAY_METRICS_DEFAULT

        # To display all metrics statistics grouped by validators?
        self.display_all_stats = (
                option == self.DISPLAY_ALL_METRICS_STATS or
                option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES)

        # To display the raw metrics values in details on file basis?
        self.display_raw_values = (
                option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES)


class FirmwareSummary:
    """Summary for touch device firmware tests."""

    def __init__(self, log_dir, display_metrics=False, debug_flag=False,
                 display_scores=False, individual_round_flag=False,
                 segment_weights=segment_weights,
                 validator_weights=validator_weights):
        """ segment_weights and validator_weights are passed as arguments
        so that it is possible to assign arbitrary weights in unit tests.
        """
        if os.path.isdir(log_dir):
            self.log_dir = log_dir
        else:
            error_msg = 'Error: The test result directory does not exist: %s'
            print error_msg % log_dir
            sys.exit(1)

        self.display_metrics = display_metrics
        self.display_scores = display_scores
        self.slog = firmware_log.SummaryLog(log_dir,
                                            segment_weights,
                                            validator_weights,
                                            individual_round_flag,
                                            debug_flag)

    def _print_summary_title(self, summary_title_str):
        """Print the summary of the test results by gesture."""
        # Create a flexible column title format according to the number of
        # firmware versions which could be 1, 2, or more.
        #
        # A typical summary title looks like
        # Test Summary ()          :    fw_11.26             fw_11.23
        #                               mean  ssd  count     mean ssd count
        # ----------------------------------------------------------------------
        #
        # The 1st line above is called title_fw.
        # The 2nd line above is called title_statistics.
        #
        # As an example for 2 firmwares, title_fw_format looks like:
        #     '{0:<37}:  {1:>12}  {2:>21}'
        title_fw_format_list = ['{0:<37}:',]
        for i in range(len(self.slog.fws)):
            format_space = 12 if i == 0 else (12 + 9)
            title_fw_format_list.append('{%d:>%d}' % (i + 1, format_space))
        title_fw_format = ' '.join(title_fw_format_list)

        # As an example for 2 firmwares, title_statistics_format looks like:
        #     '{0:>47} {1:>6} {2:>5} {3:>8} {4:>6} {5:>5}'
        title_statistics_format_list = []
        for i in range(len(self.slog.fws)):
            format_space = (12 + 35) if i == 0 else 8
            title_statistics_format_list.append('{%d:>%d}' % (3 * i,
                                                              format_space))
            title_statistics_format_list.append('{%d:>%d}' % (3 * i + 1 , 6))
            title_statistics_format_list.append('{%d:>%d}' % (3 * i + 2 , 5))
        title_statistics_format = ' '.join(title_statistics_format_list)

        # Create title_fw_list
        # As an example for two firmware versions, it looks like
        #   ['Test Summary (by gesture)', 'fw_2.4', 'fw_2.5']
        title_fw_list = [summary_title_str,] + self.slog.fws

        # Create title_statistics_list
        # As an example for two firmware versions, it looks like
        #   ['mean', 'ssd', 'count', 'mean', 'ssd', 'count', ]
        title_statistics_list = ['mean', 'ssd', 'count'] * len(self.slog.fws)

        # Print the title.
        title_fw = title_fw_format.format(*title_fw_list)
        title_statistics = title_statistics_format.format(
                *title_statistics_list)
        print '\n\n', title_fw
        print title_statistics
        print '-' * len(title_statistics)

    def _print_result_stats(self, gesture=None):
        """Print the result statistics of validators."""
        for validator in self.slog.validators:
            stat_scores_data = []
            statistics_format_list = []
            for fw in self.slog.fws:
                result = self.slog.get_result(fw=fw, gesture=gesture,
                                              validators=validator)
                scores_data = result.stat_scores.all_data
                if scores_data:
                    stat_scores_data += scores_data
                    statistics_format_list.append('{:>8.2f} {:>6.2f} {:>5}')
                else:
                    stat_scores_data.append('')
                    statistics_format_list.append('{:>21}')

            # Print the score statistics of all firmwares on the same row.
            if any(stat_scores_data):
                stat_scores_data.insert(0, validator)
                statistics_format_list.insert(0,'  {:<35}:')
                statistics_format = ' '.join(statistics_format_list)
                print statistics_format.format(*tuple(stat_scores_data))

    def _print_result_stats_by_gesture(self):
        """Print the summary of the test results by gesture."""
        self._print_summary_title('Test Summary (by gesture)')
        for gesture in self.slog.gestures:
            print gesture
            self._print_result_stats(gesture=gesture)

    def _print_result_stats_by_validator(self):
        """Print the summary of the test results by validator. The validator
        results of all gestures are combined to compute the statistics.
        """
        self._print_summary_title('Test Summary (by validator)')
        self._print_result_stats()

    def _get_metric_name_for_display(self, metric_name):
        """Get the metric name for display.
        We would like to shorten the metric name when displayed.

        @param metric_name: a metric name
        """
        return metric_name.split('--')[0]

    def _get_merged_validators(self):
        merged = defaultdict(list)
        for validator_name in self.slog.validators:
            parents = get_parent_validators(validator_name)
            for parent in parents:
                if parent in merged_validators:
                    merged[parent].append(validator_name)
                    break
            else:
                merged[validator_name] = [validator_name,]
        return sorted(merged.values())

    def _print_statistics_of_metrics(self, detailed=True, gesture=None):
        """Print the statistics of metrics by gesture or by validator.

        @param gesture: print the statistics grouped by gesture
                if this argument is specified; otherwise, by validator.
        @param detailed: print statistics for all derived validators if True;
                otherwise, print the merged statistics, e.g.,
                both StationaryFingerValidator and StationaryTapValidator
                are merged into StationaryValidator.
        """
        # Print the complete title which looks like:
        #   <title_str>  <fw1>  <fw2>  ...  <description>
        fws = self.slog.fws
        num_fws = len(fws)
        fws_str_max_width = max(map(len, fws))
        fws_str_width = max(fws_str_max_width + 1, 10)
        table_name = ('Detailed table (for debugging)' if detailed else
                      'Summary table')
        title_str = ('Metrics statistics by gesture: ' + gesture if gesture else
                     'Metrics statistics by validator')
        description_str = 'description (lower is better)'
        fw_format = '{:>%d}' % fws_str_width
        complete_title = ('{:<37}: '.format(title_str) +
                          (fw_format * num_fws).format(*fws) +
                          '  {:<40}'.format(description_str))
        print '\n' * 2
        print table_name
        print complete_title
        print '-' * len(complete_title)

        # Print the metric name and the metric stats values of every firmwares
        name_format = ' ' * 6 + '{:<31}:'
        description_format = ' {:<40}'
        float_format = '{:>%d.2f}' % fws_str_width
        blank_format = '{:>%d}' % fws_str_width

        validators = (self.slog.validators if detailed else
                      self._get_merged_validators())

        for validator in validators:
            fw_stats_values = defaultdict(dict)
            for fw in fws:
                result = self.slog.get_result(fw=fw, gesture=gesture,
                                              validators=validator)
                stat_metrics = result.stat_metrics

                for metric_name in stat_metrics.metrics_values:
                    fw_stats_values[metric_name][fw] = \
                            stat_metrics.stats_values[metric_name]

            fw_stats_values_printed = False
            for metric_name, fw_values_dict in sorted(fw_stats_values.items()):
                values = []
                values_format = ''
                for fw in fws:
                    value = fw_values_dict.get(fw, '')
                    values.append(value)
                    values_format += float_format if value else blank_format

                # The metrics of some special validators will not be shown
                # unless the display_all_stats flag is True or any stats values
                # are non-zero.
                if (validator not in conf.validators_hidden_when_no_failures or
                        self.display_metrics.display_all_stats or any(values)):
                    if not fw_stats_values_printed:
                        fw_stats_values_printed = True
                        if isinstance(validator, list):
                            print (' ' + ' {}' * len(validator)).format(*validator)
                        else:
                            print '  ' + validator
                    disp_name = self._get_metric_name_for_display(metric_name)
                    print name_format.format(disp_name),
                    print values_format.format(*values),
                    print description_format.format(
                            stat_metrics.metrics_props[metric_name].description)

    def _print_raw_metrics_values(self):
        """Print the raw metrics values."""
        # The subkey() below extracts (gesture, variation, round) from
        # metric.key which is (fw, round, gesture, variation, validator)
        subkey = lambda key: (key[2], key[3], key[1])

        # The sum_len() below is used to calculate the sum of the length
        # of the elements in the subkey.
        sum_len = lambda lst: sum([len(str(l)) if l else 0 for l in lst])

        mnprops = firmware_log.MetricNameProps()
        print '\n\nRaw metrics values'
        print '-' * 80
        for fw in self.slog.fws:
            print '\n', fw
            for validator in self.slog.validators:
                result = self.slog.get_result(fw=fw, validators=validator)
                metrics_dict = result.stat_metrics.metrics_dict
                if metrics_dict:
                    print '\n' + ' ' * 3 + validator
                for metric_name, metrics in sorted(metrics_dict.items()):
                    disp_name = self._get_metric_name_for_display(metric_name)
                    print ' ' * 6 + disp_name

                    metric_note = mnprops.metrics_props[metric_name].note
                    if metric_note:
                        msg = '** Note: value below represents '
                        print ' ' * 9 + msg + metric_note

                    # Make a metric value list sorted by
                    #   (gesture, variation, round)
                    value_list = sorted([(subkey(metric.key), metric.value)
                                         for metric in metrics])

                    max_len = max([sum_len(value[0]) for value in value_list])
                    template_prefix = ' ' * 9 + '{:<%d}: ' % (max_len + 5)
                    for (gesture, variation, round), value in value_list:
                        template = template_prefix + (
                                '{}' if isinstance(value, tuple) else '{:.2f}')
                        gvr_str = '%s.%s (%s)' % (gesture, variation, round)
                        print template.format(gvr_str, value)

    def _print_final_weighted_averages(self):
        """Print the final weighted averages of all validators."""
        title_str = 'Test Summary (final weighted averages)'
        print '\n\n' + title_str
        print '-' * len(title_str)
        weighted_average = self.slog.get_final_weighted_average()
        for fw in self.slog.fws:
            print '%s: %4.3f' % (fw, weighted_average[fw])

    def print_result_summary(self):
        """Print the summary of the test results."""
        print self.slog.test_version
        if self.display_metrics:
            self._print_statistics_of_metrics(detailed=False)
            self._print_statistics_of_metrics(detailed=True)
            if self.display_metrics.display_raw_values:
                self._print_raw_metrics_values()
        if self.display_scores:
            self._print_result_stats_by_gesture()
            self._print_result_stats_by_validator()
            self._print_final_weighted_averages()


def _usage_and_exit():
    """Print the usage message and exit."""
    prog = sys.argv[0]
    print 'Usage: $ python %s [options]\n' % prog
    print 'options:'
    print '  -D, --%s' % OPTIONS.DEBUG
    print '        enable debug flag'
    print '  -d, --%s <directory>' % OPTIONS.DIR
    print '        specify which log directory to derive the summary'
    print '  -h, --%s' % OPTIONS.HELP
    print '        show this help'
    print '  -i, --%s' % OPTIONS.INDIVIDUAL
    print '        Calculate statistics of every individual round separately'
    print '  -m, --%s <verbose_level>' % OPTIONS.METRICS
    print '        display the summary metrics.'
    print '        verbose_level:'
    print '          0: hide some metrics statistics if they passed'
    print '          1: display all metrics statistics'
    print '          2: display all metrics statistics and ' \
                        'the detailed raw metrics values (default)'
    print '  -s, --%s' % OPTIONS.SCORES
    print '        display the scores (0.0 ~ 1.0)'
    print
    print 'Examples:'
    print '    Specify the log root directory.'
    print '    $ python %s -d /tmp' % prog
    print '    Hide some metrics statistics.'
    print '    $ python %s -m 0' % prog
    print '    Display all metrics statistics.'
    print '    $ python %s -m 1' % prog
    print '    Display all metrics statistics with detailed raw metrics values.'
    print '    $ python %s         # or' % prog
    print '    $ python %s -m 2' % prog
    sys.exit(1)


def _parsing_error(msg):
    """Print the usage and exit when encountering parsing error."""
    print 'Error: %s' % msg
    _usage_and_exit()


def _parse_options():
    """Parse the options."""
    # Set the default values of options.
    options = {OPTIONS.DEBUG: False,
               OPTIONS.DIR: log_root_dir,
               OPTIONS.INDIVIDUAL: False,
               OPTIONS.METRICS: OptionsDisplayMetrics(None),
               OPTIONS.SCORES: False,
    }

    try:
        short_opt = 'Dd:him:s'
        long_opt = [OPTIONS.DEBUG,
                    OPTIONS.DIR + '=',
                    OPTIONS.HELP,
                    OPTIONS.INDIVIDUAL,
                    OPTIONS.METRICS + '=',
                    OPTIONS.SCORES,
        ]
        opts, args = getopt.getopt(sys.argv[1:], short_opt, long_opt)
    except getopt.GetoptError, err:
        _parsing_error(str(err))

    for opt, arg in opts:
        if opt in ('-h', '--%s' % OPTIONS.HELP):
            _usage_and_exit()
        elif opt in ('-D', '--%s' % OPTIONS.DEBUG):
            options[OPTIONS.DEBUG] = True
        elif opt in ('-d', '--%s' % OPTIONS.DIR):
            options[OPTIONS.DIR] = arg
            if not os.path.isdir(arg):
                print 'Error: the log directory %s does not exist.' % arg
                _usage_and_exit()
        elif opt in ('-i', '--%s' % OPTIONS.INDIVIDUAL):
            options[OPTIONS.INDIVIDUAL] = True
        elif opt in ('-m', '--%s' % OPTIONS.METRICS):
            options[OPTIONS.METRICS] = OptionsDisplayMetrics(arg)
        elif opt in ('-s', '--%s' % OPTIONS.SCORES):
            options[OPTIONS.SCORES] = True
        else:
            msg = 'This option "%s" is not supported.' % opt
            _parsing_error(opt)

    return options


if __name__ == '__main__':
    options = _parse_options()
    summary = FirmwareSummary(options[OPTIONS.DIR],
                              display_metrics=options[OPTIONS.METRICS],
                              individual_round_flag=options[OPTIONS.INDIVIDUAL],
                              display_scores=options[OPTIONS.SCORES],
                              debug_flag=options[OPTIONS.DEBUG])
    summary.print_result_summary()