1#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Report summarizer of internal test pass% from running many tests in LTP.
7
8LTP is the Linux Test Project from http://ltp.sourceforge.net/.
9
10This script serves to summarize the results of a test run by LTP test
11infrastructure.  LTP frequently runs >1000 tests so summarizing the results
12by result-type and count is useful. This script is invoked by the ltp.py
13wrapper in Autotest as a post-processing step to summarize the LTP run results
14in the Autotest log file.
15
16This script may be invoked by the command-line as follows:
17
18$ ./parse_ltp_out.py -l /mypath/ltp.out
19"""
20
21import optparse
22import os
23import re
24import sys
25
26
27SUMMARY_BORDER = 80 * '-'
28# Prefix char used in summaries:
29# +: sums into 'passing'
30# -: sums into 'notpassing'
31TEST_FILTERS = {'TPASS': '+Pass', 'TFAIL': '-Fail', 'TBROK': '-Broken',
32                'TCONF': '-Config error', 'TRETR': 'Retired',
33                'TWARN': '+Warning'}
34
35
36def parse_args(argv):
37    """Setup command line parsing options.
38
39    Args:
40        argv: command-line arguments.
41
42    Returns:
43        parsed option result from optparse.
44    """
45    parser = optparse.OptionParser('Usage: %prog --ltp-out-file=/path/ltp.out')
46    parser.add_option(
47        '-l', '--ltp-out-file',
48        help='[required] Path and file name for ltp.out [default: %default]',
49        dest='ltp_out_file',
50        default=None)
51    parser.add_option(
52        '-t', '--timings',
53        help='Show test timings in buckets [default: %default]',
54        dest='test_timings', action='store_true',
55        default=False)
56    options, args = parser.parse_args()
57    if not options.ltp_out_file:
58        parser.error('You must supply a value for --ltp-out-file.')
59
60    return options
61
62
63def _filter_and_count(ltp_out_file, test_filters):
64    """Utility function to count lines that match certain filters.
65
66    Args:
67        ltp_out_file: human-readable output file from LTP -p (ltp.out).
68        test_filters: dict of the tags to match and corresponding print tags.
69
70    Returns:
71        A dictionary with counts of the lines that matched each tag.
72    """
73    marker_line = '^<<<%s>>>$'
74    status_line_re = re.compile('^\w+ +\d+ +(\w+) +: +\w+')
75    filter_accumulator = dict.fromkeys(test_filters.keys(), 0)
76    parse_states = (
77        {'filters': {},
78         'terminator': re.compile(marker_line % 'test_output')},
79        {'filters': filter_accumulator,
80         'terminator': re.compile(marker_line % 'execution_status')})
81
82    # Simple 2-state state machine.
83    state_test_active = False
84    with open(ltp_out_file) as f:
85        for line in f:
86            state_index = int(state_test_active)
87            if re.match(parse_states[state_index]['terminator'], line):
88                # This state is terminated - proceed to next.
89                state_test_active = not state_test_active
90            else:
91                # Determine if this line matches any of the sought tags.
92                m = re.match(status_line_re, line)
93                if m and m.group(1) in parse_states[state_index]['filters']:
94                    parse_states[state_index]['filters'][m.group(1)] += 1
95    return filter_accumulator
96
97
98def _print_summary(filters, accumulator):
99    """Utility function to print the summary of the parsing of ltp.out.
100
101    Prints a count of each type of test result, then a %pass-rate score.
102
103    Args:
104        filters: map of tags sought and corresponding print headers.
105        accumulator: counts of test results with same keys as filters.
106    """
107    print SUMMARY_BORDER
108    print 'Linux Test Project (LTP) Run Summary:'
109    print SUMMARY_BORDER
110    # Size the header to the largest printable tag.
111    fmt = '%%%ss: %%s' % max(map(lambda x: len(x), filters.values()))
112    for k in sorted(filters):
113         print fmt % (filters[k], accumulator[k])
114
115    print SUMMARY_BORDER
116    # These calculations from ltprun-summary.sh script.
117    pass_count = sum([accumulator[k] for k in filters if filters[k][0] == '+'])
118    notpass_count = sum([accumulator[k] for k in filters
119                                        if filters[k][0] == '-'])
120    total_count = pass_count + notpass_count
121    if total_count:
122      score = float(pass_count) / float(total_count) * 100.0
123    else:
124      score = 0.0
125    print 'SCORE.ltp: %.2f' % score
126    print SUMMARY_BORDER
127
128
129def _filter_times(ltp_out_file):
130    """Utility function to count lines that match certain filters.
131
132    Args:
133        ltp_out_file: human-readable output file from LTP -p (ltp.out).
134
135    Returns:
136        A dictionary with test tags and corresponding times.  The dictionary is
137        a set of buckets of tests based on the test duration:
138          0: [tests that recoreded 0sec runtimes],
139          1: [tests that recorded runtimes from 0-60sec], ...
140          2: [tests that recorded runtimes from 61-120sec], ...
141    """
142    test_tag_line_re = re.compile('^tag=(\w+)\s+stime=(\d+)$')
143    test_duration_line_re = re.compile('^duration=(\d+)\s+.*')
144    filter_accumulator = {}
145    with open(ltp_out_file) as f:
146        previous_tag = None
147        previous_time_s = 0
148        recorded_tags = set()
149        for line in f:
150            tag_matches = re.match(test_tag_line_re, line)
151            if tag_matches:
152                current_tag = tag_matches.group(1)
153                if previous_tag:
154                    if previous_tag in recorded_tags:
155                        print 'WARNING: duplicate tag found: %s.' % previous_tag
156                previous_tag = current_tag
157                continue
158            duration_matches = re.match(test_duration_line_re, line)
159            if duration_matches:
160                duration = int(duration_matches.group(1))
161                if not previous_tag:
162                    print 'WARNING: duration without a tag: %s.' % duration
163                    continue
164                if duration != 0:
165                    duration = int(duration / 60) + 1
166                test_list = filter_accumulator.setdefault(duration, [])
167                test_list.append(previous_tag)
168    return filter_accumulator
169
170
171def _print_timings(accumulator):
172    """Utility function to print the summary of the parsing of ltp.out.
173
174    Prints a count of each type of test result, then a %pass-rate score.
175
176    Args:
177        filters: map of tags sought and corresponding print headers.
178        accumulator: counts of test results with same keys as filters.
179    """
180    print SUMMARY_BORDER
181    print 'Linux Test Project (LTP) Timing Summary:'
182    print SUMMARY_BORDER
183    for test_limit in sorted(accumulator.keys()):
184        print '<=%smin: %s tags: %s' % (
185            test_limit, len(accumulator[test_limit]),
186            ', '.join(sorted(accumulator[test_limit])))
187        print ''
188    print SUMMARY_BORDER
189    return
190
191
192def summarize(ltp_out_file, test_timings=None):
193    """Scan detailed output from LTP run for summary test status reporting.
194
195    Looks for all possible test result types know to LTP: pass, fail, broken,
196    config error, retired and warning.  Prints a summary.
197
198    Args:
199        ltp_out_file: human-readable output file from LTP -p (ltp.out).
200        test_timings: if True, emit an ordered summary of run timings of tests.
201    """
202    if not os.path.isfile(ltp_out_file):
203        print 'Unable to locate %s.' % ltp_out_file
204        return
205
206    _print_summary(TEST_FILTERS, _filter_and_count(ltp_out_file, TEST_FILTERS))
207    if test_timings:
208      _print_timings(_filter_times(ltp_out_file))
209
210
211def main(argv):
212    """ Parse the human-readable logs from an LTP run and print a summary.
213
214    Args:
215        argv: command-line arguments.
216    """
217    options = parse_args(argv)
218    summarize(options.ltp_out_file, options.test_timings)
219
220
221if __name__ == '__main__':
222    main(sys.argv)
223