1#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Report summarizer of internal test pass% from running many tests in LTP.
7
8LTP is the Linux Test Project from http://ltp.sourceforge.net/.
9
10This script serves to summarize the results of a test run by LTP test
11infrastructure.  LTP frequently runs >1000 tests so summarizing the results
12by result-type and count is useful. This script is invoked by the ltp.py
13wrapper in Autotest as a post-processing step to summarize the LTP run results
14in the Autotest log file.
15
16This script may be invoked by the command-line as follows:
17
18$ ./parse_ltp_out.py -l /mypath/ltp.out
19"""
20
21import optparse
22import os
23import re
24import sys
25
26
27SUMMARY_BORDER = 80 * '-'
28# Prefix char used in summaries:
29# +: sums into 'passing'
30# -: sums into 'notpassing'
31TEST_FILTERS = {'TPASS': '+Pass', 'TFAIL': '-Fail', 'TBROK': '-Broken',
32                'TCONF': '-Config error', 'TRETR': 'Retired',
33                'TWARN': '+Warning'}
34
35
36def parse_args(argv):
37    """Setup command line parsing options.
38
39    @param argv: command-line arguments.
40
41    @return parsed option result from optparse.
42    """
43    parser = optparse.OptionParser('Usage: %prog --ltp-out-file=/path/ltp.out')
44    parser.add_option(
45        '-l', '--ltp-out-file',
46        help='[required] Path and file name for ltp.out [default: %default]',
47        dest='ltp_out_file',
48        default=None)
49    parser.add_option(
50        '-t', '--timings',
51        help='Show test timings in buckets [default: %default]',
52        dest='test_timings', action='store_true',
53        default=False)
54    options, args = parser.parse_args()
55    if not options.ltp_out_file:
56        parser.error('You must supply a value for --ltp-out-file.')
57
58    return options
59
60
61def _filter_and_count(ltp_out_file, test_filters):
62    """Utility function to count lines that match certain filters.
63
64    @param ltp_out_file: human-readable output file from LTP -p (ltp.out).
65    @param test_filters: dict of the tags to match and corresponding print tags.
66
67    @return a dictionary with counts of the lines that matched each tag.
68    """
69    marker_line = '^<<<%s>>>$'
70    status_line_re = re.compile('^\w+ +\d+ +(\w+) +: +\w+')
71    filter_accumulator = dict.fromkeys(test_filters.keys(), 0)
72    parse_states = (
73        {'filters': {},
74         'terminator': re.compile(marker_line % 'test_output')},
75        {'filters': filter_accumulator,
76         'terminator': re.compile(marker_line % 'execution_status')})
77
78    # Simple 2-state state machine.
79    state_test_active = False
80    with open(ltp_out_file) as f:
81        for line in f:
82            state_index = int(state_test_active)
83            if re.match(parse_states[state_index]['terminator'], line):
84                # This state is terminated - proceed to next.
85                state_test_active = not state_test_active
86            else:
87                # Determine if this line matches any of the sought tags.
88                m = re.match(status_line_re, line)
89                if m and m.group(1) in parse_states[state_index]['filters']:
90                    parse_states[state_index]['filters'][m.group(1)] += 1
91    return filter_accumulator
92
93
94def _print_summary(filters, accumulator):
95    """Utility function to print the summary of the parsing of ltp.out.
96
97    Prints a count of each type of test result, then a %pass-rate score.
98
99    @param filters: map of tags sought and corresponding print headers.
100    @param accumulator: counts of test results with same keys as filters.
101    """
102    print SUMMARY_BORDER
103    print 'Linux Test Project (LTP) Run Summary:'
104    print SUMMARY_BORDER
105    # Size the header to the largest printable tag.
106    fmt = '%%%ss: %%s' % max(map(lambda x: len(x), filters.values()))
107    for k in sorted(filters):
108         print fmt % (filters[k], accumulator[k])
109
110    print SUMMARY_BORDER
111    # These calculations from ltprun-summary.sh script.
112    pass_count = sum([accumulator[k] for k in filters if filters[k][0] == '+'])
113    notpass_count = sum([accumulator[k] for k in filters
114                                        if filters[k][0] == '-'])
115    total_count = pass_count + notpass_count
116    if total_count:
117      score = float(pass_count) / float(total_count) * 100.0
118    else:
119      score = 0.0
120    print 'SCORE.ltp: %.2f' % score
121    print SUMMARY_BORDER
122
123
124def _filter_times(ltp_out_file):
125    """Utility function to count lines that match certain filters.
126
127    @param ltp_out_file: human-readable output file from LTP -p (ltp.out).
128
129    @return a dictionary with test tags and corresponding times.
130            The dictionary is a set of buckets of tests based on the test
131            duration:
132            0: [tests that recoreded 0sec runtimes],
133            1: [tests that recorded runtimes from 0-60sec], ...
134            2: [tests that recorded runtimes from 61-120sec], ...
135    """
136    test_tag_line_re = re.compile('^tag=(\w+)\s+stime=(\d+)$')
137    test_duration_line_re = re.compile('^duration=(\d+)\s+.*')
138    filter_accumulator = {}
139    with open(ltp_out_file) as f:
140        previous_tag = None
141        previous_time_s = 0
142        recorded_tags = set()
143        for line in f:
144            tag_matches = re.match(test_tag_line_re, line)
145            if tag_matches:
146                current_tag = tag_matches.group(1)
147                if previous_tag:
148                    if previous_tag in recorded_tags:
149                        print 'WARNING: duplicate tag found: %s.' % previous_tag
150                previous_tag = current_tag
151                continue
152            duration_matches = re.match(test_duration_line_re, line)
153            if duration_matches:
154                duration = int(duration_matches.group(1))
155                if not previous_tag:
156                    print 'WARNING: duration without a tag: %s.' % duration
157                    continue
158                if duration != 0:
159                    duration = int(duration / 60) + 1
160                test_list = filter_accumulator.setdefault(duration, [])
161                test_list.append(previous_tag)
162    return filter_accumulator
163
164
165def _print_timings(accumulator):
166    """Utility function to print the summary of the parsing of ltp.out.
167
168    Prints a count of each type of test result, then a %pass-rate score.
169
170    Args:
171    @param accumulator: counts of test results
172    """
173    print SUMMARY_BORDER
174    print 'Linux Test Project (LTP) Timing Summary:'
175    print SUMMARY_BORDER
176    for test_limit in sorted(accumulator.keys()):
177        print '<=%smin: %s tags: %s' % (
178            test_limit, len(accumulator[test_limit]),
179            ', '.join(sorted(accumulator[test_limit])))
180        print ''
181    print SUMMARY_BORDER
182    return
183
184
185def summarize(ltp_out_file, test_timings=None):
186    """Scan detailed output from LTP run for summary test status reporting.
187
188    Looks for all possible test result types know to LTP: pass, fail, broken,
189    config error, retired and warning.  Prints a summary.
190
191    @param ltp_out_file: human-readable output file from LTP -p (ltp.out).
192    @param test_timings: if True, emit an ordered summary of run timings of
193                         tests.
194    """
195    if not os.path.isfile(ltp_out_file):
196        print 'Unable to locate %s.' % ltp_out_file
197        return
198
199    _print_summary(TEST_FILTERS, _filter_and_count(ltp_out_file, TEST_FILTERS))
200    if test_timings:
201      _print_timings(_filter_times(ltp_out_file))
202
203
204def main(argv):
205    """ Parse the human-readable logs from an LTP run and print a summary.
206
207    @param argv: command-line arguments.
208    """
209    options = parse_args(argv)
210    summarize(options.ltp_out_file, options.test_timings)
211
212
213if __name__ == '__main__':
214    main(sys.argv)
215