107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty#!/usr/bin/python 207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty# Use of this source code is governed by a BSD-style license that can be 407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty# found in the LICENSE file. 507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty"""Report summarizer of internal test pass% from running many tests in LTP. 707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 807dabdb849469ff94b9fa37d037600185dd7d9b1Mike TrutyLTP is the Linux Test Project from http://ltp.sourceforge.net/. 907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 1007dabdb849469ff94b9fa37d037600185dd7d9b1Mike TrutyThis script serves to summarize the results of a test run by LTP test 1107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyinfrastructure. LTP frequently runs >1000 tests so summarizing the results 1207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyby result-type and count is useful. This script is invoked by the ltp.py 1307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutywrapper in Autotest as a post-processing step to summarize the LTP run results 1407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyin the Autotest log file. 1507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 1607dabdb849469ff94b9fa37d037600185dd7d9b1Mike TrutyThis script may be invoked by the command-line as follows: 1707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 1807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty$ ./parse_ltp_out.py -l /mypath/ltp.out 1907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty""" 2007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 2107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyimport optparse 2207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyimport os 2307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyimport re 2407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyimport sys 2507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 2607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 27496a364c6ec52e3de5ab6f394598532fe496ba41Mike TrutySUMMARY_BORDER = 80 * '-' 2807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty# Prefix char used in summaries: 2907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty# +: sums into 'passing' 3007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty# -: sums into 'notpassing' 3107dabdb849469ff94b9fa37d037600185dd7d9b1Mike TrutyTEST_FILTERS = {'TPASS': '+Pass', 'TFAIL': '-Fail', 'TBROK': '-Broken', 3207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 'TCONF': '-Config error', 'TRETR': 'Retired', 3307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 'TWARN': '+Warning'} 3407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 3507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 3607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutydef parse_args(argv): 3707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """Setup command line parsing options. 3807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 3907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Args: 4007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty argv: command-line arguments. 4107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 4207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Returns: 4307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty parsed option result from optparse. 4407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """ 4507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty parser = optparse.OptionParser('Usage: %prog --ltp-out-file=/path/ltp.out') 4607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty parser.add_option( 4707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty '-l', '--ltp-out-file', 4807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty help='[required] Path and file name for ltp.out [default: %default]', 49496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty dest='ltp_out_file', 5007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty default=None) 51496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty parser.add_option( 52496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty '-t', '--timings', 53496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty help='Show test timings in buckets [default: %default]', 54496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty dest='test_timings', action='store_true', 55496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty default=False) 5607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty options, args = parser.parse_args() 57496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if not options.ltp_out_file: 5807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty parser.error('You must supply a value for --ltp-out-file.') 5907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 6007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty return options 6107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 6207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 63496a364c6ec52e3de5ab6f394598532fe496ba41Mike Trutydef _filter_and_count(ltp_out_file, test_filters): 6407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """Utility function to count lines that match certain filters. 6507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 6607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Args: 67496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty ltp_out_file: human-readable output file from LTP -p (ltp.out). 6807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty test_filters: dict of the tags to match and corresponding print tags. 6907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 7007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Returns: 7107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty A dictionary with counts of the lines that matched each tag. 7207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """ 7307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty marker_line = '^<<<%s>>>$' 7407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty status_line_re = re.compile('^\w+ +\d+ +(\w+) +: +\w+') 7507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty filter_accumulator = dict.fromkeys(test_filters.keys(), 0) 7607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty parse_states = ( 7707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty {'filters': {}, 7807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 'terminator': re.compile(marker_line % 'test_output')}, 7907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty {'filters': filter_accumulator, 8007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 'terminator': re.compile(marker_line % 'execution_status')}) 8107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 8207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty # Simple 2-state state machine. 8307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty state_test_active = False 84496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty with open(ltp_out_file) as f: 8507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty for line in f: 8607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty state_index = int(state_test_active) 8707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty if re.match(parse_states[state_index]['terminator'], line): 8807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty # This state is terminated - proceed to next. 8907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty state_test_active = not state_test_active 9007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty else: 9107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty # Determine if this line matches any of the sought tags. 9207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty m = re.match(status_line_re, line) 9307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty if m and m.group(1) in parse_states[state_index]['filters']: 9407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty parse_states[state_index]['filters'][m.group(1)] += 1 9507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty return filter_accumulator 9607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 9707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 9807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutydef _print_summary(filters, accumulator): 9907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """Utility function to print the summary of the parsing of ltp.out. 10007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 10107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Prints a count of each type of test result, then a %pass-rate score. 10207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 10307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Args: 10407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty filters: map of tags sought and corresponding print headers. 10507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty accumulator: counts of test results with same keys as filters. 10607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """ 107496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 10807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty print 'Linux Test Project (LTP) Run Summary:' 109496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 11007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty # Size the header to the largest printable tag. 11107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty fmt = '%%%ss: %%s' % max(map(lambda x: len(x), filters.values())) 11207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty for k in sorted(filters): 11307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty print fmt % (filters[k], accumulator[k]) 11407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 115496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 11607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty # These calculations from ltprun-summary.sh script. 11707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty pass_count = sum([accumulator[k] for k in filters if filters[k][0] == '+']) 11807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty notpass_count = sum([accumulator[k] for k in filters 11907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty if filters[k][0] == '-']) 12007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty total_count = pass_count + notpass_count 12107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty if total_count: 12207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty score = float(pass_count) / float(total_count) * 100.0 12307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty else: 12407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty score = 0.0 12507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty print 'SCORE.ltp: %.2f' % score 126496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 127496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 128496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 129496a364c6ec52e3de5ab6f394598532fe496ba41Mike Trutydef _filter_times(ltp_out_file): 130496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty """Utility function to count lines that match certain filters. 131496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 132496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty Args: 133496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty ltp_out_file: human-readable output file from LTP -p (ltp.out). 13407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 135496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty Returns: 136496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty A dictionary with test tags and corresponding times. The dictionary is 137496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty a set of buckets of tests based on the test duration: 138496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 0: [tests that recoreded 0sec runtimes], 139496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 1: [tests that recorded runtimes from 0-60sec], ... 140496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 2: [tests that recorded runtimes from 61-120sec], ... 141496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty """ 142496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty test_tag_line_re = re.compile('^tag=(\w+)\s+stime=(\d+)$') 143496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty test_duration_line_re = re.compile('^duration=(\d+)\s+.*') 144496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty filter_accumulator = {} 145496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty with open(ltp_out_file) as f: 146496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty previous_tag = None 147496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty previous_time_s = 0 148496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty recorded_tags = set() 149496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty for line in f: 150496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty tag_matches = re.match(test_tag_line_re, line) 151496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if tag_matches: 152496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty current_tag = tag_matches.group(1) 153496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if previous_tag: 154496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if previous_tag in recorded_tags: 155496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print 'WARNING: duplicate tag found: %s.' % previous_tag 156496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty previous_tag = current_tag 157496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty continue 158496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty duration_matches = re.match(test_duration_line_re, line) 159496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if duration_matches: 160496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty duration = int(duration_matches.group(1)) 161496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if not previous_tag: 162496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print 'WARNING: duration without a tag: %s.' % duration 163496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty continue 164496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if duration != 0: 165496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty duration = int(duration / 60) + 1 166496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty test_list = filter_accumulator.setdefault(duration, []) 167496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty test_list.append(previous_tag) 168496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty return filter_accumulator 16907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 170496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 171496a364c6ec52e3de5ab6f394598532fe496ba41Mike Trutydef _print_timings(accumulator): 172496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty """Utility function to print the summary of the parsing of ltp.out. 173496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 174496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty Prints a count of each type of test result, then a %pass-rate score. 175496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 176496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty Args: 177496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty filters: map of tags sought and corresponding print headers. 178496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty accumulator: counts of test results with same keys as filters. 179496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty """ 180496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 181496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print 'Linux Test Project (LTP) Timing Summary:' 182496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 183496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty for test_limit in sorted(accumulator.keys()): 184496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print '<=%smin: %s tags: %s' % ( 185496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty test_limit, len(accumulator[test_limit]), 186496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty ', '.join(sorted(accumulator[test_limit]))) 187496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print '' 188496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print SUMMARY_BORDER 189496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty return 190496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 191496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty 192496a364c6ec52e3de5ab6f394598532fe496ba41Mike Trutydef summarize(ltp_out_file, test_timings=None): 19307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """Scan detailed output from LTP run for summary test status reporting. 19407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 19507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Looks for all possible test result types know to LTP: pass, fail, broken, 19607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty config error, retired and warning. Prints a summary. 19707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 19807dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Args: 199496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty ltp_out_file: human-readable output file from LTP -p (ltp.out). 200496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty test_timings: if True, emit an ordered summary of run timings of tests. 20107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """ 202496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if not os.path.isfile(ltp_out_file): 203496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty print 'Unable to locate %s.' % ltp_out_file 20407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty return 20507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 206496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty _print_summary(TEST_FILTERS, _filter_and_count(ltp_out_file, TEST_FILTERS)) 207496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty if test_timings: 208496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty _print_timings(_filter_times(ltp_out_file)) 20907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 21007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 21107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutydef main(argv): 21207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """ Parse the human-readable logs from an LTP run and print a summary. 21307dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 21407dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty Args: 21507dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty argv: command-line arguments. 21607dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty """ 21707dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty options = parse_args(argv) 218496a364c6ec52e3de5ab6f394598532fe496ba41Mike Truty summarize(options.ltp_out_file, options.test_timings) 21907dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 22007dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty 22107dabdb849469ff94b9fa37d037600185dd7d9b1Mike Trutyif __name__ == '__main__': 22207dabdb849469ff94b9fa37d037600185dd7d9b1Mike Truty main(sys.argv) 223