1#!/usr/bin/python
2# (C) Copyright IBM Corp. 2006
3# Author: Dustin Kirkland <dustin.kirkland@us.ibm.com>
4# Description:
5#  Input:  Two or more files containing results from different executions of
6#          the LTP. The input can either be file names or the url location
7#          of the ltp.results file.
8#  Output: A report on the following:
9#          - The total number of tests executed in each run
10#          - The testname, sequence number, and output of each run
11#            where the results of those runs differ
12#  Return:
13#          0 if all runs had identical results
14#          Non-zero if results differ, or bad input
15
16
17import sys, string, re
18from autotest_lib.client.common_lib import utils
19
20def usage():
21    print "\nUsage: \n\
22ltp-diff results1 results2 ... locationN \n\
23Note: location[1,2,N] may be local files or URLs of LTP results\n"
24    sys.exit(1)
25
26def get_results(results_files):
27    """
28    Download the results if needed.
29    Return results of each run in a numerically-indexed dictionary
30    of dictionaries keyed on testnames.
31    Return dictionary keyed on unique testnames across all runs.
32    """
33    r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:')
34    i = 0
35    runs = {}
36    testnames = {}
37    for file in results_files:
38        runs[i] = {}
39        try:
40            fh = utils.urlopen(file)
41            results = fh.readlines()
42            fh.close()
43        except:
44            print "ERROR: reading results resource [%s]" % (file)
45            usage()
46        for line in results:
47            try:
48                s = r.match(line)
49                testname = s.group(1)
50                status = s.group(2)
51                runs[i][testname] = status
52                testnames[testname] = 1
53            except:
54                pass
55        i += 1
56    return (runs, testnames)
57
58
59
60def compare_results(runs):
61    """
62    Loop through all testnames alpahbetically.
63    Print any testnames with differing results across runs.
64    Return 1 if any test results across runs differ.
65    Return 0 if all test results match.
66    """
67    rc = 0
68    print "LTP Test Results to Compare"
69    for i in range(len(runs)):
70        print "  Run[%d]: %d" % (i, len(runs[i].keys()))
71    print ""
72    header = 0
73    all_testnames = testnames.keys()
74    all_testnames.sort()
75    for testname in all_testnames:
76        differ = 0
77        for i in range(1,len(runs)):
78            # Must handle testcases that executed in one run
79            # but not another by setting status to "null"
80            if not runs[i].has_key(testname):
81                runs[i][testname] = "null"
82            if not runs[i-1].has_key(testname):
83                runs[i-1][testname] = "null"
84            # Check for the results inconsistencies
85            if runs[i][testname] != runs[i-1][testname]:
86                differ = 1
87        if differ:
88            if header == 0:
89                # Print the differences header only once
90                print "Tests with Inconsistent Results across Runs"
91                print "  %-35s:\t%s" % ("Testname,Sequence", "Run Results")
92                header = 1
93
94            # Print info if results differ
95            rc = 1
96            testname_cleaned = re.sub('\s+', ',', testname)
97            print "  %-35s:\t" % (testname_cleaned),
98            all_results = ""
99            for i in range(len(runs)):
100                all_results += runs[i][testname]
101                if i+1<len(runs):
102                    all_results += "/"
103            print all_results
104    if rc == 0:
105        print "All LTP results are identical"
106    return rc
107
108
109########
110# Main #
111########
112sys.argv.pop(0)
113if (len(sys.argv) < 2):
114    usage()
115(runs, testnames) = get_results(sys.argv)
116rc = compare_results(runs)
117sys.exit(rc)
118