findRegressions-simple.py revision f4594a3302466d514d8ae204d3602c1acba9d5c8 
1#!/usr/bin/env python
2import re, string, sys, os, time, math
3
4DEBUG = 0
5
6(tp, exp) = ('compile', 'exec')
7
8def parse(file):
9  f = open(file, 'r')
10  d = f.read()
11
12  # Cleanup weird stuff
13  d = re.sub(r',\d+:\d', '', d)
14
15  r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
16
17  test = {}
18  fname = ''
19  for t in r:
20    if DEBUG:
21      print t
22
23    if t[0] == 'PASS' or t[0] == 'FAIL' :
24      tmp = t[2].split('llvm-test/')
25
26      if DEBUG:
27        print tmp
28
29      if len(tmp) == 2:
30        fname = tmp[1].strip('\r\n')
31      else:
32        fname = tmp[0].strip('\r\n')
33
34      if not test.has_key(fname):
35        test[fname] = {}
36
37      test[fname][t[1] + ' state'] = t[0]
38      test[fname][t[1] + ' time'] = float('nan')
39    else :
40      try:
41        n = t[0].split('RESULT-')[1]
42
43        if DEBUG:
44          print "n == ", n;
45
46        if n == 'compile-success':
47          test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))
48
49        elif n == 'exec-success':
50          test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
51          if DEBUG:
52            print test[fname][string.replace(n, '-success', '')]
53
54        else :
55          # print "ERROR!"
56          sys.exit(1)
57
58      except:
59          continue
60
61  return test
62
63# Diff results and look for regressions.
64def diffResults(d_old, d_new):
65  regressions = {}
66  passes = {}
67  removed = ''
68
69  for x in ['compile state', 'compile time', 'exec state', 'exec time']:
70    regressions[x] = ''
71    passes[x] = ''
72
73  for t in sorted(d_old.keys()) :
74    if d_new.has_key(t):
75
76      # Check if the test passed or failed.
77      for x in ['compile state', 'compile time', 'exec state', 'exec time']:
78
79        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
80          continue
81
82        if d_old[t].has_key(x):
83          if d_new[t].has_key(x):
84
85            if d_old[t][x] == 'PASS':
86              if d_new[t][x] != 'PASS':
87                regressions[x] += t + "\n"
88            else:
89              if d_new[t][x] == 'PASS':
90                passes[x] += t + "\n"
91
92          else :
93            regressions[x] += t + "\n"
94
95        if x == 'compile state' or x == 'exec state':
96          continue
97
98        # For execution time, if there is no result it's a fail.
99        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
100          continue
101        elif not d_new[t].has_key(x):
102          regressions[x] += t + "\n"
103        elif not d_old[t].has_key(x):
104          passes[x] += t + "\n"
105
106        if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
107          continue
108
109        elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
110          passes[x] += t + "\n"
111
112        elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
113          regressions[x] += t + ": NaN%\n"
114
115        if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \
116              (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05:
117          regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n"
118
119    else :
120      removed += t + "\n"
121
122  if len(regressions['compile state']) != 0:
123    print 'REGRESSION: Compilation Failed'
124    print regressions['compile state']
125
126  if len(regressions['exec state']) != 0:
127    print 'REGRESSION: Execution Failed'
128    print regressions['exec state']
129
130  if len(regressions['compile time']) != 0:
131    print 'REGRESSION: Compilation Time'
132    print regressions['compile time']
133
134  if len(regressions['exec time']) != 0:
135    print 'REGRESSION: Execution Time'
136    print regressions['exec time']
137
138  if len(passes['compile state']) != 0:
139    print 'NEW PASSES: Compilation'
140    print passes['compile state']
141
142  if len(passes['exec state']) != 0:
143    print 'NEW PASSES: Execution'
144    print passes['exec state']
145
146  if len(removed) != 0:
147    print 'REMOVED TESTS'
148    print removed
149
150# Main
151if len(sys.argv) < 3 :
152  print 'Usage:', sys.argv[0], '<old log> <new log>'
153  sys.exit(-1)
154
155d_old = parse(sys.argv[1])
156d_new = parse(sys.argv[2])
157
158diffResults(d_old, d_new)
159