1f4594a3302466d514d8ae204d3602c1acba9d5c8Bill Wendling#!/usr/bin/env python
23a8eaa736f89a1462c2458828a42906cad17c560Bill Wendlingimport re, string, sys, os, time, math
3067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
4067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsDEBUG = 0
5067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
63a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling(tp, exp) = ('compile', 'exec')
7067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
8067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef parse(file):
93a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling  f = open(file, 'r')
10067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  d = f.read()
11067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
123a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling  # Cleanup weird stuff
133a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling  d = re.sub(r',\d+:\d', '', d)
143a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
15067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
163a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
17067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  test = {}
18067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  fname = ''
19067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  for t in r:
20067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands    if DEBUG:
21067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      print t
223a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
23067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands    if t[0] == 'PASS' or t[0] == 'FAIL' :
243a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling      tmp = t[2].split('llvm-test/')
25067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
26067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      if DEBUG:
27067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        print tmp
283a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
29067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      if len(tmp) == 2:
30067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        fname = tmp[1].strip('\r\n')
31067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      else:
32067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        fname = tmp[0].strip('\r\n')
333a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
343a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling      if not test.has_key(fname):
35067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        test[fname] = {}
363a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
373a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling      test[fname][t[1] + ' state'] = t[0]
383a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling      test[fname][t[1] + ' time'] = float('nan')
39067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands    else :
40067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      try:
41067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        n = t[0].split('RESULT-')[1]
423a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
43067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        if DEBUG:
443a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          print "n == ", n;
45067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
463a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        if n == 'compile-success':
473a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))
483a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
493a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        elif n == 'exec-success':
503a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
51067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands          if DEBUG:
523a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling            print test[fname][string.replace(n, '-success', '')]
533a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
54067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        else :
553a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          # print "ERROR!"
56067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands          sys.exit(1)
573a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
58067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      except:
59067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands          continue
60067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
61067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  return test
62067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
63067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands# Diff results and look for regressions.
64067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef diffResults(d_old, d_new):
65b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  regressions = {}
66b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  passes = {}
67b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  removed = ''
68b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
69b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  for x in ['compile state', 'compile time', 'exec state', 'exec time']:
70b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    regressions[x] = ''
71b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    passes[x] = ''
72067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
73067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands  for t in sorted(d_old.keys()) :
743a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling    if d_new.has_key(t):
753a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
76067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands      # Check if the test passed or failed.
773a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling      for x in ['compile state', 'compile time', 'exec state', 'exec time']:
783a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
793a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
803a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          continue
813a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
82067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands        if d_old[t].has_key(x):
83067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands          if d_new[t].has_key(x):
843a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
85067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands            if d_old[t][x] == 'PASS':
86067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands              if d_new[t][x] != 'PASS':
87b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling                regressions[x] += t + "\n"
88067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands            else:
89067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands              if d_new[t][x] == 'PASS':
90b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling                passes[x] += t + "\n"
913a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
92067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands          else :
93b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling            regressions[x] += t + "\n"
943a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
953a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        if x == 'compile state' or x == 'exec state':
963a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          continue
973a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
983a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        # For execution time, if there is no result it's a fail.
993a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
1003a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          continue
1013a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        elif not d_new[t].has_key(x):
102b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling          regressions[x] += t + "\n"
1033a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        elif not d_old[t].has_key(x):
104b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling          passes[x] += t + "\n"
1053a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
1063a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
1073a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling          continue
1083a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
1093a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]):
110b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling          passes[x] += t + "\n"
1113a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
1123a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling        elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
113b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling          regressions[x] += t + ": NaN%\n"
1143a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
1153df9f541a093614b62e49bfa3c0a9529f36d11dbBill Wendling        if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \
1163df9f541a093614b62e49bfa3c0a9529f36d11dbBill Wendling              (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05:
117b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling          regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n"
1183a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling
119067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands    else :
120b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling      removed += t + "\n"
121b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
122b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(regressions['compile state']) != 0:
123b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'REGRESSION: Compilation Failed'
124b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print regressions['compile state']
125b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
126b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(regressions['exec state']) != 0:
127b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'REGRESSION: Execution Failed'
128b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print regressions['exec state']
129b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
130b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(regressions['compile time']) != 0:
131b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'REGRESSION: Compilation Time'
132b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print regressions['compile time']
133b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
134b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(regressions['exec time']) != 0:
135b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'REGRESSION: Execution Time'
136b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print regressions['exec time']
137b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
138b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(passes['compile state']) != 0:
139b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'NEW PASSES: Compilation'
140b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print passes['compile state']
141b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
142b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(passes['exec state']) != 0:
143b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'NEW PASSES: Execution'
144b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print passes['exec state']
145b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling
146b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling  if len(removed) != 0:
147b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print 'REMOVED TESTS'
148b9ad624fcd63b3137d13b0dd5b13fc4eddb42067Bill Wendling    print removed
149067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
1503a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling# Main
151067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsif len(sys.argv) < 3 :
1523a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling  print 'Usage:', sys.argv[0], '<old log> <new log>'
1533a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling  sys.exit(-1)
154067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
155067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_old = parse(sys.argv[1])
156067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_new = parse(sys.argv[2])
157067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands
158067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsdiffResults(d_old, d_new)
159