1#!/usr/bin/env python
2import re, string, sys, os, time
3
4DEBUG = 0
5testDirName = 'llvm-test'
6test      = ['compile', 'llc', 'jit', 'cbe']
7exectime     = ['llc-time', 'jit-time', 'cbe-time',]
8comptime     = ['llc', 'jit-comptime', 'compile']
9
10(tp, exp) = ('compileTime_', 'executeTime_')
11
12def parse(file):
13  f=open(file, 'r')
14  d = f.read()
15
16  #Cleanup weird stuff
17  d = re.sub(r',\d+:\d','', d)
18
19  r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d)
20
21  test = {}
22  fname = ''
23  for t in r:
24    if DEBUG:
25      print t
26    if t[0] == 'PASS' or t[0] == 'FAIL' :
27      tmp = t[2].split(testDirName)
28
29      if DEBUG:
30        print tmp
31
32      if len(tmp) == 2:
33        fname = tmp[1].strip('\r\n')
34      else:
35        fname = tmp[0].strip('\r\n')
36
37      if not test.has_key(fname) :
38        test[fname] = {}
39
40      for k in test:
41        test[fname][k] = 'NA'
42        test[fname][t[1]] = t[0]
43        if DEBUG:
44          print test[fname][t[1]]
45    else :
46      try:
47        n = t[0].split('RESULT-')[1]
48
49        if DEBUG:
50          print n;
51
52        if n == 'llc' or n == 'jit-comptime' or n == 'compile':
53          test[fname][tp + n] = float(t[2].split(' ')[2])
54          if DEBUG:
55            print test[fname][tp + n]
56
57        elif n.endswith('-time') :
58            test[fname][exp + n] = float(t[2].strip('\r\n'))
59            if DEBUG:
60              print test[fname][exp + n]
61
62        else :
63          print "ERROR!"
64          sys.exit(1)
65
66      except:
67          continue
68
69  return test
70
71# Diff results and look for regressions.
72def diffResults(d_old, d_new):
73
74  for t in sorted(d_old.keys()) :
75    if DEBUG:
76      print t
77
78    if d_new.has_key(t) :
79
80      # Check if the test passed or failed.
81      for x in test:
82        if d_old[t].has_key(x):
83          if d_new[t].has_key(x):
84            if d_old[t][x] == 'PASS':
85              if d_new[t][x] != 'PASS':
86                print t + " *** REGRESSION (" + x + ")\n"
87            else:
88              if d_new[t][x] == 'PASS':
89                print t + " * NEW PASS (" + x + ")\n"
90
91          else :
92            print t + "*** REGRESSION (" + x + ")\n"
93
94        # For execution time, if there is no result, its a fail.
95        for x in exectime:
96          if d_old[t].has_key(tp + x):
97            if not d_new[t].has_key(tp + x):
98              print t + " *** REGRESSION (" + tp + x + ")\n"
99
100          else :
101            if d_new[t].has_key(tp + x):
102              print t + " * NEW PASS (" + tp + x + ")\n"
103
104
105        for x in comptime:
106          if d_old[t].has_key(exp + x):
107            if not d_new[t].has_key(exp + x):
108              print t + " *** REGRESSION (" + exp + x + ")\n"
109
110          else :
111            if d_new[t].has_key(exp + x):
112              print t + " * NEW PASS (" + exp + x + ")\n"
113
114    else :
115      print t + ": Removed from test-suite.\n"
116
117
118#Main
119if len(sys.argv) < 3 :
120    print 'Usage:', sys.argv[0], \
121          '<old log> <new log>'
122    sys.exit(-1)
123
124d_old = parse(sys.argv[1])
125d_new = parse(sys.argv[2])
126
127
128diffResults(d_old, d_new)
129
130
131