1067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands#!/usr/bin/python 262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendlingimport re, string, sys, os, time, math 3067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 4067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsDEBUG = 0 5067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 662c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling(tp, exp) = ('compile', 'exec') 7067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 8067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef parse(file): 962c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling f = open(file, 'r') 10067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands d = f.read() 11067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 1262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling # Cleanup weird stuff 1362c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling d = re.sub(r',\d+:\d', '', d) 1462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 15067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) 1662c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 17067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test = {} 18067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = '' 19067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for t in r: 20067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 21067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t 2262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 23067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if t[0] == 'PASS' or t[0] == 'FAIL' : 2462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling tmp = t[2].split('llvm-test/') 25067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 26067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 27067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print tmp 2862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 29067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if len(tmp) == 2: 30067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = tmp[1].strip('\r\n') 31067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else: 32067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = tmp[0].strip('\r\n') 3362c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 3462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if not test.has_key(fname): 35067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname] = {} 3662c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 3762c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling test[fname][t[1] + ' state'] = t[0] 3862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling test[fname][t[1] + ' time'] = float('nan') 39067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 40067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands try: 41067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands n = t[0].split('RESULT-')[1] 4262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 43067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 4462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling print "n == ", n; 45067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 4662c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if n == 'compile-success': 4762c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n')) 4862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 4962c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling elif n == 'exec-success': 5062c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n')) 51067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 5262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling print test[fname][string.replace(n, '-success', '')] 5362c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 54067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 5562c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling # print "ERROR!" 56067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands sys.exit(1) 5762c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 58067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands except: 59067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands continue 60067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 61067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands return test 62067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 63067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands# Diff results and look for regressions. 64067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef diffResults(d_old, d_new): 6542e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions = {} 6642e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling passes = {} 6742e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling removed = '' 6842e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 6942e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling for x in ['compile state', 'compile time', 'exec state', 'exec time']: 7042e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions[x] = '' 7142e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling passes[x] = '' 72067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 73067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for t in sorted(d_old.keys()) : 7462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if d_new.has_key(t): 7562c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 76067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands # Check if the test passed or failed. 7762c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling for x in ['compile state', 'compile time', 'exec state', 'exec time']: 7862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 7962c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if not d_old[t].has_key(x) and not d_new[t].has_key(x): 8062c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling continue 8162c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 82067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t].has_key(x): 83067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t].has_key(x): 8462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 85067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t][x] == 'PASS': 86067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t][x] != 'PASS': 8742e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions[x] += t + "\n" 88067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else: 89067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t][x] == 'PASS': 9042e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling passes[x] += t + "\n" 9162c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 92067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 9342e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions[x] += t + "\n" 9462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 9562c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if x == 'compile state' or x == 'exec state': 9662c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling continue 9762c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 9862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling # For execution time, if there is no result it's a fail. 9962c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if not d_old[t].has_key(x) and not d_new[t].has_key(x): 10062c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling continue 10162c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling elif not d_new[t].has_key(x): 10242e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions[x] += t + "\n" 10362c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling elif not d_old[t].has_key(x): 10442e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling passes[x] += t + "\n" 10562c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 10662c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 10762c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling continue 10862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 10962c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): 11042e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling passes[x] += t + "\n" 11162c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 11262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 11342e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions[x] += t + ": NaN%\n" 11462c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 115cfdee6a7df4b5c6bf3dae8b0999a01b87087b482Bill Wendling if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \ 116cfdee6a7df4b5c6bf3dae8b0999a01b87087b482Bill Wendling (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05: 11742e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n" 11862c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling 119067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 12042e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling removed += t + "\n" 12142e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 12242e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(regressions['compile state']) != 0: 12342e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'REGRESSION: Compilation Failed' 12442e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print regressions['compile state'] 12542e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 12642e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(regressions['exec state']) != 0: 12742e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'REGRESSION: Execution Failed' 12842e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print regressions['exec state'] 12942e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 13042e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(regressions['compile time']) != 0: 13142e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'REGRESSION: Compilation Time' 13242e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print regressions['compile time'] 13342e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 13442e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(regressions['exec time']) != 0: 13542e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'REGRESSION: Execution Time' 13642e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print regressions['exec time'] 13742e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 13842e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(passes['compile state']) != 0: 13942e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'NEW PASSES: Compilation' 14042e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print passes['compile state'] 14142e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 14242e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(passes['exec state']) != 0: 14342e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'NEW PASSES: Execution' 14442e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print passes['exec state'] 14542e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling 14642e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling if len(removed) != 0: 14742e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print 'REMOVED TESTS' 14842e0a001f4e35a6436f198f8b97c0dc14adfa153Bill Wendling print removed 149067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 15062c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling# Main 151067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsif len(sys.argv) < 3 : 15262c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling print 'Usage:', sys.argv[0], '<old log> <new log>' 15362c2e52ff1b7d54c8f45e15a8827a770ed65340cBill Wendling sys.exit(-1) 154067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 155067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_old = parse(sys.argv[1]) 156067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_new = parse(sys.argv[2]) 157067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 158067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsdiffResults(d_old, d_new) 159