findRegressions-simple.py revision 067e2e2adb0c1e857b0ff5c25403a7585d7f9fad
1067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands#!/usr/bin/python 2067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsimport re, string, sys, os, time 3067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 4067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsDEBUG = 0 5067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandstestDirName = 'llvm-test' 6067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandstest = ['compile', 'llc', 'jit', 'cbe'] 7067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsexectime = ['llc-time', 'jit-time', 'cbe-time',] 8067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandscomptime = ['llc', 'jit-comptime', 'compile'] 9067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 10067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands(tp, exp) = ('compileTime_', 'executeTime_') 11067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 12067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef parse(file): 13067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands f=open(file, 'r') 14067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands d = f.read() 15067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 16067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands #Cleanup weird stuff 17067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands d = re.sub(r',\d+:\d','', d) 18067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 19067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) 20067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 21067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test = {} 22067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = '' 23067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for t in r: 24067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 25067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t 26067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if t[0] == 'PASS' or t[0] == 'FAIL' : 27067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands tmp = t[2].split(testDirName) 28067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 29067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 30067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print tmp 31067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 32067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if len(tmp) == 2: 33067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = tmp[1].strip('\r\n') 34067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else: 35067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = tmp[0].strip('\r\n') 36067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 37067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if not test.has_key(fname) : 38067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname] = {} 39067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 40067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for k in test: 41067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname][k] = 'NA' 42067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname][t[1]] = t[0] 43067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 44067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print test[fname][t[1]] 45067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 46067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands try: 47067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands n = t[0].split('RESULT-')[1] 48067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 49067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 50067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print n; 51067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 52067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if n == 'llc' or n == 'jit-comptime' or n == 'compile': 53067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname][tp + n] = float(t[2].split(' ')[2]) 54067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 55067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print test[fname][tp + n] 56067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 57067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands elif n.endswith('-time') : 58067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname][exp + n] = float(t[2].strip('\r\n')) 59067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 60067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print test[fname][exp + n] 61067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 62067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 63067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print "ERROR!" 64067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands sys.exit(1) 65067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 66067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands except: 67067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands continue 68067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 69067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands return test 70067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 71067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands# Diff results and look for regressions. 72067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef diffResults(d_old, d_new): 73067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 74067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for t in sorted(d_old.keys()) : 75067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 76067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t 77067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 78067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new.has_key(t) : 79067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 80067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands # Check if the test passed or failed. 81067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for x in test: 82067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t].has_key(x): 83067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t].has_key(x): 84067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t][x] == 'PASS': 85067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t][x] != 'PASS': 86067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + " *** REGRESSION (" + x + ")\n" 87067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else: 88067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t][x] == 'PASS': 89067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + " * NEW PASS (" + x + ")\n" 90067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 91067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 92067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + "*** REGRESSION (" + x + ")\n" 93067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 94067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands # For execution time, if there is no result, its a fail. 95067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for x in exectime: 96067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t].has_key(tp + x): 97067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if not d_new[t].has_key(tp + x): 98067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + " *** REGRESSION (" + tp + x + ")\n" 99067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 100067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 101067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t].has_key(tp + x): 102067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + " * NEW PASS (" + tp + x + ")\n" 103067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 104067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 105067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for x in comptime: 106067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t].has_key(exp + x): 107067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if not d_new[t].has_key(exp + x): 108067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + " *** REGRESSION (" + exp + x + ")\n" 109067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 110067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 111067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t].has_key(exp + x): 112067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + " * NEW PASS (" + exp + x + ")\n" 113067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 114067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 115067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t + ": Removed from test-suite.\n" 116067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 117067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 118067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands#Main 119067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsif len(sys.argv) < 3 : 120067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print 'Usage:', sys.argv[0], \ 121067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands '<old log> <new log>' 122067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands sys.exit(-1) 123067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 124067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_old = parse(sys.argv[1]) 125067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_new = parse(sys.argv[2]) 126067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 127067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 128067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsdiffResults(d_old, d_new) 129067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 130067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 131