findRegressions-simple.py revision 3a8eaa736f89a1462c2458828a42906cad17c560
1067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands#!/usr/bin/python 23a8eaa736f89a1462c2458828a42906cad17c560Bill Wendlingimport re, string, sys, os, time, math 3067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 4067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsDEBUG = 0 5067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 63a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling(tp, exp) = ('compile', 'exec') 7067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 8067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef parse(file): 93a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling f = open(file, 'r') 10067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands d = f.read() 11067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 123a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling # Cleanup weird stuff 133a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling d = re.sub(r',\d+:\d', '', d) 143a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 15067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) 163a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 17067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test = {} 18067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = '' 19067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for t in r: 20067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 21067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print t 223a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 23067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if t[0] == 'PASS' or t[0] == 'FAIL' : 243a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling tmp = t[2].split('llvm-test/') 25067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 26067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 27067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands print tmp 283a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 29067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if len(tmp) == 2: 30067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = tmp[1].strip('\r\n') 31067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else: 32067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands fname = tmp[0].strip('\r\n') 333a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 343a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if not test.has_key(fname): 35067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands test[fname] = {} 363a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 373a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling test[fname][t[1] + ' state'] = t[0] 383a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling test[fname][t[1] + ' time'] = float('nan') 39067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 40067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands try: 41067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands n = t[0].split('RESULT-')[1] 423a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 43067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 443a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print "n == ", n; 45067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 463a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if n == 'compile-success': 473a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n')) 483a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 493a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling elif n == 'exec-success': 503a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n')) 51067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if DEBUG: 523a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print test[fname][string.replace(n, '-success', '')] 533a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 54067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 553a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling # print "ERROR!" 56067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands sys.exit(1) 573a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 58067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands except: 59067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands continue 60067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 61067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands return test 62067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 63067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands# Diff results and look for regressions. 64067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsdef diffResults(d_old, d_new): 65067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 66067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands for t in sorted(d_old.keys()) : 673a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if d_new.has_key(t): 683a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 69067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands # Check if the test passed or failed. 703a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling for x in ['compile state', 'compile time', 'exec state', 'exec time']: 713a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 723a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if not d_old[t].has_key(x) and not d_new[t].has_key(x): 733a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling continue 743a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 75067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t].has_key(x): 76067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t].has_key(x): 773a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 78067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_old[t][x] == 'PASS': 79067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t][x] != 'PASS': 803a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " *** REGRESSION (" + x + " now fails)" 81067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else: 82067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands if d_new[t][x] == 'PASS': 833a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " * NEW PASS (" + x + " now fails)" 843a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 85067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 863a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + "*** REGRESSION (" + x + " now fails)" 873a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 883a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if x == 'compile state' or x == 'exec state': 893a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling continue 903a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 913a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling # For execution time, if there is no result it's a fail. 923a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if not d_old[t].has_key(x) and not d_new[t].has_key(x): 933a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling continue 943a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling elif not d_new[t].has_key(x): 953a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " *** REGRESSION (" + x + ")" 963a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling elif not d_old[t].has_key(x): 973a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " * NEW PASS (" + x + ")" 983a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 993a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 1003a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling continue 1013a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 1023a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): 1033a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " * NEW PASS (" + x + ")" 1043a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 1053a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 1063a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " *** REGRESSION (" + x + ")" 1073a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 1083a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling if d_new[t][x] > d_old[t][x] and \ 1093a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling (d_new[t][x] - d_old[t][x]) / d_new[t][x] > .05: 1103a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + " *** REGRESSION (" + x + ")" 1113a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling 112067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands else : 1133a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print t + ": Removed from test-suite." 114067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 1153a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling# Main 116067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsif len(sys.argv) < 3 : 1173a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling print 'Usage:', sys.argv[0], '<old log> <new log>' 1183a8eaa736f89a1462c2458828a42906cad17c560Bill Wendling sys.exit(-1) 119067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 120067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_old = parse(sys.argv[1]) 121067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sandsd_new = parse(sys.argv[2]) 122067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan Sands 123067e2e2adb0c1e857b0ff5c25403a7585d7f9fadDuncan SandsdiffResults(d_old, d_new) 124