1f4594a3302466d514d8ae204d3602c1acba9d5c8Bill Wendling#!/usr/bin/env python 2aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingimport re, string, sys, os, time 3aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 4aec401194796ff0fd640e667227c7a752abcdd5bBill WendlingDEBUG = 0 5aec401194796ff0fd640e667227c7a752abcdd5bBill WendlingtestDirName = 'llvm-test' 6aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingtest = ['compile', 'llc', 'jit', 'cbe'] 7aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingexectime = ['llc-time', 'jit-time', 'cbe-time',] 8aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingcomptime = ['llc', 'jit-comptime', 'compile'] 9aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 10aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling(tp, exp) = ('compileTime_', 'executeTime_') 11aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 12aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingdef parse(file): 13aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling f=open(file, 'r') 14aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling d = f.read() 15aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 16aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling #Cleanup weird stuff 17aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling d = re.sub(r',\d+:\d','', d) 18aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 19aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) 20aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 21aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling test = {} 22aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling fname = '' 23aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling for t in r: 24aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 25aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t 26aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if t[0] == 'PASS' or t[0] == 'FAIL' : 27aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling tmp = t[2].split(testDirName) 28aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 29aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 30aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print tmp 31aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 32aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if len(tmp) == 2: 33aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling fname = tmp[1].strip('\r\n') 34aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else: 35aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling fname = tmp[0].strip('\r\n') 36aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 37aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if not test.has_key(fname) : 38aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling test[fname] = {} 39aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 40aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling for k in test: 41aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling test[fname][k] = 'NA' 42aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling test[fname][t[1]] = t[0] 43aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 44aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print test[fname][t[1]] 45aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else : 46aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling try: 47aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling n = t[0].split('RESULT-')[1] 48aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 49aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 50aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print n; 51aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 52aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if n == 'llc' or n == 'jit-comptime' or n == 'compile': 53aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling test[fname][tp + n] = float(t[2].split(' ')[2]) 54aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 55aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print test[fname][tp + n] 56aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 57aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling elif n.endswith('-time') : 58aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling test[fname][exp + n] = float(t[2].strip('\r\n')) 59aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 60aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print test[fname][exp + n] 61aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 62aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else : 63aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print "ERROR!" 64aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling sys.exit(1) 65aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 66aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling except: 67aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling continue 68aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 69aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling return test 70aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 71aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling# Diff results and look for regressions. 72aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingdef diffResults(d_old, d_new): 73aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 74aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling for t in sorted(d_old.keys()) : 75aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if DEBUG: 76aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t 77aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 78aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_new.has_key(t) : 79aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 80aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling # Check if the test passed or failed. 81aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling for x in test: 82aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_old[t].has_key(x): 83aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_new[t].has_key(x): 84aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_old[t][x] == 'PASS': 85aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_new[t][x] != 'PASS': 86aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + " *** REGRESSION (" + x + ")\n" 87aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else: 88aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_new[t][x] == 'PASS': 89aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + " * NEW PASS (" + x + ")\n" 90aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 91aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else : 92aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + "*** REGRESSION (" + x + ")\n" 93aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 94aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling # For execution time, if there is no result, its a fail. 95aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling for x in exectime: 96aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_old[t].has_key(tp + x): 97aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if not d_new[t].has_key(tp + x): 98aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + " *** REGRESSION (" + tp + x + ")\n" 99aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 100aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else : 101aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_new[t].has_key(tp + x): 102aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + " * NEW PASS (" + tp + x + ")\n" 103aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 104aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 105aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling for x in comptime: 106aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_old[t].has_key(exp + x): 107aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if not d_new[t].has_key(exp + x): 108aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + " *** REGRESSION (" + exp + x + ")\n" 109aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 110aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else : 111aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling if d_new[t].has_key(exp + x): 112aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + " * NEW PASS (" + exp + x + ")\n" 113aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 114aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling else : 115aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print t + ": Removed from test-suite.\n" 116aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 117aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 118aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling#Main 119aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingif len(sys.argv) < 3 : 120aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling print 'Usage:', sys.argv[0], \ 121aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling '<old log> <new log>' 122aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling sys.exit(-1) 123aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 124aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingd_old = parse(sys.argv[1]) 125aec401194796ff0fd640e667227c7a752abcdd5bBill Wendlingd_new = parse(sys.argv[2]) 126aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 127aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 128aec401194796ff0fd640e667227c7a752abcdd5bBill WendlingdiffResults(d_old, d_new) 129aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 130aec401194796ff0fd640e667227c7a752abcdd5bBill Wendling 131