1#!/usr/bin/env python 2 3import testlog_parser, sys, os, xml, glob, re 4from table_formatter import * 5from optparse import OptionParser 6 7numeric_re = re.compile("(\d+)") 8cvtype_re = re.compile("(8U|8S|16U|16S|32S|32F|64F)C(\d{1,3})") 9cvtypes = { '8U': 0, '8S': 1, '16U': 2, '16S': 3, '32S': 4, '32F': 5, '64F': 6 } 10 11convert = lambda text: int(text) if text.isdigit() else text 12keyselector = lambda a: cvtype_re.sub(lambda match: " " + str(cvtypes.get(match.group(1), 7) + (int(match.group(2))-1) * 8) + " ", a) 13alphanum_keyselector = lambda key: [ convert(c) for c in numeric_re.split(keyselector(key)) ] 14 15def getSetName(tset, idx, columns, short = True): 16 if columns and len(columns) > idx: 17 prefix = columns[idx] 18 else: 19 prefix = None 20 if short and prefix: 21 return prefix 22 name = tset[0].replace(".xml","").replace("_", "\n") 23 if prefix: 24 return prefix + "\n" + ("-"*int(len(max(prefix.split("\n"), key=len))*1.5)) + "\n" + name 25 return name 26 27if __name__ == "__main__": 28 if len(sys.argv) < 2: 29 print >> sys.stderr, "Usage:\n", os.path.basename(sys.argv[0]), "<log_name1>.xml [<log_name2>.xml ...]" 30 exit(0) 31 32 parser = OptionParser() 33 parser.add_option("-o", "--output", dest="format", help="output results in text format (can be 'txt', 'html' or 'auto' - default)", metavar="FMT", default="auto") 34 parser.add_option("-m", "--metric", dest="metric", help="output metric", metavar="NAME", default="gmean") 35 parser.add_option("-u", "--units", dest="units", help="units for output values (s, ms (default), mks, ns or ticks)", metavar="UNITS", default="ms") 36 parser.add_option("-f", "--filter", dest="filter", help="regex to filter tests", metavar="REGEX", default=None) 37 parser.add_option("", "--module", dest="module", default=None, metavar="NAME", help="module prefix for test names") 38 parser.add_option("", "--columns", dest="columns", default=None, metavar="NAMES", help="comma-separated list of column aliases") 39 parser.add_option("", "--no-relatives", action="store_false", dest="calc_relatives", default=True, help="do not output relative values") 40 parser.add_option("", "--with-cycles-reduction", action="store_true", dest="calc_cr", default=False, help="output cycle reduction percentages") 41 parser.add_option("", "--with-score", action="store_true", dest="calc_score", default=False, help="output automatic classification of speedups") 42 parser.add_option("", "--progress", action="store_true", dest="progress_mode", default=False, help="enable progress mode") 43 parser.add_option("", "--regressions", dest="regressions", default=None, metavar="LIST", help="comma-separated custom regressions map: \"[r][c]#current-#reference\" (indexes of columns are 0-based, \"r\" - reverse flag, \"c\" - color flag for base data)") 44 parser.add_option("", "--show-all", action="store_true", dest="showall", default=False, help="also include empty and \"notrun\" lines") 45 parser.add_option("", "--match", dest="match", default=None) 46 parser.add_option("", "--match-replace", dest="match_replace", default="") 47 parser.add_option("", "--regressions-only", dest="regressionsOnly", default=None, metavar="X-FACTOR", help="show only tests with performance regressions not") 48 parser.add_option("", "--intersect-logs", dest="intersect_logs", default=False, help="show only tests present in all log files") 49 (options, args) = parser.parse_args() 50 51 options.generateHtml = detectHtmlOutputType(options.format) 52 if options.metric not in metrix_table: 53 options.metric = "gmean" 54 if options.metric.endswith("%") or options.metric.endswith("$"): 55 options.calc_relatives = False 56 options.calc_cr = False 57 if options.columns: 58 options.columns = [s.strip().replace("\\n", "\n") for s in options.columns.split(",")] 59 60 if options.regressions: 61 assert not options.progress_mode, 'unsupported mode' 62 63 def parseRegressionColumn(s): 64 """ Format: '[r][c]<uint>-<uint>' """ 65 reverse = s.startswith('r') 66 if reverse: 67 s = s[1:] 68 addColor = s.startswith('c') 69 if addColor: 70 s = s[1:] 71 parts = s.split('-', 1) 72 link = (int(parts[0]), int(parts[1]), reverse, addColor) 73 assert link[0] != link[1] 74 return link 75 76 options.regressions = [parseRegressionColumn(s) for s in options.regressions.split(',')] 77 78 # expand wildcards and filter duplicates 79 files = [] 80 seen = set() 81 for arg in args: 82 if ("*" in arg) or ("?" in arg): 83 flist = [os.path.abspath(f) for f in glob.glob(arg)] 84 flist = sorted(flist, key= lambda text: str(text).replace("M", "_")) 85 files.extend([ x for x in flist if x not in seen and not seen.add(x)]) 86 else: 87 fname = os.path.abspath(arg) 88 if fname not in seen and not seen.add(fname): 89 files.append(fname) 90 91 # read all passed files 92 test_sets = [] 93 for arg in files: 94 try: 95 tests = testlog_parser.parseLogFile(arg) 96 if options.filter: 97 expr = re.compile(options.filter) 98 tests = [t for t in tests if expr.search(str(t))] 99 if options.match: 100 tests = [t for t in tests if t.get("status") != "notrun"] 101 if tests: 102 test_sets.append((os.path.basename(arg), tests)) 103 except IOError as err: 104 sys.stderr.write("IOError reading \"" + arg + "\" - " + str(err) + os.linesep) 105 except xml.parsers.expat.ExpatError as err: 106 sys.stderr.write("ExpatError reading \"" + arg + "\" - " + str(err) + os.linesep) 107 108 if not test_sets: 109 sys.stderr.write("Error: no test data found" + os.linesep) 110 quit() 111 112 setsCount = len(test_sets) 113 114 if options.regressions is None: 115 reference = -1 if options.progress_mode else 0 116 options.regressions = [(i, reference, False, True) for i in range(1, len(test_sets))] 117 118 for link in options.regressions: 119 (i, ref, reverse, addColor) = link 120 assert i >= 0 and i < setsCount 121 assert ref < setsCount 122 123 # find matches 124 test_cases = {} 125 126 name_extractor = lambda name: str(name) 127 if options.match: 128 reg = re.compile(options.match) 129 name_extractor = lambda name: reg.sub(options.match_replace, str(name)) 130 131 for i in range(setsCount): 132 for case in test_sets[i][1]: 133 name = name_extractor(case) 134 if options.module: 135 name = options.module + "::" + name 136 if name not in test_cases: 137 test_cases[name] = [None] * setsCount 138 test_cases[name][i] = case 139 140 # build table 141 getter = metrix_table[options.metric][1] 142 getter_score = metrix_table["score"][1] if options.calc_score else None 143 getter_p = metrix_table[options.metric + "%"][1] if options.calc_relatives else None 144 getter_cr = metrix_table[options.metric + "$"][1] if options.calc_cr else None 145 tbl = table(metrix_table[options.metric][0]) 146 147 # header 148 tbl.newColumn("name", "Name of Test", align = "left", cssclass = "col_name") 149 for i in range(setsCount): 150 tbl.newColumn(str(i), getSetName(test_sets[i], i, options.columns, False), align = "center") 151 152 def addHeaderColumns(suffix, description, cssclass): 153 for link in options.regressions: 154 (i, ref, reverse, addColor) = link 155 if reverse: 156 i, ref = ref, i 157 current_set = test_sets[i] 158 current = getSetName(current_set, i, options.columns) 159 if ref >= 0: 160 reference_set = test_sets[ref] 161 reference = getSetName(reference_set, ref, options.columns) 162 else: 163 reference = 'previous' 164 tbl.newColumn(str(i) + '-' + str(ref) + suffix, '%s\nvs\n%s\n(%s)' % (current, reference, description), align='center', cssclass=cssclass) 165 166 if options.calc_cr: 167 addHeaderColumns(suffix='$', description='cycles reduction', cssclass='col_cr') 168 if options.calc_relatives: 169 addHeaderColumns(suffix='%', description='x-factor', cssclass='col_rel') 170 if options.calc_score: 171 addHeaderColumns(suffix='S', description='score', cssclass='col_name') 172 173 # rows 174 prevGroupName = None 175 needNewRow = True 176 lastRow = None 177 for name in sorted(test_cases.iterkeys(), key=alphanum_keyselector): 178 cases = test_cases[name] 179 if needNewRow: 180 lastRow = tbl.newRow() 181 if not options.showall: 182 needNewRow = False 183 tbl.newCell("name", name) 184 185 groupName = next(c for c in cases if c).shortName() 186 if groupName != prevGroupName: 187 prop = lastRow.props.get("cssclass", "") 188 if "firstingroup" not in prop: 189 lastRow.props["cssclass"] = prop + " firstingroup" 190 prevGroupName = groupName 191 192 for i in range(setsCount): 193 case = cases[i] 194 if case is None: 195 if options.intersect_logs: 196 needNewRow = False 197 break 198 tbl.newCell(str(i), "-") 199 else: 200 status = case.get("status") 201 if status != "run": 202 tbl.newCell(str(i), status, color="red") 203 else: 204 val = getter(case, cases[0], options.units) 205 if val: 206 needNewRow = True 207 tbl.newCell(str(i), formatValue(val, options.metric, options.units), val) 208 209 if needNewRow: 210 for link in options.regressions: 211 (i, reference, reverse, addColor) = link 212 if reverse: 213 i, reference = reference, i 214 tblCellID = str(i) + '-' + str(reference) 215 case = cases[i] 216 if case is None: 217 if options.calc_relatives: 218 tbl.newCell(tblCellID + "%", "-") 219 if options.calc_cr: 220 tbl.newCell(tblCellID + "$", "-") 221 if options.calc_score: 222 tbl.newCell(tblCellID + "$", "-") 223 else: 224 status = case.get("status") 225 if status != "run": 226 tbl.newCell(str(i), status, color="red") 227 if status != "notrun": 228 needNewRow = True 229 if options.calc_relatives: 230 tbl.newCell(tblCellID + "%", "-", color="red") 231 if options.calc_cr: 232 tbl.newCell(tblCellID + "$", "-", color="red") 233 if options.calc_score: 234 tbl.newCell(tblCellID + "S", "-", color="red") 235 else: 236 val = getter(case, cases[0], options.units) 237 def getRegression(fn): 238 if fn and val: 239 for j in reversed(range(i)) if reference < 0 else [reference]: 240 r = cases[j] 241 if r is not None and r.get("status") == 'run': 242 return fn(case, r, options.units) 243 valp = getRegression(getter_p) if options.calc_relatives or options.progress_mode else None 244 valcr = getRegression(getter_cr) if options.calc_cr else None 245 val_score = getRegression(getter_score) if options.calc_score else None 246 if not valp: 247 color = None 248 elif valp > 1.05: 249 color = 'green' 250 elif valp < 0.95: 251 color = 'red' 252 else: 253 color = None 254 if addColor: 255 if not reverse: 256 tbl.newCell(str(i), formatValue(val, options.metric, options.units), val, color=color) 257 else: 258 r = cases[reference] 259 if r is not None and r.get("status") == 'run': 260 val = getter(r, cases[0], options.units) 261 tbl.newCell(str(reference), formatValue(val, options.metric, options.units), val, color=color) 262 if options.calc_relatives: 263 tbl.newCell(tblCellID + "%", formatValue(valp, "%"), valp, color=color, bold=color) 264 if options.calc_cr: 265 tbl.newCell(tblCellID + "$", formatValue(valcr, "$"), valcr, color=color, bold=color) 266 if options.calc_score: 267 tbl.newCell(tblCellID + "S", formatValue(val_score, "S"), val_score, color = color, bold = color) 268 269 if not needNewRow: 270 tbl.trimLastRow() 271 272 if options.regressionsOnly: 273 for r in reversed(range(len(tbl.rows))): 274 for i in range(1, len(options.regressions) + 1): 275 val = tbl.rows[r].cells[len(tbl.rows[r].cells) - i].value 276 if val is not None and val < float(options.regressionsOnly): 277 break 278 else: 279 tbl.rows.pop(r) 280 281 # output table 282 if options.generateHtml: 283 if options.format == "moinwiki": 284 tbl.htmlPrintTable(sys.stdout, True) 285 else: 286 htmlPrintHeader(sys.stdout, "Summary report for %s tests from %s test logs" % (len(test_cases), setsCount)) 287 tbl.htmlPrintTable(sys.stdout) 288 htmlPrintFooter(sys.stdout) 289 else: 290 tbl.consolePrintTable(sys.stdout) 291 292 if options.regressionsOnly: 293 sys.exit(len(tbl.rows)) 294