1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7from collections import defaultdict 8import json 9import os 10import re 11import subprocess 12import sys 13 14import suppressions 15 16 17def ReadReportsFromFile(filename): 18 """ Returns a list of (report_hash, report) and the URL of the report on the 19 waterfall. 20 """ 21 input_file = file(filename, 'r') 22 # reports is a list of (error hash, report) pairs. 23 reports = [] 24 in_suppression = False 25 cur_supp = [] 26 # This stores the last error hash found while reading the file. 27 last_hash = "" 28 for line in input_file: 29 line = line.strip() 30 line = line.replace("</span><span class=\"stdout\">", "") 31 line = line.replace("</span><span class=\"stderr\">", "") 32 line = line.replace("<", "<") 33 line = line.replace(">", ">") 34 if in_suppression: 35 if line == "}": 36 cur_supp += ["}"] 37 reports += [[last_hash, "\n".join(cur_supp)]] 38 in_suppression = False 39 cur_supp = [] 40 last_hash = "" 41 else: 42 cur_supp += [" "*3 + line] 43 elif line == "{": 44 in_suppression = True 45 cur_supp = ["{"] 46 elif line.find("Suppression (error hash=#") == 0: 47 last_hash = line[25:41] 48 # The line at the end of the file is assumed to store the URL of the report. 49 return reports,line 50 51def Demangle(names): 52 """ Demangle a list of C++ symbols, return a list of human-readable symbols. 53 """ 54 # -n is not the default on Mac. 55 args = ['c++filt', '-n'] 56 pipe = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 57 stdout, _ = pipe.communicate(input='\n'.join(names)) 58 demangled = stdout.split("\n") 59 # Each line ends with a newline, so the final entry of the split output 60 # will always be ''. 61 assert len(demangled) == len(names) 62 return demangled 63 64def GetSymbolsFromReport(report): 65 """Extract all symbols from a suppression report.""" 66 symbols = [] 67 prefix = "fun:" 68 prefix_len = len(prefix) 69 for line in report.splitlines(): 70 index = line.find(prefix) 71 if index != -1: 72 symbols.append(line[index + prefix_len:]) 73 return symbols 74 75def PrintTopSymbols(symbol_reports, top_count): 76 """Print the |top_count| symbols with the most occurrences.""" 77 boring_symbols=['malloc', '_Znw*', 'TestBody'] 78 sorted_reports = sorted(filter(lambda x:x[0] not in boring_symbols, 79 symbol_reports.iteritems()), 80 key=lambda x:len(x[1]), reverse=True) 81 symbols = symbol_reports.keys() 82 demangled = Demangle(symbols) 83 assert len(demangled) == len(symbols) 84 symboltable = dict(zip(symbols, demangled)) 85 86 print "\n" 87 print "Top %d symbols" % top_count 88 for (symbol, suppressions) in sorted_reports[:top_count]: 89 print "%4d occurrences : %s" % (len(suppressions), symboltable[symbol]) 90 91def ReadHashExclusions(exclusions): 92 input_file = file(exclusions, 'r') 93 contents = json.load(input_file) 94 return contents['hashes'] 95 96 97def main(argv): 98 supp = suppressions.GetSuppressions() 99 100 # all_reports is a map {report: list of urls containing this report} 101 all_reports = defaultdict(list) 102 report_hashes = {} 103 symbol_reports = defaultdict(list) 104 105 # Create argument parser. 106 parser = argparse.ArgumentParser() 107 parser.add_argument('--top-symbols', type=int, default=0, 108 help='Print a list of the top <n> symbols') 109 parser.add_argument('--symbol-filter', action='append', 110 help='Filter out all suppressions not containing the specified symbol(s). ' 111 'Matches against the mangled names.') 112 parser.add_argument('--exclude-symbol', action='append', 113 help='Filter out all suppressions containing the specified symbol(s). ' 114 'Matches against the mangled names.') 115 parser.add_argument('--exclude-hashes', action='append', 116 help='Specify a .json file with a list of hashes to exclude.') 117 118 parser.add_argument('reports', metavar='report file', nargs='+', 119 help='List of report files') 120 args = parser.parse_args(argv) 121 122 # exclude_hashes is a list of strings, each string an error hash. 123 exclude_hashes = [] 124 125 exclude_hashes = [] 126 if args.exclude_hashes: 127 for excl in args.exclude_hashes: 128 print "reading exclusion", excl 129 exclude_hashes += ReadHashExclusions(excl) 130 131 for f in args.reports: 132 f_reports, url = ReadReportsFromFile(f) 133 for (hash, report) in f_reports: 134 if hash in exclude_hashes: 135 continue 136 all_reports[report] += [url] 137 report_hashes[report] = hash 138 139 reports_count = 0 140 for r in all_reports: 141 cur_supp = supp['common_suppressions'] 142 if all([re.search("%20Mac%20|mac_valgrind", url) 143 for url in all_reports[r]]): 144 # Include mac suppressions if the report is only present on Mac 145 cur_supp += supp['mac_suppressions'] 146 elif all([re.search("Windows%20", url) for url in all_reports[r]]): 147 # Include win32 suppressions if the report is only present on Windows 148 cur_supp += supp['win_suppressions'] 149 elif all([re.search("Linux%20", url) for url in all_reports[r]]): 150 cur_supp += supp['linux_suppressions'] 151 if all(["DrMemory" in url for url in all_reports[r]]): 152 cur_supp += supp['drmem_suppressions'] 153 if all(["DrMemory%20full" in url for url in all_reports[r]]): 154 cur_supp += supp['drmem_full_suppressions'] 155 156 # Test if this report is already suppressed 157 skip = False 158 for s in cur_supp: 159 if s.Match(r.split("\n")): 160 skip = True 161 break 162 163 # Skip reports if none of the symbols are in the report. 164 if args.symbol_filter and all(not s in r for s in args.symbol_filter): 165 skip = True 166 if args.exclude_symbol and any(s in r for s in args.exclude_symbol): 167 skip = True 168 169 if not skip: 170 reports_count += 1 171 print "===================================" 172 print "This report observed at" 173 for url in all_reports[r]: 174 print " %s" % url 175 print "didn't match any suppressions:" 176 print "Suppression (error hash=#%s#):" % (report_hashes[r]) 177 print r 178 print "===================================" 179 180 if args.top_symbols > 0: 181 symbols = GetSymbolsFromReport(r) 182 for symbol in symbols: 183 symbol_reports[symbol].append(report_hashes[r]) 184 185 if reports_count > 0: 186 print ("%d unique reports don't match any of the suppressions" % 187 reports_count) 188 if args.top_symbols > 0: 189 PrintTopSymbols(symbol_reports, args.top_symbols) 190 191 else: 192 print "Congratulations! All reports are suppressed!" 193 # TODO(timurrrr): also make sure none of the old suppressions 194 # were narrowed too much. 195 196 197if __name__ == "__main__": 198 main(sys.argv[1:]) 199