1#!/usr/bin/env python
2# Copyright (c) 2011 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# drmemory_analyze.py
7
8''' Given a Dr. Memory output file, parses errors and uniques them.'''
9
10from collections import defaultdict
11import common
12import hashlib
13import logging
14import optparse
15import os
16import re
17import subprocess
18import sys
19import time
20
21class DrMemoryError:
22  def __init__(self, report, suppression, testcase):
23    self._report = report
24    self._testcase = testcase
25
26    # Chromium-specific transformations of the suppressions:
27    # Replace 'any_test.exe' and 'chrome.dll' with '*', then remove the
28    # Dr.Memory-generated error ids from the name= lines as they don't
29    # make sense in a multiprocess report.
30    supp_lines = suppression.split("\n")
31    for l in xrange(len(supp_lines)):
32      if supp_lines[l].startswith("name="):
33        supp_lines[l] = "name=<insert_a_suppression_name_here>"
34      if supp_lines[l].startswith("chrome.dll!"):
35        supp_lines[l] = supp_lines[l].replace("chrome.dll!", "*!")
36      bang_index = supp_lines[l].find("!")
37      d_exe_index = supp_lines[l].find(".exe!")
38      if bang_index >= 4 and d_exe_index + 4 == bang_index:
39        supp_lines[l] = "*" + supp_lines[l][bang_index:]
40    self._suppression = "\n".join(supp_lines)
41
42  def __str__(self):
43    output = self._report + "\n"
44    if self._testcase:
45      output += "The report came from the `%s` test.\n" % self._testcase
46    output += "Suppression (error hash=#%016X#):\n" % self.ErrorHash()
47    output += ("  For more info on using suppressions see "
48        "http://dev.chromium.org/developers/how-tos/using-drmemory#TOC-Suppressing-error-reports-from-the-\n")
49    output += "{\n%s\n}\n" % self._suppression
50    return output
51
52  # This is a device-independent hash identifying the suppression.
53  # By printing out this hash we can find duplicate reports between tests and
54  # different shards running on multiple buildbots
55  def ErrorHash(self):
56    return int(hashlib.md5(self._suppression).hexdigest()[:16], 16)
57
58  def __hash__(self):
59    return hash(self._suppression)
60
61  def __eq__(self, rhs):
62    return self._suppression == rhs
63
64
65class DrMemoryAnalyzer:
66  ''' Given a set of Dr.Memory output files, parse all the errors out of
67  them, unique them and output the results.'''
68
69  def __init__(self):
70    self.known_errors = set()
71    self.error_count = 0;
72
73  def ReadLine(self):
74    self.line_ = self.cur_fd_.readline()
75
76  def ReadSection(self):
77    result = [self.line_]
78    self.ReadLine()
79    while len(self.line_.strip()) > 0:
80      result.append(self.line_)
81      self.ReadLine()
82    return result
83
84  def ParseReportFile(self, filename, testcase):
85    ret = []
86
87    # First, read the generated suppressions file so we can easily lookup a
88    # suppression for a given error.
89    supp_fd = open(filename.replace("results", "suppress"), 'r')
90    generated_suppressions = {}  # Key -> Error #, Value -> Suppression text.
91    for line in supp_fd:
92      # NOTE: this regexp looks fragile. Might break if the generated
93      # suppression format slightly changes.
94      m = re.search("# Suppression for Error #([0-9]+)", line.strip())
95      if not m:
96        continue
97      error_id = int(m.groups()[0])
98      assert error_id not in generated_suppressions
99      # OK, now read the next suppression:
100      cur_supp = ""
101      for supp_line in supp_fd:
102        if supp_line.startswith("#") or supp_line.strip() == "":
103          break
104        cur_supp += supp_line
105      generated_suppressions[error_id] = cur_supp.strip()
106    supp_fd.close()
107
108    self.cur_fd_ = open(filename, 'r')
109    while True:
110      self.ReadLine()
111      if (self.line_ == ''): break
112
113      match = re.search("^Error #([0-9]+): (.*)", self.line_)
114      if match:
115        error_id = int(match.groups()[0])
116        self.line_ = match.groups()[1].strip() + "\n"
117        report = "".join(self.ReadSection()).strip()
118        suppression = generated_suppressions[error_id]
119        ret.append(DrMemoryError(report, suppression, testcase))
120
121      if re.search("SUPPRESSIONS USED:", self.line_):
122        self.ReadLine()
123        while self.line_.strip() != "":
124          line = self.line_.strip()
125          (count, name) = re.match(" *([0-9\?]+)x(?: \(.*?\))?: (.*)",
126                                   line).groups()
127          if (count == "?"):
128            # Whole-module have no count available: assume 1
129            count = 1
130          else:
131            count = int(count)
132          self.used_suppressions[name] += count
133          self.ReadLine()
134
135      if self.line_.startswith("ASSERT FAILURE"):
136        ret.append(self.line_.strip())
137
138    self.cur_fd_.close()
139    return ret
140
141  def Report(self, filenames, testcase, check_sanity):
142    sys.stdout.flush()
143    # TODO(timurrrr): support positive tests / check_sanity==True
144    self.used_suppressions = defaultdict(int)
145
146    to_report = []
147    reports_for_this_test = set()
148    for f in filenames:
149      cur_reports = self.ParseReportFile(f, testcase)
150
151      # Filter out the reports that were there in previous tests.
152      for r in cur_reports:
153        if r in reports_for_this_test:
154          # A similar report is about to be printed for this test.
155          pass
156        elif r in self.known_errors:
157          # A similar report has already been printed in one of the prev tests.
158          to_report.append("This error was already printed in some "
159                           "other test, see 'hash=#%016X#'" % r.ErrorHash())
160          reports_for_this_test.add(r)
161        else:
162          self.known_errors.add(r)
163          reports_for_this_test.add(r)
164          to_report.append(r)
165
166    common.PrintUsedSuppressionsList(self.used_suppressions)
167
168    if not to_report:
169      logging.info("PASS: No error reports found")
170      return 0
171
172    sys.stdout.flush()
173    sys.stderr.flush()
174    logging.info("Found %i error reports" % len(to_report))
175    for report in to_report:
176      self.error_count += 1
177      logging.info("Report #%d\n%s" % (self.error_count, report))
178    logging.info("Total: %i error reports" % len(to_report))
179    sys.stdout.flush()
180    return -1
181
182
183def main():
184  '''For testing only. The DrMemoryAnalyze class should be imported instead.'''
185  parser = optparse.OptionParser("usage: %prog <files to analyze>")
186
187  (options, args) = parser.parse_args()
188  if len(args) == 0:
189    parser.error("no filename specified")
190  filenames = args
191
192  logging.getLogger().setLevel(logging.INFO)
193  return DrMemoryAnalyzer().Report(filenames, None, False)
194
195
196if __name__ == '__main__':
197  sys.exit(main())
198