check_clusterfuzz.py revision 3b9bc31999c9787eb726ecdbfd5796bfdec32a18 
1#!/usr/bin/env python
2# Copyright 2014 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""
7Script to check for new clusterfuzz issues since the last rolled v8 revision.
8
9Returns a json list with test case IDs if any.
10
11Security considerations: The security key and request data must never be
12written to public logs. Public automated callers of this script should
13suppress stdout and stderr and only process contents of the results_file.
14"""
15
16
17import argparse
18import httplib
19import json
20import os
21import re
22import sys
23import urllib
24import urllib2
25
26
27# Constants to git repos.
28BASE_URL = "https://chromium.googlesource.com"
29DEPS_LOG = BASE_URL + "/chromium/src/+log/master/DEPS?format=JSON"
30
31# Constants for retrieving v8 rolls.
32CRREV = "https://cr-rev.appspot.com/_ah/api/crrev/v1/commit/%s"
33V8_COMMIT_RE = re.compile(
34    r"^Update V8 to version \d+\.\d+\.\d+ \(based on ([a-fA-F0-9]+)\)\..*")
35
36# Constants for the clusterfuzz backend.
37HOSTNAME = "backend-dot-cluster-fuzz.appspot.com"
38
39# Crash patterns.
40V8_INTERNAL_RE = re.compile(r"^v8::internal.*")
41ANY_RE = re.compile(r".*")
42
43# List of all api requests.
44BUG_SPECS = [
45  {
46    "args": {
47      "job_type": "linux_asan_chrome_v8",
48      "reproducible": "True",
49      "open": "True",
50      "bug_information": "",
51    },
52    "crash_state": V8_INTERNAL_RE,
53  },
54  {
55    "args": {
56      "job_type": "linux_asan_d8",
57      "reproducible": "True",
58      "open": "True",
59      "bug_information": "",
60    },
61    "crash_state": ANY_RE,
62  },
63  {
64    "args": {
65      "job_type": "linux_asan_d8_dbg",
66      "reproducible": "True",
67      "open": "True",
68      "bug_information": "",
69    },
70    "crash_state": ANY_RE,
71  },
72  {
73    "args": {
74      "job_type": "linux_asan_d8_ignition_dbg",
75      "reproducible": "True",
76      "open": "True",
77      "bug_information": "",
78    },
79    "crash_state": ANY_RE,
80  },
81  {
82    "args": {
83      "job_type": "linux_asan_d8_v8_arm_dbg",
84      "reproducible": "True",
85      "open": "True",
86      "bug_information": "",
87    },
88    "crash_state": ANY_RE,
89  },
90  {
91    "args": {
92      "job_type": "linux_asan_d8_v8_arm64_dbg",
93      "reproducible": "True",
94      "open": "True",
95      "bug_information": "",
96    },
97    "crash_state": ANY_RE,
98  },
99  {
100    "args": {
101      "job_type": "linux_asan_d8_v8_mipsel_dbg",
102      "reproducible": "True",
103      "open": "True",
104      "bug_information": "",
105    },
106    "crash_state": ANY_RE,
107  },
108]
109
110
111def GetRequest(url):
112  url_fh = urllib2.urlopen(url, None, 60)
113  try:
114    return url_fh.read()
115  finally:
116    url_fh.close()
117
118
119def GetLatestV8InChromium():
120  """Returns the commit position number of the latest v8 roll in chromium."""
121
122  # Check currently rolled v8 revision.
123  result = GetRequest(DEPS_LOG)
124  if not result:
125    return None
126
127  # Strip security header and load json.
128  commits = json.loads(result[5:])
129
130  git_revision = None
131  for commit in commits["log"]:
132    # Get latest commit that matches the v8 roll pattern. Ignore cherry-picks.
133    match = re.match(V8_COMMIT_RE, commit["message"])
134    if match:
135      git_revision = match.group(1)
136      break
137  else:
138    return None
139
140  # Get commit position number for v8 revision.
141  result = GetRequest(CRREV % git_revision)
142  if not result:
143    return None
144
145  commit = json.loads(result)
146  assert commit["repo"] == "v8/v8"
147  return commit["number"]
148
149
150def APIRequest(key, **params):
151  """Send a request to the clusterfuzz api.
152
153  Returns a json dict of the response.
154  """
155
156  params["api_key"] = key
157  params = urllib.urlencode(params)
158
159  headers = {"Content-type": "application/x-www-form-urlencoded"}
160
161  try:
162    conn = httplib.HTTPSConnection(HOSTNAME)
163    conn.request("POST", "/_api/", params, headers)
164
165    response = conn.getresponse()
166
167    # Never leak "data" into public logs.
168    data = response.read()
169  except:
170    raise Exception("ERROR: Connection problem.")
171
172  try:
173    return json.loads(data)
174  except:
175    raise Exception("ERROR: Could not read response. Is your key valid?")
176
177  return None
178
179
180def Main():
181  parser = argparse.ArgumentParser()
182  parser.add_argument("-k", "--key-file", required=True,
183                      help="A file with the clusterfuzz api key.")
184  parser.add_argument("-r", "--results-file",
185                      help="A file to write the results to.")
186  options = parser.parse_args()
187
188  # Get api key. The key's content must never be logged.
189  assert options.key_file
190  with open(options.key_file) as f:
191    key = f.read().strip()
192  assert key
193
194  revision_number = GetLatestV8InChromium()
195
196  results = []
197  for spec in BUG_SPECS:
198    args = dict(spec["args"])
199    # Use incremented revision as we're interested in all revision greater than
200    # what's currently rolled into chromium.
201    if revision_number:
202      args["revision_greater_or_equal"] = str(int(revision_number) + 1)
203
204    # Never print issue details in public logs.
205    issues = APIRequest(key, **args)
206    assert issues is not None
207    for issue in issues:
208      if re.match(spec["crash_state"], issue["crash_state"]):
209        results.append(issue["id"])
210
211  if options.results_file:
212    with open(options.results_file, "w") as f:
213      f.write(json.dumps(results))
214  else:
215    print results
216
217
218if __name__ == "__main__":
219  sys.exit(Main())
220