rietveld_service.py revision 4a4f2fe02baf385f6c24fc98c6e17bf6ac5e0724 
1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Defines common functionality used for interacting with Rietveld."""
6
7import json
8import logging
9import mimetypes
10import urllib
11
12import httplib2
13from oauth2client import client
14
15from google.appengine.ext import ndb
16
17_EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email'
18PROJECTHOSTING_SCOPE = 'https://www.googleapis.com/auth/projecthosting'
19
20_DESCRIPTION = """This patch was automatically uploaded by the Chrome Perf
21Dashboard (https://chromeperf.appspot.com). It is being used to run a perf
22bisect try job. It should not be submitted."""
23
24
25class RietveldConfig(ndb.Model):
26  """Configuration info for a Rietveld service account.
27
28  The data is stored only in the App Engine datastore (and the cloud console)
29  and not the code because it contains sensitive information like private keys.
30  """
31  client_email = ndb.TextProperty()
32  service_account_key = ndb.TextProperty()
33
34  # The protocol and domain of the Rietveld host. Should not contain path.
35  server_url = ndb.TextProperty()
36
37  # The protocol and domain of the Internal Rietveld host which is used
38  # to create issues for internal only tests.
39  internal_server_url = ndb.TextProperty()
40
41
42def Credentials(config, scope):
43  """Returns a credentials object used to authenticate a Http object."""
44  return client.SignedJwtAssertionCredentials(
45      config.client_email, config.service_account_key, scope)
46
47
48def GetDefaultRietveldConfig():
49  """Returns the default rietveld config entity from the datastore."""
50  return ndb.Key(RietveldConfig, 'default_rietveld_config').get()
51
52
53class RietveldService(object):
54  """Implements a Python API to Rietveld via HTTP.
55
56  Authentication is handled via an OAuth2 access token minted from an RSA key
57  associated with a service account (which can be created via the Google API
58  console). For this to work, the Rietveld instance to talk to must be
59  configured to allow the service account client ID as OAuth2 audience (see
60  Rietveld source). Both the RSA key and the server URL are provided via static
61  application configuration.
62  """
63
64  def __init__(self, internal_only=False):
65    self.internal_only = internal_only
66    self._config = None
67    self._http = None
68
69  def Config(self):
70    if not self._config:
71      self._config = GetDefaultRietveldConfig()
72    return self._config
73
74  def _Http(self):
75    if not self._http:
76      self._http = httplib2.Http()
77      creds = Credentials(self.Config(), _EMAIL_SCOPE)
78      creds.authorize(self._http)
79    return self._http
80
81  def _XsrfToken(self):
82    """Requests a XSRF token from Rietveld."""
83    return self._MakeRequest(
84        'xsrf_token', headers={'X-Requesting-XSRF-Token': 1})[1]
85
86  def _MakeRequest(self, path, *args, **kwwargs):
87    """Makes a request to the Rietveld server."""
88    if self.internal_only:
89      server_url = self.Config().internal_server_url
90    else:
91      server_url = self.Config().server_url
92    url = '%s/%s' % (server_url, path)
93    response, content = self._Http().request(url, *args, **kwwargs)
94    return (response, content)
95
96  def _EncodeMultipartFormData(self, fields, files):
97    """Encode form fields for multipart/form-data.
98
99    Args:
100      fields: A sequence of (name, value) elements for regular form fields.
101      files: A sequence of (name, filename, value) elements for data to be
102             uploaded as files.
103    Returns:
104      (content_type, body) ready for httplib.HTTP instance.
105
106    Source:
107      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
108    """
109    boundary = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
110    crlf = '\r\n'
111    lines = []
112    for (key, value) in fields:
113      lines.append('--' + boundary)
114      lines.append('Content-Disposition: form-data; name="%s"' % key)
115      lines.append('')
116      if isinstance(value, unicode):
117        value = value.encode('utf-8')
118      lines.append(value)
119    for (key, filename, value) in files:
120      lines.append('--' + boundary)
121      lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
122                   (key, filename))
123      content_type = (mimetypes.guess_type(filename)[0] or
124                      'application/octet-stream')
125      lines.append('Content-Type: %s' % content_type)
126      lines.append('')
127      if isinstance(value, unicode):
128        value = value.encode('utf-8')
129      lines.append(value)
130    lines.append('--' + boundary + '--')
131    lines.append('')
132    body = crlf.join(lines)
133    content_type = 'multipart/form-data; boundary=%s' % boundary
134    return content_type, body
135
136  def UploadPatch(self, subject, patch, base_checksum, base_hashes,
137                  base_content, config_path):
138    """Uploads the given patch file contents to Rietveld.
139
140    The process of creating an issue and uploading the patch requires several
141    HTTP requests to Rietveld.
142
143    Rietveld API documentation: https://code.google.com/p/rietveld/wiki/APIs
144    For specific implementation in Rietveld codebase, see http://goo.gl/BW205J.
145
146    Args:
147      subject: Title of the job, as it will appear in rietveld.
148      patch: The patch, which is a specially-formatted string.
149      base_checksum: Base md5 checksum to send.
150      base_hashes: "Base hashes" string to send.
151      base_content: Base config file contents.
152      config_path: Path to the config file.
153
154    Returns:
155      A (issue ID, patchset ID) pair. These are strings that contain numerical
156      IDs. If the patch upload was unsuccessful, then (None, None) is returned.
157    """
158    base = 'https://chromium.googlesource.com/chromium/src.git@master'
159    repo_guid = 'c14d891d44f0afff64e56ed7c9702df1d807b1ee'
160    form_fields = [
161        ('subject', subject),
162        ('description', _DESCRIPTION),
163        ('base', base),
164        ('xsrf_token', self._XsrfToken()),
165        ('repo_guid', repo_guid),
166        ('content_upload', '1'),
167        ('base_hashes', base_hashes),
168    ]
169    uploaded_diff_file = [('data', 'data.diff', patch)]
170    ctype, body = self._EncodeMultipartFormData(
171        form_fields, uploaded_diff_file)
172    response, content = self._MakeRequest(
173        'upload', method='POST', body=body, headers={'content-type': ctype})
174    if response.get('status') != '200':
175      logging.error('Error %s uploading to /upload', response.get('status'))
176      logging.error(content)
177      return (None, None)
178
179    # There should always be 3 lines in the request, but sometimes Rietveld
180    # returns 2 lines. Log the content so we can debug further.
181    logging.info('Response from Rietveld /upload:\n%s', content)
182    if not content.startswith('Issue created.'):
183      logging.error('Unexpected response: %s', content)
184      return (None, None)
185    lines = content.splitlines()
186    if len(lines) < 2:
187      logging.error('Unexpected response %s', content)
188      return (None, None)
189
190    msg = lines[0]
191    issue_id = msg[msg.rfind('/')+1:]
192    patchset_id = lines[1].strip()
193    patches = [x.split(' ', 1) for x in lines[2:]]
194    request_path = '%d/upload_content/%d/%d' % (
195        int(issue_id), int(patchset_id), int(patches[0][0]))
196    form_fields = [
197        ('filename', config_path),
198        ('status', 'M'),
199        ('checksum', base_checksum),
200        ('is_binary', str(False)),
201        ('is_current', str(False)),
202    ]
203    uploaded_diff_file = [('data', config_path, base_content)]
204    ctype, body = self._EncodeMultipartFormData(form_fields, uploaded_diff_file)
205    response, content = self._MakeRequest(
206        request_path, method='POST', body=body, headers={'content-type': ctype})
207    if response.get('status') != '200':
208      logging.error(
209          'Error %s uploading to %s', response.get('status'), request_path)
210      logging.error(content)
211      return (None, None)
212
213    request_path = '%s/upload_complete/%s' % (issue_id, patchset_id)
214    response, content = self._MakeRequest(request_path, method='POST')
215    if response.get('status') != '200':
216      logging.error(
217          'Error %s uploading to %s', response.get('status'), request_path)
218      logging.error(content)
219      return (None, None)
220    return issue_id, patchset_id
221
222  def TryPatch(self, tryserver_master, issue_id, patchset_id, bot):
223    """Sends a request to try the given patchset on the given trybot.
224
225    To see exactly how this request is handled, you can see the try_patchset
226    handler in the Chromium branch of Rietveld: http://goo.gl/U6tJQZ
227
228    Args:
229      tryserver_master: Master name, e.g. "tryserver.chromium.perf".
230      issue_id: Rietveld issue ID.
231      patchset_id: Patchset ID (returned when a patch is uploaded).
232      bot: Bisect bot name.
233
234    Returns:
235      True if successful, False otherwise.
236    """
237    args = {
238        'xsrf_token': self._XsrfToken(),
239        'builders': json.dumps({bot: ['defaulttests']}),
240        'master': tryserver_master,
241        'reason': 'Perf bisect',
242        'clobber': 'False',
243    }
244    request_path = '%s/try/%s' % (issue_id, patchset_id)
245    response, content = self._MakeRequest(
246        request_path, method='POST', body=urllib.urlencode(args))
247    if response.get('status') != '200':
248      status = response.get('status')
249      logging.error(
250          'Error %s POSTing to /%s/try/%s', status, issue_id, patchset_id)
251      logging.error(content)
252      return False
253    return True
254