1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This module contains functionality for starting build try jobs via HTTP.
6
7This includes both sending a request to start a job, and also related code
8for querying the status of the job.
9
10This module can be either run as a stand-alone script to send a request to a
11builder, or imported and used by calling the public functions below.
12"""
13
14import getpass
15import json
16import optparse
17import os
18import sys
19import urllib
20import urllib2
21
22# URL template for fetching JSON data about builds.
23BUILDER_JSON_URL = ('%(server_url)s/json/builders/%(bot_name)s/builds/'
24                    '%(build_num)s?as_text=1&filter=0')
25
26# URL template for displaying build steps.
27BUILDER_HTML_URL = ('%(server_url)s/builders/%(bot_name)s/builds/%(build_num)s')
28
29# Try server status page for the perf try server.
30PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
31
32# Hostname of the tryserver where perf bisect builders are hosted.
33# This is used for posting build requests to the tryserver.
34PERF_BISECT_BUILDER_HOST = 'master4.golo.chromium.org'
35
36# The default 'try_job_port' on tryserver to post build request.
37PERF_BISECT_BUILDER_PORT = 8341
38
39# Status codes that can be returned by the GetBuildStatus method.
40# From buildbot.status.builder.
41# See: http://docs.buildbot.net/current/developer/results.html
42SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, TRYPENDING = range(7)
43
44OK = (SUCCESS, WARNINGS)  # These indicate build is complete.
45FAILED = (FAILURE, EXCEPTION, SKIPPED)  # These indicate build failure.
46PENDING = (RETRY, TRYPENDING)  # These indicate in progress or in pending queue.
47
48
49class ServerAccessError(Exception):
50
51  def __str__(self):
52    return '%s\nSorry, cannot connect to server.' % self.args[0]
53
54
55def PostTryJob(host, port, params):
56  """Sends a build request to the server using the HTTP protocol.
57
58  The required parameters are:
59    'revision': "src@rev", where rev is a git hash or SVN revision.
60    'bot': Name of builder bot to use, e.g. "win_perf_bisect_builder".
61
62  Args:
63    host: Hostname of the try server.
64    port: Port of the try server.
65    params: A dictionary of parameters to be sent in the POST request.
66
67  Returns:
68    True if the request is posted successfully.
69
70  Raises:
71    ServerAccessError: Failed to make a request to the try server.
72    ValueError: Not all of the expected inputs were given.
73  """
74  if not params.get('revision'):
75    raise ValueError('Missing revision number.')
76  if not params.get('bot'):
77    raise ValueError('Missing bot name.')
78
79  base_url = 'http://%s:%s/send_try_patch' % (host, port)
80  print 'Posting build request by HTTP.'
81  print 'URL: %s, params: %s' % (base_url, params)
82
83  connection = None
84  try:
85    print 'Opening connection...'
86    connection = urllib2.urlopen(base_url, urllib.urlencode(params))
87    print 'Done, request sent to server to produce build.'
88  except IOError as e:
89    raise ServerAccessError('%s is unaccessible. Reason: %s' % (base_url, e))
90  if not connection:
91    raise ServerAccessError('%s is unaccessible.' % base_url)
92
93  response = connection.read()
94  print 'Received response from server: %s' % response
95  if response != 'OK':
96    raise ServerAccessError('Response was not "OK".')
97  return True
98
99
100def _IsBuildRunning(build_data):
101  """Checks whether the build is in progress on buildbot.
102
103  Presence of currentStep element in build JSON indicates build is in progress.
104
105  Args:
106    build_data: A dictionary with build data, loaded from buildbot JSON API.
107
108  Returns:
109    True if build is in progress, otherwise False.
110  """
111  current_step = build_data.get('currentStep')
112  if (current_step and current_step.get('isStarted') and
113      current_step.get('results') is None):
114    return True
115  return False
116
117
118def _IsBuildFailed(build_data):
119  """Checks whether the build failed on buildbot.
120
121  Sometime build status is marked as failed even though compile and packaging
122  steps are successful. This may happen due to some intermediate steps of less
123  importance such as gclient revert, generate_telemetry_profile are failed.
124  Therefore we do an addition check to confirm if build was successful by
125  calling _IsBuildSuccessful.
126
127  Args:
128    build_data: A dictionary with build data, loaded from buildbot JSON API.
129
130  Returns:
131    True if revision is failed build, otherwise False.
132  """
133  if (build_data.get('results') in FAILED and
134      not _IsBuildSuccessful(build_data)):
135    return True
136  return False
137
138
139def _IsBuildSuccessful(build_data):
140  """Checks whether the build succeeded on buildbot.
141
142  We treat build as successful if the package_build step is completed without
143  any error i.e., when results attribute of the this step has value 0 or 1
144  in its first element.
145
146  Args:
147    build_data: A dictionary with build data, loaded from buildbot JSON API.
148
149  Returns:
150    True if revision is successfully build, otherwise False.
151  """
152  if build_data.get('steps'):
153    for item in build_data.get('steps'):
154      # The 'results' attribute of each step consists of two elements,
155      # results[0]: This represents the status of build step.
156      # See: http://docs.buildbot.net/current/developer/results.html
157      # results[1]: List of items, contains text if step fails, otherwise empty.
158      if (item.get('name') == 'package_build' and
159          item.get('isFinished') and
160          item.get('results')[0] in OK):
161        return True
162  return False
163
164
165def _FetchBuilderData(builder_url):
166  """Fetches JSON data for the all the builds from the tryserver.
167
168  Args:
169    builder_url: A tryserver URL to fetch builds information.
170
171  Returns:
172    A dictionary with information of all build on the tryserver.
173  """
174  data = None
175  try:
176    url = urllib2.urlopen(builder_url)
177  except urllib2.URLError as e:
178    print ('urllib2.urlopen error %s, waterfall status page down.[%s]' % (
179        builder_url, str(e)))
180    return None
181  if url is not None:
182    try:
183      data = url.read()
184    except IOError as e:
185      print 'urllib2 file object read error %s, [%s].' % (builder_url, str(e))
186  return data
187
188
189def _GetBuildData(buildbot_url):
190  """Gets build information for the given build id from the tryserver.
191
192  Args:
193    buildbot_url: A tryserver URL to fetch build information.
194
195  Returns:
196    A dictionary with build information if build exists, otherwise None.
197  """
198  builds_json = _FetchBuilderData(buildbot_url)
199  if builds_json:
200    return json.loads(builds_json)
201  return None
202
203
204def _GetBuildBotUrl(builder_host, builder_port):
205  """Gets build bot URL for fetching build info.
206
207  Bisect builder bots are hosted on tryserver.chromium.perf, though we cannot
208  access this tryserver using host and port number directly, so we use another
209  tryserver URL for the perf tryserver.
210
211  Args:
212    builder_host: Hostname of the server where the builder is hosted.
213    builder_port: Port number of ther server where the builder is hosted.
214
215  Returns:
216    URL of the buildbot as a string.
217  """
218  if (builder_host == PERF_BISECT_BUILDER_HOST and
219      builder_port == PERF_BISECT_BUILDER_PORT):
220    return PERF_TRY_SERVER_URL
221  else:
222    return 'http://%s:%s' % (builder_host, builder_port)
223
224
225def GetBuildStatus(build_num, bot_name, builder_host, builder_port):
226  """Gets build status from the buildbot status page for a given build number.
227
228  Args:
229    build_num: A build number on tryserver to determine its status.
230    bot_name: Name of the bot where the build information is scanned.
231    builder_host: Hostname of the server where the builder is hosted.
232    builder_port: Port number of the server where the builder is hosted.
233
234  Returns:
235    A pair which consists of build status (SUCCESS, FAILED or PENDING) and a
236    link to build status page on the waterfall.
237  """
238  results_url = None
239  if build_num:
240    # Get the URL for requesting JSON data with status information.
241    server_url = _GetBuildBotUrl(builder_host, builder_port)
242    buildbot_url = BUILDER_JSON_URL % {
243        'server_url': server_url,
244        'bot_name': bot_name,
245        'build_num': build_num,
246    }
247    build_data = _GetBuildData(buildbot_url)
248    if build_data:
249      # Link to build on the buildbot showing status of build steps.
250      results_url = BUILDER_HTML_URL % {
251          'server_url': server_url,
252          'bot_name': bot_name,
253          'build_num': build_num,
254      }
255      if _IsBuildFailed(build_data):
256        return (FAILED, results_url)
257
258      elif _IsBuildSuccessful(build_data):
259        return (OK, results_url)
260  return (PENDING, results_url)
261
262
263def GetBuildNumFromBuilder(build_reason, bot_name, builder_host, builder_port):
264  """Gets build number on build status page for a given 'build reason'.
265
266  This function parses the JSON data from buildbot page and collects basic
267  information about the all the builds, and then uniquely identifies the build
268  based on the 'reason' attribute in builds's JSON data.
269
270  The 'reason' attribute set is when a build request is posted, and it is used
271  to identify the build on status page.
272
273  Args:
274    build_reason: A unique build name set to build on tryserver.
275    bot_name: Name of the bot where the build information is scanned.
276    builder_host: Hostname of the server where the builder is hosted.
277    builder_port: Port number of ther server where the builder is hosted.
278
279  Returns:
280    A build number as a string if found, otherwise None.
281  """
282  # Gets the buildbot url for the given host and port.
283  server_url = _GetBuildBotUrl(builder_host, builder_port)
284  buildbot_url = BUILDER_JSON_URL % {
285      'server_url': server_url,
286      'bot_name': bot_name,
287      'build_num': '_all',
288  }
289  builds_json = _FetchBuilderData(buildbot_url)
290  if builds_json:
291    builds_data = json.loads(builds_json)
292    for current_build in builds_data:
293      if builds_data[current_build].get('reason') == build_reason:
294        return builds_data[current_build].get('number')
295  return None
296
297
298def _GetRequestParams(options):
299  """Extracts request parameters which will be passed to PostTryJob.
300
301  Args:
302    options: The options object parsed from the command line.
303
304  Returns:
305    A dictionary with parameters to pass to PostTryJob.
306  """
307  params = {
308      'user': options.user,
309      'name': options.name,
310  }
311  # Add other parameters if they're available in the options object.
312  for key in ['email', 'revision', 'root', 'bot', 'patch']:
313    option = getattr(options, key)
314    if option:
315      params[key] = option
316  return params
317
318
319def _GenParser():
320  """Returns a parser for getting command line arguments."""
321  usage = ('%prog [options]\n'
322           'Post a build request to the try server for the given revision.')
323  parser = optparse.OptionParser(usage=usage)
324  parser.add_option('-H', '--host',
325                    help='Host address of the try server (required).')
326  parser.add_option('-P', '--port', type='int',
327                    help='HTTP port of the try server (required).')
328  parser.add_option('-u', '--user', default=getpass.getuser(),
329                    dest='user',
330                    help='Owner user name [default: %default]')
331  parser.add_option('-e', '--email',
332                    default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
333                                           os.environ.get('EMAIL_ADDRESS')),
334                    help=('Email address where to send the results. Use either '
335                          'the TRYBOT_RESULTS_EMAIL_ADDRESS environment '
336                          'variable or EMAIL_ADDRESS to set the email address '
337                          'the try bots report results to [default: %default]'))
338  parser.add_option('-n', '--name',
339                    default='try_job_http',
340                    help='Descriptive name of the try job')
341  parser.add_option('-b', '--bot',
342                    help=('Only one builder per run may be specified; to post '
343                          'jobs on on multiple builders, run script for each '
344                          'builder separately.'))
345  parser.add_option('-r', '--revision',
346                    help=('Revision to use for the try job. The revision may '
347                          'be determined by the try server; see its waterfall '
348                          'for more info.'))
349  parser.add_option('--root',
350                    help=('Root to use for the patch; base subdirectory for '
351                          'patch created in a subdirectory.'))
352  parser.add_option('--patch',
353                    help='Patch information.')
354  return parser
355
356
357def Main(_):
358  """Posts a try job based on command line parameters."""
359  parser = _GenParser()
360  options, _ = parser.parse_args()
361  if not options.host or not options.port:
362    parser.print_help()
363    return 1
364  params = _GetRequestParams(options)
365  PostTryJob(options.host, options.port, params)
366
367
368if __name__ == '__main__':
369  sys.exit(Main(sys.argv))
370