buildbot_utils.py revision 6c438e04d12f5dd94dd060c9ab41a61766dc313e 
1# Copyright 2014 Google Inc. All Rights Reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for launching and accessing ChromeOS buildbots."""
5
6from __future__ import print_function
7
8import os
9import time
10import urllib2
11
12from cros_utils import command_executer
13from cros_utils import logger
14from cros_utils import buildbot_json
15
16SLEEP_TIME = 600  # 10 minutes; time between polling of buildbot.
17TIME_OUT = 28800  # Decide the build is dead or will never finish
18# after this time (8 hours).
19OK_STATUS = [  # List of result status values that are 'ok'.
20    # This was obtained from:
21    #   https://chromium.googlesource.com/chromium/tools/build/+/
22    #       master/third_party/buildbot_8_4p1/buildbot/status/results.py
23    0,  # "success"
24    1,  # "warnings"
25    6,  # "retry"
26]
27
28
29class BuildbotTimeout(Exception):
30  """Exception to throw when a buildbot operation timesout."""
31  pass
32
33
34def ParseReportLog(url, build):
35  """Scrape the trybot image name off the Reports log page.
36
37    This takes the URL for a trybot Reports Stage web page,
38    and a trybot build type, such as 'daisy-release'.  It
39    opens the web page and parses it looking for the trybot
40    artifact name (e.g. something like
41    'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
42    artifact name, if found.
43    """
44  trybot_image = ''
45  url += '/text'
46  newurl = url.replace('uberchromegw', 'chromegw')
47  webpage = urllib2.urlopen(newurl)
48  data = webpage.read()
49  lines = data.split('\n')
50  for l in lines:
51    if l.find('Artifacts') > 0 and l.find('trybot') > 0:
52      trybot_name = 'trybot-%s' % build
53      start_pos = l.find(trybot_name)
54      end_pos = l.find('@https://storage')
55      trybot_image = l[start_pos:end_pos]
56
57  return trybot_image
58
59
60def GetBuildData(buildbot_queue, build_id):
61  """Find the Reports stage web page for a trybot build.
62
63    This takes the name of a buildbot_queue, such as 'daisy-release'
64    and a build id (the build number), and uses the json buildbot api to
65    find the Reports stage web page for that build, if it exists.
66    """
67  builder = buildbot_json.Buildbot(
68      'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue]
69  build_data = builder.builds[build_id].data
70  logs = build_data['logs']
71  for l in logs:
72    fname = l[1]
73    if 'steps/Report/' in fname:
74      return fname
75
76  return ''
77
78
79def FindBuildRecordFromLog(description, log_info):
80  """Find the right build record in the build logs.
81
82    Get the first build record from build log with a reason field
83    that matches 'description'. ('description' is a special tag we
84    created when we launched the buildbot, so we could find it at this
85    point.)
86    """
87
88  current_line = 1
89  while current_line < len(log_info):
90    my_dict = {}
91    # Read all the lines from one "Build" to the next into my_dict
92    while True:
93      key = log_info[current_line].split(':')[0].strip()
94      value = log_info[current_line].split(':', 1)[1].strip()
95      my_dict[key] = value
96      current_line += 1
97      if 'Build' in key or current_line == len(log_info):
98        break
99    try:
100      # Check to see of the build record is the right one.
101      if str(description) in my_dict['reason']:
102        # We found a match; we're done.
103        return my_dict
104    except KeyError:
105      print("reason is not in dictionary: '%s'" % repr(my_dict))
106    else:
107      # Keep going.
108      continue
109
110  # We hit the bottom of the log without a match.
111  return {}
112
113
114def GetBuildInfo(file_dir, builder):
115  """Get all the build records for the trybot builds.
116
117    file_dir is the toolchain_utils directory.
118    """
119  ce = command_executer.GetCommandExecuter()
120  commands = ('{0}/cros_utils/buildbot_json.py builds '
121              'http://chromegw/i/tryserver.chromiumos/'.format(file_dir))
122
123  if builder:
124    # For release builds, get logs from the 'release' builder.
125    if builder.endswith('-release'):
126      commands += ' -b release'
127    elif builder.endswith('-gcc-toolchain'):
128      commands += ' -b gcc_toolchain'
129    elif builder.endswith('-llvm-toolchain'):
130      commands += ' -b llvm_toolchain'
131    elif builder.endswith('-toolchain'):
132      commands += ' -b etc'
133    else:
134      commands += ' -b %s' % builder
135  _, buildinfo, _ = ce.RunCommandWOutput(commands, print_to_console=False)
136  build_log = buildinfo.splitlines()
137  return build_log
138
139
140def FindArchiveImage(chromeos_root, build, build_id):
141  """Returns name of the trybot artifact for board/build_id."""
142  ce = command_executer.GetCommandExecuter()
143  command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s'
144             '/chromiumos_test_image.tar.xz' % (build, build_id))
145  _, out, _ = ce.ChrootRunCommandWOutput(
146      chromeos_root, command, print_to_console=False)
147  #
148  # If build_id is not unique, there may be multiple archive images
149  # to choose from; sort them & pick the first (newest).
150  #
151  # If there are multiple archive images found, out will look something
152  # like this:
153  #
154  # 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz
155  #  gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz'
156  #
157  out = out.rstrip('\n')
158  tmp_list = out.split('\n')
159  # After stripping the final '\n' and splitting on any other '\n', we get
160  # something like this:
161  #  tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' ,
162  #               'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ]
163  #
164  #  If we sort this in descending order, we should end up with the most
165  #  recent test image first, so that's what we do here.
166  #
167  if len(tmp_list) > 1:
168    tmp_list = sorted(tmp_list, reverse=True)
169  out = tmp_list[0]
170
171  trybot_image = ''
172  trybot_name = 'trybot-%s' % build
173  if out and out.find(trybot_name) > 0:
174    start_pos = out.find(trybot_name)
175    end_pos = out.find('/chromiumos_test_image')
176    trybot_image = out[start_pos:end_pos]
177
178  return trybot_image
179
180
181def GetTrybotImage(chromeos_root,
182                   buildbot_name,
183                   patch_list,
184                   build_tag,
185                   other_flags=[],
186                   build_toolchain=False):
187  """Launch buildbot and get resulting trybot artifact name.
188
189    This function launches a buildbot with the appropriate flags to
190    build the test ChromeOS image, with the current ToT mobile compiler.  It
191    checks every 10 minutes to see if the trybot has finished.  When the trybot
192    has finished, it parses the resulting report logs to find the trybot
193    artifact (if one was created), and returns that artifact name.
194
195    chromeos_root is the path to the ChromeOS root, needed for finding chromite
196    and launching the buildbot.
197
198    buildbot_name is the name of the buildbot queue, such as lumpy-release or
199    daisy-paladin.
200
201    patch_list a python list of the patches, if any, for the buildbot to use.
202
203    build_tag is a (unique) string to be used to look up the buildbot results
204    from among all the build records.
205    """
206  ce = command_executer.GetCommandExecuter()
207  cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
208  base_dir = os.getcwd()
209  patch_arg = ''
210  if patch_list:
211    for p in patch_list:
212      patch_arg = patch_arg + ' -g ' + repr(p)
213  toolchain_flags = ''
214  if build_toolchain:
215    toolchain_flags += '--latest-toolchain'
216  os.chdir(cbuildbot_path)
217  if other_flags:
218    optional_flags = ' '.join(other_flags)
219  else:
220    optional_flags = ''
221
222  # Launch buildbot with appropriate flags.
223  build = buildbot_name
224  description = build_tag
225  command_prefix = ''
226  if not patch_arg:
227    command_prefix = 'yes | '
228  command = ('%s ./cbuildbot --remote --nochromesdk %s'
229             ' --remote-description=%s %s %s %s' % (command_prefix,
230                                                    optional_flags, description,
231                                                    toolchain_flags, patch_arg,
232                                                    build))
233  _, out, _ = ce.RunCommandWOutput(command)
234  if 'Tryjob submitted!' not in out:
235    logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
236                                '%s' % command)
237  os.chdir(base_dir)
238
239  build_id = 0
240  build_status = None
241  # Wait for  buildbot to finish running (check every 10 minutes).  Wait
242  # 10 minutes before the first check to give the buildbot time to launch
243  # (so we don't start looking for build data before it's out there).
244  time.sleep(SLEEP_TIME)
245  done = False
246  pending = True
247  # pending_time is the time between when we submit the job and when the
248  # buildbot actually launches the build.  running_time is the time between
249  # when the buildbot job launches and when it finishes.  The job is
250  # considered 'pending' until we can find an entry for it in the buildbot
251  # logs.
252  pending_time = SLEEP_TIME
253  running_time = 0
254  while not done:
255    done = True
256    build_info = GetBuildInfo(base_dir, build)
257    if not build_info:
258      if pending_time > TIME_OUT:
259        logger.GetLogger().LogFatal('Unable to get build logs for target %s.' %
260                                    build)
261      else:
262        pending_message = 'Unable to find build log; job may be pending.'
263        done = False
264
265    if done:
266      data_dict = FindBuildRecordFromLog(description, build_info)
267      if not data_dict:
268        # Trybot job may be pending (not actually launched yet).
269        if pending_time > TIME_OUT:
270          logger.GetLogger().LogFatal('Unable to find build record for trybot'
271                                      ' %s.' % description)
272        else:
273          pending_message = 'Unable to find build record; job may be pending.'
274          done = False
275
276      else:
277        # Now that we have actually found the entry for the build
278        # job in the build log, we know the job is actually
279        # runnning, not pending, so we flip the 'pending' flag.  We
280        # still have to wait for the buildbot job to finish running
281        # however.
282        pending = False
283        if 'True' in data_dict['completed']:
284          build_id = data_dict['number']
285          build_status = int(data_dict['result'])
286        else:
287          done = False
288
289    if not done:
290      if pending:
291        logger.GetLogger().LogOutput(pending_message)
292        logger.GetLogger().LogOutput('Current pending time: %d minutes.' %
293                                     (pending_time / 60))
294        pending_time += SLEEP_TIME
295      else:
296        logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time /
297                                                                  60))
298        logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
299        running_time += SLEEP_TIME
300
301      time.sleep(SLEEP_TIME)
302      if running_time > TIME_OUT:
303        done = True
304
305  trybot_image = ''
306
307  if build.endswith('-toolchain'):
308    # For rotating testers, we don't care about their build_status
309    # result, because if any HWTest failed it will be non-zero.
310    trybot_image = FindArchiveImage(chromeos_root, build, build_id)
311  else:
312    # The nightly performance tests do not run HWTests, so if
313    # their build_status is non-zero, we do care.  In this case
314    # non-zero means the image itself probably did not build.
315    if build_status in OK_STATUS:
316      trybot_image = FindArchiveImage(chromeos_root, build, build_id)
317  if not trybot_image:
318    logger.GetLogger().LogError('Trybot job %s failed with status %d;'
319                                ' no trybot image generated.' %
320                                (description, build_status))
321
322  logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image)
323  logger.GetLogger().LogOutput('build_status is %d' % build_status)
324  return trybot_image
325
326
327def DoesImageExist(chromeos_root, build):
328  """Check if the image for the given build exists."""
329
330  ce = command_executer.GetCommandExecuter()
331  command = ('gsutil ls gs://chromeos-image-archive/%s'
332             '/chromiumos_test_image.tar.xz' % (build))
333  ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
334  return not ret
335
336
337def WaitForImage(chromeos_root, build):
338  """Wait for an image to be ready."""
339
340  elapsed_time = 0
341  while elapsed_time < TIME_OUT:
342    if DoesImageExist(chromeos_root, build):
343      return
344    logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
345                                 build)
346    time.sleep(SLEEP_TIME)
347    elapsed_time += SLEEP_TIME
348
349  logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
350                               (build, (TIME_OUT / 3600)))
351  raise BuildbotTimeout('Timeout while waiting for image %s' % build)
352