buildbot_utils.py revision 097419758febae084477a01fd92d678218b578e5 
1# Copyright 2014 Google Inc. All Rights Reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for launching and accessing ChromeOS buildbots."""
5
6from __future__ import print_function
7
8import os
9import time
10import urllib2
11
12from cros_utils import command_executer
13from cros_utils import logger
14from cros_utils import buildbot_json
15
16SLEEP_TIME = 600  # 10 minutes; time between polling of buildbot.
17TIME_OUT = 28800  # Decide the build is dead or will never finish
18# after this time (8 hours).
19OK_STATUS = [  # List of result status values that are 'ok'.
20    # This was obtained from:
21    #   https://chromium.googlesource.com/chromium/tools/build/+/
22    #       master/third_party/buildbot_8_4p1/buildbot/status/results.py
23    0,  # "success"
24    1,  # "warnings"
25    6,  # "retry"
26]
27
28
29class BuildbotTimeout(Exception):
30  """Exception to throw when a buildbot operation timesout."""
31  pass
32
33
34def ParseReportLog(url, build):
35  """Scrape the trybot image name off the Reports log page.
36
37    This takes the URL for a trybot Reports Stage web page,
38    and a trybot build type, such as 'daisy-release'.  It
39    opens the web page and parses it looking for the trybot
40    artifact name (e.g. something like
41    'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
42    artifact name, if found.
43    """
44  trybot_image = ''
45  url += '/text'
46  newurl = url.replace('uberchromegw', 'chromegw')
47  webpage = urllib2.urlopen(newurl)
48  data = webpage.read()
49  lines = data.split('\n')
50  for l in lines:
51    if l.find('Artifacts') > 0 and l.find('trybot') > 0:
52      trybot_name = 'trybot-%s' % build
53      start_pos = l.find(trybot_name)
54      end_pos = l.find('@https://storage')
55      trybot_image = l[start_pos:end_pos]
56
57  return trybot_image
58
59
60def GetBuildData(buildbot_queue, build_id):
61  """Find the Reports stage web page for a trybot build.
62
63    This takes the name of a buildbot_queue, such as 'daisy-release'
64    and a build id (the build number), and uses the json buildbot api to
65    find the Reports stage web page for that build, if it exists.
66    """
67  builder = buildbot_json.Buildbot(
68      'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue]
69  build_data = builder.builds[build_id].data
70  logs = build_data['logs']
71  for l in logs:
72    fname = l[1]
73    if 'steps/Report/' in fname:
74      return fname
75
76  return ''
77
78
79def FindBuildRecordFromLog(description, log_info):
80  """Find the right build record in the build logs.
81
82    Get the first build record from build log with a reason field
83    that matches 'description'. ('description' is a special tag we
84    created when we launched the buildbot, so we could find it at this
85    point.)
86    """
87
88  current_line = 1
89  while current_line < len(log_info):
90    my_dict = {}
91    # Read all the lines from one "Build" to the next into my_dict
92    while True:
93      key = log_info[current_line].split(':')[0].strip()
94      value = log_info[current_line].split(':', 1)[1].strip()
95      my_dict[key] = value
96      current_line += 1
97      if 'Build' in key or current_line == len(log_info):
98        break
99    try:
100      # Check to see of the build record is the right one.
101      if str(description) in my_dict['reason']:
102        # We found a match; we're done.
103        return my_dict
104    except KeyError:
105      print("reason is not in dictionary: '%s'" % repr(my_dict))
106    else:
107      # Keep going.
108      continue
109
110  # We hit the bottom of the log without a match.
111  return {}
112
113
114def GetBuildInfo(file_dir, builder):
115  """Get all the build records for the trybot builds.
116
117    file_dir is the toolchain_utils directory.
118    """
119  ce = command_executer.GetCommandExecuter()
120  commands = ('{0}/cros_utils/buildbot_json.py builds '
121              'http://chromegw/i/tryserver.chromiumos/'.format(file_dir))
122
123  if builder:
124    # For release builds, get logs from the 'release' builder.
125    if builder.endswith('-release'):
126      commands += ' -b release'
127    elif builder.endswith('-gcc-toolchain'):
128      commands += ' -b gcc_toolchain'
129    elif builder.endswith('-llvm-toolchain'):
130      commands += ' -b llvm_toolchain'
131    elif builder.endswith('-toolchain'):
132      commands += ' -b etc'
133    else:
134      commands += ' -b %s' % builder
135  _, buildinfo, _ = ce.RunCommandWOutput(commands, print_to_console=False)
136  build_log = buildinfo.splitlines()
137  return build_log
138
139
140def FindArchiveImage(chromeos_root, build, build_id):
141  """Returns name of the trybot artifact for board/build_id."""
142  ce = command_executer.GetCommandExecuter()
143  command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s'
144             '/chromiumos_test_image.tar.xz' % (build, build_id))
145  _, out, _ = ce.ChrootRunCommandWOutput(
146      chromeos_root, command, print_to_console=False)
147  #
148  # If build_id is not unique, there may be multiple archive images
149  # to choose from; sort them & pick the first (newest).
150  #
151  # If there are multiple archive images found, out will look something
152  # like this:
153  #
154  # 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz
155  #  gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz'
156  #
157  out = out.rstrip('\n')
158  tmp_list = out.split('\n')
159  # After stripping the final '\n' and splitting on any other '\n', we get
160  # something like this:
161  #  tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' ,
162  #               'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ]
163  #
164  #  If we sort this in descending order, we should end up with the most
165  #  recent test image first, so that's what we do here.
166  #
167  if len(tmp_list) > 1:
168    tmp_list = sorted(tmp_list, reverse=True)
169  out = tmp_list[0]
170
171  trybot_image = ''
172  trybot_name = 'trybot-%s' % build
173  if out and out.find(trybot_name) > 0:
174    start_pos = out.find(trybot_name)
175    end_pos = out.find('/chromiumos_test_image')
176    trybot_image = out[start_pos:end_pos]
177
178  return trybot_image
179
180
181def GetTrybotImage(chromeos_root,
182                   buildbot_name,
183                   patch_list,
184                   build_tag,
185                   other_flags=[],
186                   build_toolchain=False,
187                   async=False):
188  """Launch buildbot and get resulting trybot artifact name.
189
190    This function launches a buildbot with the appropriate flags to
191    build the test ChromeOS image, with the current ToT mobile compiler.  It
192    checks every 10 minutes to see if the trybot has finished.  When the trybot
193    has finished, it parses the resulting report logs to find the trybot
194    artifact (if one was created), and returns that artifact name.
195
196    chromeos_root is the path to the ChromeOS root, needed for finding chromite
197    and launching the buildbot.
198
199    buildbot_name is the name of the buildbot queue, such as lumpy-release or
200    daisy-paladin.
201
202    patch_list a python list of the patches, if any, for the buildbot to use.
203
204    build_tag is a (unique) string to be used to look up the buildbot results
205    from among all the build records.
206    """
207  ce = command_executer.GetCommandExecuter()
208  cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
209  base_dir = os.getcwd()
210  patch_arg = ''
211  if patch_list:
212    for p in patch_list:
213      patch_arg = patch_arg + ' -g ' + repr(p)
214  toolchain_flags = ''
215  if build_toolchain:
216    toolchain_flags += '--latest-toolchain'
217  os.chdir(cbuildbot_path)
218  if other_flags:
219    optional_flags = ' '.join(other_flags)
220  else:
221    optional_flags = ''
222
223  # Launch buildbot with appropriate flags.
224  build = buildbot_name
225  description = build_tag
226  command_prefix = ''
227  if not patch_arg:
228    command_prefix = 'yes | '
229  command = ('%s ./cbuildbot --remote --nochromesdk %s'
230             ' --remote-description=%s %s %s %s' % (command_prefix,
231                                                    optional_flags, description,
232                                                    toolchain_flags, patch_arg,
233                                                    build))
234  _, out, _ = ce.RunCommandWOutput(command)
235  if 'Tryjob submitted!' not in out:
236    logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
237                                '%s' % command)
238
239  if async:
240    # Do not wait for trybot job to finish; return immediately
241    return 0
242
243  os.chdir(base_dir)
244
245  build_id = 0
246  build_status = None
247  # Wait for  buildbot to finish running (check every 10 minutes).  Wait
248  # 10 minutes before the first check to give the buildbot time to launch
249  # (so we don't start looking for build data before it's out there).
250  time.sleep(SLEEP_TIME)
251  done = False
252  pending = True
253  # pending_time is the time between when we submit the job and when the
254  # buildbot actually launches the build.  running_time is the time between
255  # when the buildbot job launches and when it finishes.  The job is
256  # considered 'pending' until we can find an entry for it in the buildbot
257  # logs.
258  pending_time = SLEEP_TIME
259  running_time = 0
260  while not done:
261    done = True
262    build_info = GetBuildInfo(base_dir, build)
263    if not build_info:
264      if pending_time > TIME_OUT:
265        logger.GetLogger().LogFatal('Unable to get build logs for target %s.' %
266                                    build)
267      else:
268        pending_message = 'Unable to find build log; job may be pending.'
269        done = False
270
271    if done:
272      data_dict = FindBuildRecordFromLog(description, build_info)
273      if not data_dict:
274        # Trybot job may be pending (not actually launched yet).
275        if pending_time > TIME_OUT:
276          logger.GetLogger().LogFatal('Unable to find build record for trybot'
277                                      ' %s.' % description)
278        else:
279          pending_message = 'Unable to find build record; job may be pending.'
280          done = False
281
282      else:
283        # Now that we have actually found the entry for the build
284        # job in the build log, we know the job is actually
285        # runnning, not pending, so we flip the 'pending' flag.  We
286        # still have to wait for the buildbot job to finish running
287        # however.
288        pending = False
289        if 'True' in data_dict['completed']:
290          build_id = data_dict['number']
291          build_status = int(data_dict['result'])
292        else:
293          done = False
294
295    if not done:
296      if pending:
297        logger.GetLogger().LogOutput(pending_message)
298        logger.GetLogger().LogOutput('Current pending time: %d minutes.' %
299                                     (pending_time / 60))
300        pending_time += SLEEP_TIME
301      else:
302        logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time /
303                                                                  60))
304        logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
305        running_time += SLEEP_TIME
306
307      time.sleep(SLEEP_TIME)
308      if running_time > TIME_OUT:
309        done = True
310
311  trybot_image = ''
312
313  if build.endswith('-toolchain'):
314    # For rotating testers, we don't care about their build_status
315    # result, because if any HWTest failed it will be non-zero.
316    trybot_image = FindArchiveImage(chromeos_root, build, build_id)
317  else:
318    # The nightly performance tests do not run HWTests, so if
319    # their build_status is non-zero, we do care.  In this case
320    # non-zero means the image itself probably did not build.
321    if build_status in OK_STATUS:
322      trybot_image = FindArchiveImage(chromeos_root, build, build_id)
323  if not trybot_image:
324    logger.GetLogger().LogError('Trybot job %s failed with status %d;'
325                                ' no trybot image generated.' %
326                                (description, build_status))
327
328  logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image)
329  logger.GetLogger().LogOutput('build_status is %d' % build_status)
330  return trybot_image
331
332
333def DoesImageExist(chromeos_root, build):
334  """Check if the image for the given build exists."""
335
336  ce = command_executer.GetCommandExecuter()
337  command = ('gsutil ls gs://chromeos-image-archive/%s'
338             '/chromiumos_test_image.tar.xz' % (build))
339  ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
340  return not ret
341
342
343def WaitForImage(chromeos_root, build):
344  """Wait for an image to be ready."""
345
346  elapsed_time = 0
347  while elapsed_time < TIME_OUT:
348    if DoesImageExist(chromeos_root, build):
349      return
350    logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
351                                 build)
352    time.sleep(SLEEP_TIME)
353    elapsed_time += SLEEP_TIME
354
355  logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
356                               (build, (TIME_OUT / 3600)))
357  raise BuildbotTimeout('Timeout while waiting for image %s' % build)
358