remote_gcc_build.py revision 9099a788cd7124024559c064e425ed9caef6e0ac 
1#!/usr/bin/python2
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Script to use remote try-bot build image with local gcc."""
7
8from __future__ import print_function
9
10import argparse
11import glob
12import os
13import re
14import shutil
15import socket
16import sys
17import tempfile
18import time
19
20from cros_utils import command_executer
21from cros_utils import logger
22from cros_utils import manifest_versions
23from cros_utils import misc
24
25BRANCH = 'the_actual_branch_used_in_this_script'
26TMP_BRANCH = 'tmp_branch'
27SLEEP_TIME = 600
28
29# pylint: disable=anomalous-backslash-in-string
30
31def GetPatchNum(output):
32  lines = output.splitlines()
33  line = [l for l in lines if 'googlesource' in l][0]
34  patch_num = re.findall(r'\d+', line)[0]
35  if 'chrome-internal' in line:
36    patch_num = '*' + patch_num
37  return str(patch_num)
38
39
40def GetPatchString(patch):
41  if patch:
42    return '+'.join(patch)
43  return 'NO_PATCH'
44
45
46def FindVersionForToolchain(branch, chromeos_root):
47  """Find the version number in artifacts link in the tryserver email."""
48  # For example: input:  toolchain-3701.42.B
49  #              output: R26-3701.42.1
50  digits = branch.split('-')[1].split('B')[0]
51  manifest_dir = os.path.join(chromeos_root, 'manifest-internal')
52  os.chdir(manifest_dir)
53  major_version = digits.split('.')[0]
54  ce = command_executer.GetCommandExecuter()
55  command = 'repo sync . && git branch -a | grep {0}'.format(major_version)
56  _, branches, _ = ce.RunCommandWOutput(command, print_to_console=False)
57  m = re.search(r'(R\d+)', branches)
58  if not m:
59    logger.GetLogger().LogFatal('Cannot find version for branch {0}'
60                                .format(branch))
61  version = m.group(0) + '-' + digits + '1'
62  return version
63
64
65def FindBuildId(description):
66  """Find the build id of the build at trybot server."""
67  running_time = 0
68  while True:
69    (result, number) = FindBuildIdFromLog(description)
70    if result >= 0:
71      return (result, number)
72    logger.GetLogger().LogOutput('{0} minutes passed.'
73                                 .format(running_time / 60))
74    logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
75    time.sleep(SLEEP_TIME)
76    running_time += SLEEP_TIME
77
78
79def FindBuildIdFromLog(description):
80  """Get the build id from build log."""
81  # returns tuple (result, buildid)
82  # result == 0, buildid > 0, the build was successful and we have a build id
83  # result > 0, buildid > 0,  the whole build failed for some reason but we
84  #                           do have a build id.
85  # result == -1, buildid == -1, we have not found a finished build for this
86  #                              description yet
87
88  file_dir = os.path.dirname(os.path.realpath(__file__))
89  commands = ('{0}/cros_utils/buildbot_json.py builds '
90              'http://chromegw/p/tryserver.chromiumos/'.format(file_dir))
91  ce = command_executer.GetCommandExecuter()
92  _, buildinfo, _ = ce.RunCommandWOutput(commands, print_to_console=False)
93
94  my_info = buildinfo.splitlines()
95  current_line = 1
96  running_job = False
97  result = -1
98
99  # result == 0, we have a successful build
100  # result > 0, we have a failed build but build id may be valid
101  # result == -1, we have not found a finished build for this description
102  while current_line < len(my_info):
103    my_dict = {}
104    while True:
105      key = my_info[current_line].split(':')[0].strip()
106      value = my_info[current_line].split(':', 1)[1].strip()
107      my_dict[key] = value
108      current_line += 1
109      if 'Build' in key or current_line == len(my_info):
110        break
111    if ('True' not in my_dict['completed'] and
112        str(description) in my_dict['reason']):
113      running_job = True
114    if ('True' not in my_dict['completed'] or
115        str(description) not in my_dict['reason']):
116      continue
117    result = int(my_dict['result'])
118    build_id = int(my_dict['number'])
119    if result == 0:
120      return (result, build_id)
121    else:
122      # Found a finished failed build.
123      # Keep searching to find a successful one
124      pass
125
126  if result > 0 and not running_job:
127    return (result, build_id)
128  return (-1, -1)
129
130
131def DownloadImage(target, index, dest, version):
132  """Download artifacts from cloud."""
133  if not os.path.exists(dest):
134    os.makedirs(dest)
135
136  rversion = manifest_versions.RFormatCrosVersion(version)
137  print(str(rversion))
138  #  ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}"
139  #            .format(target, rversion, index))
140  ls_cmd = ('gsutil ls gs://chromeos-image-archive/trybot-{0}/*-b{2}'
141            .format(target, index))
142
143  download_cmd = ('$(which gsutil) cp {0} {1}'.format('{0}', dest))
144  ce = command_executer.GetCommandExecuter()
145
146  _, out, _ = ce.RunCommandWOutput(ls_cmd, print_to_console=True)
147  lines = out.splitlines()
148  download_files = ['autotest.tar', 'chromeos-chrome', 'chromiumos_test_image',
149                    'debug.tgz', 'sysroot_chromeos-base_chromeos-chrome.tar.xz']
150  for line in lines:
151    if any([e in line for e in download_files]):
152      cmd = download_cmd.format(line)
153      if ce.RunCommand(cmd):
154        logger.GetLogger().LogFatal('Command {0} failed, existing...'
155                                    .format(cmd))
156
157
158def UnpackImage(dest):
159  """Unpack the image, the chroot build dir."""
160  chrome_tbz2 = glob.glob(dest + '/*.tbz2')[0]
161  commands = ('tar xJf {0}/sysroot_chromeos-base_chromeos-chrome.tar.xz '
162              '-C {0} &&'
163              'tar xjf {1} -C {0} &&'
164              'tar xzf {0}/debug.tgz  -C {0}/usr/lib/ &&'
165              'tar xf {0}/autotest.tar -C {0}/usr/local/ &&'
166              'tar xJf {0}/chromiumos_test_image.tar.xz -C {0}'
167              .format(dest, chrome_tbz2))
168  ce = command_executer.GetCommandExecuter()
169  return ce.RunCommand(commands)
170
171
172def RemoveOldBranch():
173  """Remove the branch with name BRANCH."""
174  ce = command_executer.GetCommandExecuter()
175  command = 'git rev-parse --abbrev-ref HEAD'
176  _, out, _ = ce.RunCommandWOutput(command)
177  if BRANCH in out:
178    command = 'git checkout -B {0}'.format(TMP_BRANCH)
179    ce.RunCommand(command)
180  command = "git commit -m 'nouse'"
181  ce.RunCommand(command)
182  command = 'git branch -D {0}'.format(BRANCH)
183  ce.RunCommand(command)
184
185
186def UploadManifest(manifest, chromeos_root, branch='master'):
187  """Copy the manifest to $chromeos_root/manifest-internal and upload."""
188  chromeos_root = misc.CanonicalizePath(chromeos_root)
189  manifest_dir = os.path.join(chromeos_root, 'manifest-internal')
190  os.chdir(manifest_dir)
191  ce = command_executer.GetCommandExecuter()
192
193  RemoveOldBranch()
194
195  if branch != 'master':
196    branch = '{0}'.format(branch)
197  command = 'git checkout -b {0} -t cros-internal/{1}'.format(BRANCH, branch)
198  ret = ce.RunCommand(command)
199  if ret:
200    raise RuntimeError('Command {0} failed'.format(command))
201
202  # We remove the default.xml, which is the symbolic link of full.xml.
203  # After that, we copy our xml file to default.xml.
204  # We did this because the full.xml might be updated during the
205  # run of the script.
206  os.remove(os.path.join(manifest_dir, 'default.xml'))
207  shutil.copyfile(manifest, os.path.join(manifest_dir, 'default.xml'))
208  return UploadPatch(manifest)
209
210
211def GetManifestPatch(manifests, version, chromeos_root, branch='master'):
212  """Return a gerrit patch number given a version of manifest file."""
213  temp_dir = tempfile.mkdtemp()
214  to_file = os.path.join(temp_dir, 'default.xml')
215  manifests.GetManifest(version, to_file)
216  return UploadManifest(to_file, chromeos_root, branch)
217
218
219def UploadPatch(source):
220  """Up load patch to gerrit, return patch number."""
221  commands = ('git add -A . &&'
222              "git commit -m 'test' -m 'BUG=None' -m 'TEST=None' "
223              "-m 'hostname={0}' -m 'source={1}'"
224              .format(socket.gethostname(), source))
225  ce = command_executer.GetCommandExecuter()
226  ce.RunCommand(commands)
227
228  commands = ('yes | repo upload .   --cbr --no-verify')
229  _, _, err = ce.RunCommandWOutput(commands)
230  return GetPatchNum(err)
231
232
233def ReplaceSysroot(chromeos_root, dest_dir, target):
234  """Copy unpacked sysroot and image to chromeos_root."""
235  ce = command_executer.GetCommandExecuter()
236  # get the board name from "board-release". board may contain "-"
237  board = target.rsplit('-', 1)[0]
238  board_dir = os.path.join(chromeos_root, 'chroot', 'build', board)
239  command = 'sudo rm -rf {0}'.format(board_dir)
240  ce.RunCommand(command)
241
242  command = 'sudo mv {0} {1}'.format(dest_dir, board_dir)
243  ce.RunCommand(command)
244
245  image_dir = os.path.join(chromeos_root, 'src', 'build', 'images', board,
246                           'latest')
247  command = 'rm -rf {0} && mkdir -p {0}'.format(image_dir)
248  ce.RunCommand(command)
249
250  command = 'mv {0}/chromiumos_test_image.bin {1}'.format(board_dir, image_dir)
251  return ce.RunCommand(command)
252
253
254def GccBranchForToolchain(branch):
255  if branch == 'toolchain-3428.65.B':
256    return 'release-R25-3428.B'
257  else:
258    return None
259
260
261def GetGccBranch(branch):
262  """Get the remote branch name from branch or version."""
263  ce = command_executer.GetCommandExecuter()
264  command = 'git branch -a | grep {0}'.format(branch)
265  _, out, _ = ce.RunCommandWOutput(command)
266  if not out:
267    release_num = re.match(r'.*(R\d+)-*', branch)
268    if release_num:
269      release_num = release_num.group(0)
270      command = 'git branch -a | grep {0}'.format(release_num)
271      _, out, _ = ce.RunCommandWOutput(command)
272      if not out:
273        GccBranchForToolchain(branch)
274  if not out:
275    out = 'remotes/cros/master'
276  new_branch = out.splitlines()[0]
277  return new_branch
278
279
280def UploadGccPatch(chromeos_root, gcc_dir, branch):
281  """Upload local gcc to gerrit and get the CL number."""
282  ce = command_executer.GetCommandExecuter()
283  gcc_dir = misc.CanonicalizePath(gcc_dir)
284  gcc_path = os.path.join(chromeos_root, 'src/third_party/gcc')
285  assert os.path.isdir(gcc_path), ('{0} is not a valid chromeos root'
286                                   .format(chromeos_root))
287  assert os.path.isdir(gcc_dir), ('{0} is not a valid dir for gcc'
288                                  'source'.format(gcc_dir))
289  os.chdir(gcc_path)
290  RemoveOldBranch()
291  if not branch:
292    branch = 'master'
293  branch = GetGccBranch(branch)
294  command = ('git checkout -b {0} -t {1} && ' 'rm -rf *'.format(BRANCH, branch))
295  ce.RunCommand(command, print_to_console=False)
296
297  command = ("rsync -az --exclude='*.svn' --exclude='*.git'"
298             ' {0}/ .'.format(gcc_dir))
299  ce.RunCommand(command)
300  return UploadPatch(gcc_dir)
301
302
303def RunRemote(chromeos_root, branch, patches, is_local, target, chrome_version,
304              dest_dir):
305  """The actual running commands."""
306  ce = command_executer.GetCommandExecuter()
307
308  if is_local:
309    local_flag = '--local -r {0}'.format(dest_dir)
310  else:
311    local_flag = '--remote'
312  patch = ''
313  for p in patches:
314    patch += ' -g {0}'.format(p)
315  cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
316  os.chdir(cbuildbot_path)
317  branch_flag = ''
318  if branch != 'master':
319    branch_flag = ' -b {0}'.format(branch)
320  chrome_version_flag = ''
321  if chrome_version:
322    chrome_version_flag = ' --chrome_version={0}'.format(chrome_version)
323  description = '{0}_{1}_{2}'.format(branch, GetPatchString(patches), target)
324  command = ('yes | ./cbuildbot {0} {1} {2} {3} {4} {5}'
325             ' --remote-description={6}'
326             ' --chrome_rev=tot'.format(patch, branch_flag, chrome_version,
327                                        local_flag, chrome_version_flag, target,
328                                        description))
329  ce.RunCommand(command)
330
331  return description
332
333
334def Main(argv):
335  """The main function."""
336  # Common initializations
337  parser = argparse.ArgumentParser()
338  parser.add_argument('-c',
339                      '--chromeos_root',
340                      required=True,
341                      dest='chromeos_root',
342                      help='The chromeos_root')
343  parser.add_argument('-g',
344                      '--gcc_dir',
345                      default='',
346                      dest='gcc_dir',
347                      help='The gcc dir')
348  parser.add_argument('-t',
349                      '--target',
350                      required=True,
351                      dest='target',
352                      help=('The target to be build, the list is at'
353                            ' $(chromeos_root)/chromite/buildbot/cbuildbot'
354                            ' --list -all'))
355  parser.add_argument('-l', '--local', action='store_true')
356  parser.add_argument('-d',
357                      '--dest_dir',
358                      dest='dest_dir',
359                      help=('The dir to build the whole chromeos if'
360                            ' --local is set'))
361  parser.add_argument('--chrome_version',
362                      dest='chrome_version',
363                      default='',
364                      help='The chrome version to use. '
365                      'Default it will use the latest one.')
366  parser.add_argument('--chromeos_version',
367                      dest='chromeos_version',
368                      default='',
369                      help=('The chromeos version to use.'
370                            '(1) A release version in the format: '
371                            "'\d+\.\d+\.\d+\.\d+.*'"
372                            "(2) 'latest_lkgm' for the latest lkgm version"))
373  parser.add_argument('-r',
374                      '--replace_sysroot',
375                      action='store_true',
376                      help=('Whether or not to replace the build/$board dir'
377                            'under the chroot of chromeos_root and copy '
378                            'the image to src/build/image/$board/latest.'
379                            ' Default is False'))
380  parser.add_argument('-b',
381                      '--branch',
382                      dest='branch',
383                      default='',
384                      help=('The branch to run trybot, default is None'))
385  parser.add_argument('-p',
386                      '--patch',
387                      dest='patch',
388                      default='',
389                      help=('The patches to be applied, the patches numbers '
390                            "be seperated by ','"))
391
392  script_dir = os.path.dirname(os.path.realpath(__file__))
393
394  args = parser.parse_args(argv[1:])
395  target = args.target
396  if args.patch:
397    patch = args.patch.split(',')
398  else:
399    patch = []
400  chromeos_root = misc.CanonicalizePath(args.chromeos_root)
401  if args.chromeos_version and args.branch:
402    raise RuntimeError('You can not set chromeos_version and branch at the '
403                      'same time.')
404
405  manifests = None
406  if args.branch:
407    chromeos_version = ''
408    branch = args.branch
409  else:
410    chromeos_version = args.chromeos_version
411    manifests = manifest_versions.ManifestVersions()
412    if chromeos_version == 'latest_lkgm':
413      chromeos_version = manifests.TimeToVersion(time.mktime(time.gmtime()))
414      logger.GetLogger().LogOutput('found version %s for latest LKGM' %
415                                   (chromeos_version))
416    # TODO: this script currently does not handle the case where the version
417    # is not in the "master" branch
418    branch = 'master'
419
420  if chromeos_version:
421    manifest_patch = GetManifestPatch(manifests, chromeos_version,
422                                      chromeos_root)
423    patch.append(manifest_patch)
424  if args.gcc_dir:
425    # TODO: everytime we invoke this script we are getting a different
426    # patch for GCC even if GCC has not changed. The description should
427    # be based on the MD5 of the GCC patch contents.
428    patch.append(UploadGccPatch(chromeos_root, args.gcc_dir, branch))
429  description = RunRemote(chromeos_root, branch, patch, args.local, target,
430                          args.chrome_version, args.dest_dir)
431  if args.local or not args.dest_dir:
432    # TODO: We are not checktng the result of cbuild_bot in here!
433    return 0
434
435  # return value:
436  # 0 => build bot was successful and image was put where requested
437  # 1 => Build bot FAILED but image was put where requested
438  # 2 => Build bot failed or BUild bot was successful but and image was
439  #      not generated or could not be put where expected
440
441  os.chdir(script_dir)
442  dest_dir = misc.CanonicalizePath(args.dest_dir)
443  (bot_result, build_id) = FindBuildId(description)
444  if bot_result > 0 and build_id > 0:
445    logger.GetLogger().LogError('Remote trybot failed but image was generated')
446    bot_result = 1
447  elif bot_result > 0:
448    logger.GetLogger().LogError('Remote trybot failed. No image was generated')
449    return 2
450  if 'toolchain' in branch:
451    chromeos_version = FindVersionForToolchain(branch, chromeos_root)
452    assert not manifest_versions.IsRFormatCrosVersion(chromeos_version)
453  DownloadImage(target, build_id, dest_dir, chromeos_version)
454  ret = UnpackImage(dest_dir)
455  if ret != 0:
456    return 2
457  # todo: return a more inteligent return value
458  if not args.replace_sysroot:
459    return bot_result
460
461  ret = ReplaceSysroot(chromeos_root, args.dest_dir, target)
462  if ret != 0:
463    return 2
464
465  # got an image and we were successful in placing it where requested
466  return bot_result
467
468
469if __name__ == '__main__':
470  retval = Main(sys.argv)
471  sys.exit(retval)
472