remote_gcc_build.py revision 6845e5f094d67d45a6908e2cfdfed25476b8e343 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Script to use remote try-bot build image with local gcc."""
8
9import argparse
10import glob
11import os
12import re
13import shutil
14import socket
15import sys
16import tempfile
17import time
18
19from utils import command_executer
20from utils import logger
21from utils import misc
22
23BRANCH = "the_actual_branch_used_in_this_script"
24TMP_BRANCH = "tmp_branch"
25SLEEP_TIME = 600
26
27
28def GetPatchNum(output):
29  lines = output.splitlines()
30  line = [l for l in lines if "gerrit" in l][0]
31  patch_num = re.findall(r"\d+", line)[0]
32  if "gerrit-int" in line:
33    patch_num = "*" + patch_num
34  return str(patch_num)
35
36
37def GetPatchString(patch):
38  if patch:
39    return "+".join(patch)
40  return "NO_PATCH"
41
42
43def FindVersionForToolchain(branch, chromeos_root):
44  """Find the version number in artifacts link in the tryserver email."""
45  # For example: input:  toolchain-3701.42.B
46  #              output: R26-3701.42.1
47  digits = branch.split("-")[1].split("B")[0]
48  manifest_dir = os.path.join(chromeos_root, "manifest-internal")
49  os.chdir(manifest_dir)
50  major_version = digits.split(".")[0]
51  ce = command_executer.GetCommandExecuter()
52  command = "repo sync . && git branch -a | grep {0}".format(major_version)
53  _, branches, _ = ce.RunCommand(command, return_output=True,
54                                 print_to_console=False)
55  m = re.search(r"(R\d+)", branches)
56  if not m:
57    logger.GetLogger().LogFatal("Cannot find version for branch {0}"
58                                .format(branch))
59  version = m.group(0)+"-"+digits+"1"
60  return version
61
62
63def FindResultIndex(reason):
64  """Find the build id of the build at trybot server."""
65  running_time = 0
66  while True:
67    num = GetBuildNumber(reason)
68    if num >= 0:
69      return num
70    logger.GetLogger().LogOutput("{0} minutes passed."
71                                 .format(running_time / 60))
72    logger.GetLogger().LogOutput("Sleeping {0} seconds.".format(SLEEP_TIME))
73    time.sleep(SLEEP_TIME)
74    running_time += SLEEP_TIME
75
76
77def GetBuildNumber(reason):
78  """Get the build num from build log."""
79  # returns 0 if only failed build found.
80  # returns -1 if no finished build found or one build is running.
81  # returns build number is successful build found.
82
83  file_dir = os.path.dirname(os.path.realpath(__file__))
84  commands = ("{0}/utils/buildbot_json.py builds "
85              "http://chromegw/p/tryserver.chromiumos/"
86              .format(file_dir))
87  ce = command_executer.GetCommandExecuter()
88  _, buildinfo, _ = ce.RunCommand(commands, return_output=True,
89                                  print_to_console=False)
90
91  my_info = buildinfo.splitlines()
92  current_line = 1
93  running_job = False
94  number = -1
95  # number > 0, we have a successful build.
96  # number = 0, we have a failed build.
97  # number = -1, we do not have a finished build for this description.
98  while current_line < len(my_info):
99    my_dict = {}
100    while True:
101      key = my_info[current_line].split(":")[0].strip()
102      value = my_info[current_line].split(":", 1)[1].strip()
103      my_dict[key] = value
104      current_line += 1
105      if "Build" in key or current_line == len(my_info):
106        break
107    if ("True" not in my_dict["completed"] and
108        str(reason) in my_dict["reason"]):
109      running_job = True
110    if ("True" not in my_dict["completed"] or
111        str(reason) not in my_dict["reason"]):
112      continue
113    if my_dict["result"] == "0":
114      number = int(my_dict["number"])
115      return number
116    else:
117     # Record found a finished failed build.
118     # Keep searching to find a successful one.
119      number = 0
120  if number == 0 and not running_job:
121    return 0
122  return -1
123
124
125def DownloadImage(target, index, dest, version):
126  """Download artifacts from cloud."""
127  match = re.search(r"(.*-\d+\.\d+\.\d+)", version)
128  version = match.group(0)
129  if not os.path.exists(dest):
130    os.makedirs(dest)
131
132  ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}"
133            .format(target, version, index))
134
135  download_cmd = ("$(which gsutil) cp {0} {1}".format("{0}", dest))
136  ce = command_executer.GetCommandExecuter()
137
138  _, out, _ = ce.RunCommand(ls_cmd, return_output=True, print_to_console=True)
139  lines = out.splitlines()
140  download_files = ["autotest.tar", "chromeos-chrome",
141                    "chromiumos_test_image", "debug.tgz",
142                    "sysroot_chromeos-base_chromeos-chrome.tar.xz"
143                   ]
144  for line in lines:
145    if any([e in line for e in download_files]):
146      cmd = download_cmd.format(line)
147      if ce.RunCommand(cmd):
148        logger.GetLogger().LogFatal("Command {0} failed, existing..."
149                                    .format(cmd))
150
151
152def UnpackImage(dest):
153  """Unpack the image, the chroot build dir."""
154  chrome_tbz2 = glob.glob(dest+"/*.tbz2")[0]
155  commands = ("tar xJf {0}/sysroot_chromeos-base_chromeos-chrome.tar.xz "
156              "-C {0} &&"
157              "tar xjf {1} -C {0} &&"
158              "tar xzf {0}/debug.tgz  -C {0}/usr/lib/ &&"
159              "tar xf {0}/autotest.tar -C {0}/usr/local/ &&"
160              "tar xJf {0}/chromiumos_test_image.tar.xz -C {0}"
161              .format(dest, chrome_tbz2))
162  ce = command_executer.GetCommandExecuter()
163  return ce.RunCommand(commands)
164
165
166def GetManifest(version, to_file):
167  """Get the manifest file from a given chromeos-internal version."""
168  temp_dir = tempfile.mkdtemp()
169  version = version.split("-", 1)[1]
170  os.chdir(temp_dir)
171  command = ("git clone "
172             "ssh://gerrit-int.chromium.org:29419/"
173             "chromeos/manifest-versions.git")
174  ce = command_executer.GetCommandExecuter()
175  ce.RunCommand(command)
176  files = [os.path.join(r, f)
177           for r, _, fs in os.walk(".")
178           for f in fs if version in f]
179  if files:
180    command = "cp {0} {1} && rm -rf {2}".format(files[0], to_file, temp_dir)
181    ret = ce.RunCommand(command)
182    if ret:
183      raise Exception("Cannot copy manifest to {0}".format(to_file))
184  else:
185    command = "rm -rf {0}".format(temp_dir)
186    ce.RunCommand(command)
187    raise Exception("Version {0} is not available.".format(version))
188
189
190def RemoveOldBranch():
191  """Remove the branch with name BRANCH."""
192  ce = command_executer.GetCommandExecuter()
193  command = "git rev-parse --abbrev-ref HEAD"
194  _, out, _ = ce.RunCommand(command, return_output=True)
195  if BRANCH in out:
196    command = "git checkout -B {0}".format(TMP_BRANCH)
197    ce.RunCommand(command)
198  command = "git commit -m 'nouse'"
199  ce.RunCommand(command)
200  command = "git branch -D {0}".format(BRANCH)
201  ce.RunCommand(command)
202
203
204def UploadManifest(manifest, chromeos_root, branch="master"):
205  """Copy the manifest to $chromeos_root/manifest-internal and upload."""
206  chromeos_root = misc.CanonicalizePath(chromeos_root)
207  manifest_dir = os.path.join(chromeos_root, "manifest-internal")
208  os.chdir(manifest_dir)
209  ce = command_executer.GetCommandExecuter()
210
211  RemoveOldBranch()
212
213  if branch != "master":
214    branch = "{0}".format(branch)
215  command = "git checkout -b {0} -t cros-internal/{1}".format(BRANCH, branch)
216  ret = ce.RunCommand(command)
217  if ret:
218    raise Exception("Command {0} failed".format(command))
219
220  # We remove the default.xml, which is the symbolic link of full.xml.
221  # After that, we copy our xml file to default.xml.
222  # We did this because the full.xml might be updated during the
223  # run of the script.
224  os.remove(os.path.join(manifest_dir, "default.xml"))
225  shutil.copyfile(manifest, os.path.join(manifest_dir, "default.xml"))
226  return UploadPatch(manifest)
227
228
229def GetManifestPatch(version, chromeos_root, branch="master"):
230  """Return a gerrit patch number given a version of manifest file."""
231  temp_dir = tempfile.mkdtemp()
232  to_file = os.path.join(temp_dir, "default.xml")
233  GetManifest(version, to_file)
234  return UploadManifest(to_file, chromeos_root, branch)
235
236
237def UploadPatch(source):
238  """Up load patch to gerrit, return patch number."""
239  commands = ("git add -A . &&"
240              "git commit -m 'test' -m 'BUG=None' -m 'TEST=None' "
241              "-m 'hostname={0}' -m 'source={1}'"
242              .format(socket.gethostname(), source))
243  ce = command_executer.GetCommandExecuter()
244  ce.RunCommand(commands)
245
246  commands = ("yes | repo upload .   --cbr --no-verify")
247  _, _, err = ce.RunCommand(commands, return_output=True)
248  return GetPatchNum(err)
249
250
251def ReplaceSysroot(chromeos_root, dest_dir, target):
252  """Copy unpacked sysroot and image to chromeos_root."""
253  ce = command_executer.GetCommandExecuter()
254  # get the board name from "board-release". board may contain "-"
255  board = target.rsplit("-", 1)[0]
256  board_dir = os.path.join(chromeos_root, "chroot", "build", board)
257  command = "sudo rm -rf {0}".format(board_dir)
258  ce.RunCommand(command)
259
260  command = "sudo mv {0} {1}".format(dest_dir, board_dir)
261  ce.RunCommand(command)
262
263  image_dir = os.path.join(chromeos_root, "src", "build", "images",
264                           board, "latest")
265  command = "rm -rf {0} && mkdir -p {0}".format(image_dir)
266  ce.RunCommand(command)
267
268  command = "mv {0}/chromiumos_test_image.bin {1}".format(board_dir, image_dir)
269  return ce.RunCommand(command)
270
271
272def GccBranchForToolchain(branch):
273  if branch == "toolchain-3428.65.B":
274    return "release-R25-3428.B"
275  else:
276    return None
277
278
279def GetGccBranch(branch):
280  """Get the remote branch name from branch or version."""
281  ce = command_executer.GetCommandExecuter()
282  command = "git branch -a | grep {0}".format(branch)
283  _, out, _ = ce.RunCommand(command, return_output=True)
284  if not out:
285    release_num = re.match(r".*(R\d+)-*", branch)
286    if release_num:
287      release_num = release_num.group(0)
288      command = "git branch -a | grep {0}".format(release_num)
289      _, out, _ = ce.RunCommand(command, return_output=True)
290      if not out:
291        GccBranchForToolchain(branch)
292  if not out:
293    out = "remotes/cros/master"
294  new_branch = out.splitlines()[0]
295  return new_branch
296
297
298def UploadGccPatch(chromeos_root, gcc_dir, branch):
299  """Upload local gcc to gerrit and get the CL number."""
300  ce = command_executer.GetCommandExecuter()
301  gcc_dir = misc.CanonicalizePath(gcc_dir)
302  gcc_path = os.path.join(chromeos_root, "src/third_party/gcc")
303  assert os.path.isdir(gcc_path), ("{0} is not a valid chromeos root"
304                                   .format(chromeos_root))
305  assert os.path.isdir(gcc_dir), ("{0} is not a valid dir for gcc"
306                                  "source".format(gcc_dir))
307  os.chdir(gcc_path)
308  RemoveOldBranch()
309  if not branch:
310    branch = "master"
311  branch = GetGccBranch(branch)
312  command = ("git checkout -b {0} -t {1} && "
313             "rm -rf *".format(BRANCH, branch))
314  ce.RunCommand(command, print_to_console=False)
315
316  command = ("rsync -az --exclude='*.svn' --exclude='*.git'"
317             " {0}/ .".format(gcc_dir))
318  ce.RunCommand(command)
319  return UploadPatch(gcc_dir)
320
321
322def RunRemote(chromeos_root, branch, patches, is_local,
323              target, chrome_version, dest_dir):
324  """The actual running commands."""
325  ce = command_executer.GetCommandExecuter()
326
327  if is_local:
328    local_flag = "--local -r {0}".format(dest_dir)
329  else:
330    local_flag = "--remote"
331  patch = ""
332  for p in patches:
333    patch += " -g {0}".format(p)
334  cbuildbot_path = os.path.join(chromeos_root, "chromite/buildbot")
335  os.chdir(cbuildbot_path)
336  branch_flag = ""
337  if branch != "master":
338    branch_flag = " -b {0}".format(branch)
339  chrome_version_flag = ""
340  if chrome_version:
341    chrome_version_flag = " --chrome_version={0}".format(chrome_version)
342  description = "{0}_{1}_{2}".format(branch, GetPatchString(patches), target)
343  command = ("yes | ./cbuildbot {0} {1} {2} {3} {4} {5}"
344             " --remote-description={6}"
345             .format(patch, branch_flag, chrome_version, local_flag,
346                     chrome_version_flag, target, description))
347  ce.RunCommand(command)
348  return description
349
350
351def Main(argv):
352  """The main function."""
353  # Common initializations
354  parser = argparse.ArgumentParser()
355  parser.add_argument("-c", "--chromeos_root", required=True,
356                      dest="chromeos_root", help="The chromeos_root")
357  parser.add_argument("-g", "--gcc_dir", default="", dest="gcc_dir",
358                      help="The gcc dir")
359  parser.add_argument("-t", "--target", required=True, dest="target",
360                      help=("The target to be build, the list is at"
361                            " $(chromeos_root)/chromite/buildbot/cbuildbot"
362                            " --list -all"))
363  parser.add_argument("-l", "--local", action="store_true")
364  parser.add_argument("-d", "--dest_dir", dest="dest_dir",
365                      help=("The dir to build the whole chromeos if"
366                            " --local is set"))
367  parser.add_argument("--chrome_version", dest="chrome_version",
368                      default="", help="The chrome version to use. "
369                      "Default it will use the latest one.")
370  parser.add_argument("--chromeos_version", dest="chromeos_version",
371                      default="", help="The chromeos version to use.")
372
373  parser.add_argument("-r", "--replace_sysroot", action="store_true",
374                      help=("Whether or not to replace the build/$board dir"
375                            "under the chroot of chromeos_root and copy "
376                            "the image to src/build/image/$board/latest."
377                            " Default is False"))
378  parser.add_argument("-b", "--branch", dest="branch", default="",
379                      help=("The branch to run trybot, default is None"))
380  parser.add_argument("-p", "--patch", dest="patch", default="",
381                      help=("The patches to be applied, the patches numbers "
382                            "be seperated by ','"))
383
384  script_dir = os.path.dirname(os.path.realpath(__file__))
385
386  args = parser.parse_args(argv[1:])
387  target = args.target
388  if args.patch:
389    patch = args.patch.split(",")
390  else:
391    patch = []
392  chromeos_root = misc.CanonicalizePath(args.chromeos_root)
393  branch = args.branch
394  # descritption is the keyword of the build in build log.
395  # Here we use [{branch)_{patchnumber}_{target}]
396  description = "{0}_{1}_{2}".format(branch, GetPatchString(patch), target)
397  if args.chromeos_version and args.branch:
398    raise Exception("You can not set chromeos_version and branch at the "
399                    "same time.")
400  chromeos_version = args.chromeos_version
401  if args.branch:
402    chromeos_version = 0
403  else:
404    chromeos_version = args.chromeos_version
405  if chromeos_version:
406    manifest_patch = GetManifestPatch(chromeos_version,
407                                      chromeos_root)
408    patch.append(manifest_patch)
409  if args.gcc_dir:
410    if not branch:
411      branch = chromeos_version
412    patch.append(UploadGccPatch(chromeos_root, args.gcc_dir, branch))
413  index = 0
414  description = RunRemote(chromeos_root, branch, patch, args.local,
415                          target, args.chrome_version, args.dest_dir)
416  if args.local or not args.dest_dir:
417    return 0
418  os.chdir(script_dir)
419  dest_dir = misc.CanonicalizePath(args.dest_dir)
420  if index <= 0:
421    index = FindResultIndex(description)
422  if not index:
423    logger.GetLogger().LogFatal("Remote trybot failed.")
424  if "toolchain" in branch:
425    chromeos_version = FindVersionForToolchain(branch, chromeos_root)
426  DownloadImage(target, index, dest_dir, chromeos_version)
427  ret = UnpackImage(dest_dir)
428  if not args.replace_sysroot:
429    return ret
430  else:
431    return ReplaceSysroot(chromeos_root, args.dest_dir, target)
432
433if __name__ == "__main__":
434  retval = Main(sys.argv)
435  sys.exit(retval)
436