build_tc.py revision 8ac4ec04a55eff35b33977dbb5a0964fd62cacd8 
1#!/usr/bin/python
2#
3# Copyright 2010 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Script to build the ChromeOS toolchain.
8
9This script sets up the toolchain if you give it the gcctools directory.
10"""
11
12__author__ = "asharif@google.com (Ahmad Sharif)"
13
14import getpass
15import optparse
16import os
17import sys
18import tempfile
19
20import tc_enter_chroot
21from utils import command_executer
22from utils import constants
23from utils import misc
24
25
26class ToolchainPart(object):
27  def __init__(self, name, source_path, chromeos_root, board, incremental,
28               build_env, gcc_enable_ccache=False):
29    self._name = name
30    self._source_path = misc.CanonicalizePath(source_path)
31    self._chromeos_root = chromeos_root
32    self._board = board
33    self._ctarget = misc.GetCtargetFromBoard(self._board,
34                                              self._chromeos_root)
35    self.tag = "%s-%s" % (name, self._ctarget)
36    self._ce = command_executer.GetCommandExecuter()
37    self._mask_file = os.path.join(
38        self._chromeos_root,
39        "chroot",
40        "etc/portage/package.mask/cross-%s" % self._ctarget)
41    self._new_mask_file = None
42
43    self._chroot_source_path = os.path.join(constants.MOUNTED_TOOLCHAIN_ROOT,
44                                            self._name).lstrip("/")
45    self._incremental = incremental
46    self._build_env = build_env
47    self._gcc_enable_ccache = gcc_enable_ccache
48
49  def RunSetupBoardIfNecessary(self):
50    cross_symlink = os.path.join(
51        self._chromeos_root,
52        "chroot",
53        "usr/local/bin/emerge-%s" % self._board)
54    if not os.path.exists(cross_symlink):
55      command = ("%s/setup_board --board=%s" %
56                 (misc.CHROMEOS_SCRIPTS_DIR, self._board))
57      self._ce.ChrootRunCommand(self._chromeos_root, command)
58
59  def Build(self):
60    rv = 1
61    try:
62      self.UninstallTool()
63      self.MoveMaskFile()
64      self.MountSources(False)
65      self.RemoveCompiledFile()
66      rv = self.BuildTool()
67    finally:
68      self.UnMoveMaskFile()
69    return rv
70
71  def RemoveCompiledFile(self):
72    compiled_file = os.path.join(self._chromeos_root,
73                                 "chroot",
74                                 "var/tmp/portage/cross-%s" % self._ctarget,
75                                 "%s-9999" % self._name,
76                                 ".compiled")
77    command = "rm -f %s" % compiled_file
78    self._ce.RunCommand(command)
79
80  def MountSources(self, unmount_source):
81    mount_points = []
82    mounted_source_path = os.path.join(self._chromeos_root,
83                                       "chroot",
84                                       self._chroot_source_path)
85    src_mp = tc_enter_chroot.MountPoint(
86        self._source_path,
87        mounted_source_path,
88        getpass.getuser(),
89        "ro")
90    mount_points.append(src_mp)
91
92    build_suffix = "build-%s" % self._ctarget
93    build_dir = "%s-%s" % (self._source_path, build_suffix)
94
95    if not self._incremental and os.path.exists(build_dir):
96      command = "rm -rf %s/*" % build_dir
97      self._ce.RunCommand(command)
98
99    # Create a -build directory for the objects.
100    command = "mkdir -p %s" % build_dir
101    self._ce.RunCommand(command)
102
103    mounted_build_dir = os.path.join(
104        self._chromeos_root, "chroot", "%s-%s" %
105        (self._chroot_source_path, build_suffix))
106    build_mp = tc_enter_chroot.MountPoint(
107        build_dir,
108        mounted_build_dir,
109        getpass.getuser())
110    mount_points.append(build_mp)
111
112    if unmount_source:
113      unmount_statuses = [mp.UnMount() == 0 for mp in mount_points]
114      assert all(unmount_statuses), "Could not unmount all mount points!"
115    else:
116      mount_statuses = [mp.DoMount() == 0 for mp in mount_points]
117
118      if not all(mount_statuses):
119        mounted = [mp for mp, status in zip(mount_points, mount_statuses) if status]
120        unmount_statuses = [mp.UnMount() == 0 for mp in mounted]
121        assert all(unmount_statuses), "Could not unmount all mount points!"
122
123
124  def UninstallTool(self):
125    command = "sudo CLEAN_DELAY=0 emerge -C cross-%s/%s" % (self._ctarget, self._name)
126    self._ce.ChrootRunCommand(self._chromeos_root, command)
127
128  def BuildTool(self):
129    env = self._build_env
130    # FEATURES=buildpkg adds minutes of time so we disable it.
131    # TODO(shenhan): keep '-sandbox' for a while for compatibility, then remove
132    # it after a while.
133    features = "nostrip userpriv userfetch -usersandbox -sandbox noclean -buildpkg"
134    env["FEATURES"] = features
135
136    if self._incremental:
137      env["FEATURES"] += " keepwork"
138
139    if "USE" in env:
140      env["USE"] += " multislot mounted_%s" % self._name
141    else:
142      env["USE"] = "multislot mounted_%s" % self._name
143
144    # Disable ccache in our compilers. cache may be problematic for us.
145    # It ignores compiler environments settings and it is not clear if
146    # the cache hit algorithm verifies all the compiler binaries or
147    # just the driver.
148    if self._name == "gcc" and not self._gcc_enable_ccache:
149      env["USE"] += " -wrapper_ccache"
150
151    env["%s_SOURCE_PATH" % self._name.upper()] = (
152        os.path.join("/", self._chroot_source_path))
153    env["ACCEPT_KEYWORDS"] = "~*"
154    env_string = " ".join(["%s=\"%s\"" % var for var in env.items()])
155    command = "emerge =cross-%s/%s-9999" % (self._ctarget, self._name)
156    full_command = "sudo %s %s" % (env_string, command)
157    rv = self._ce.ChrootRunCommand(self._chromeos_root, full_command)
158    if rv != 0:
159      return rv
160    if self._name == "gcc":
161      command = ("sudo cp -r /usr/lib/gcc/%s /build/%s/usr/lib/gcc/." %
162                 (self._ctarget, self._board))
163      rv = self._ce.ChrootRunCommand(self._chromeos_root, command)
164    return rv
165
166  def MoveMaskFile(self):
167    self._new_mask_file = None
168    if os.path.isfile(self._mask_file):
169      self._new_mask_file = tempfile.mktemp()
170      command = "sudo mv %s %s" % (self._mask_file, self._new_mask_file)
171      self._ce.RunCommand(command)
172
173  def UnMoveMaskFile(self):
174    if self._new_mask_file:
175      command = "sudo mv %s %s" % (self._new_mask_file, self._mask_file)
176      self._ce.RunCommand(command)
177
178
179def Main(argv):
180  """The main function."""
181  # Common initializations
182  parser = optparse.OptionParser()
183  parser.add_option("-c",
184                    "--chromeos_root",
185                    dest="chromeos_root",
186                    default="../../",
187                    help=("ChromeOS root checkout directory"
188                          " uses ../.. if none given."))
189  parser.add_option("-g",
190                    "--gcc_dir",
191                    dest="gcc_dir",
192                    help="The directory where gcc resides.")
193  parser.add_option("--binutils_dir",
194                    dest="binutils_dir",
195                    help="The directory where binutils resides.")
196  parser.add_option("-x",
197                    "--gdb_dir",
198                    dest="gdb_dir",
199                    help="The directory where gdb resides.")
200  parser.add_option("-b",
201                    "--board",
202                    dest="board",
203                    default="x86-agz",
204                    help="The target board.")
205  parser.add_option("-n",
206                    "--noincremental",
207                    dest="noincremental",
208                    default=False,
209                    action="store_true",
210                    help="Use FEATURES=keepwork to do incremental builds.")
211  parser.add_option("--cflags",
212                    dest="cflags",
213                    default="",
214                    help="Build a compiler with specified CFLAGS")
215  parser.add_option("--cxxflags",
216                    dest="cxxflags",
217                    default="",
218                    help="Build a compiler with specified CXXFLAGS")
219  parser.add_option("--cflags_for_target",
220                    dest="cflags_for_target",
221                    default="",
222                    help="Build the target libraries with specified flags")
223  parser.add_option("--cxxflags_for_target",
224                    dest="cxxflags_for_target",
225                    default="",
226                    help="Build the target libraries with specified flags")
227  parser.add_option("--ldflags",
228                    dest="ldflags",
229                    default="",
230                    help="Build a compiler with specified LDFLAGS")
231  parser.add_option("-d",
232                    "--debug",
233                    dest="debug",
234                    default=False,
235                    action="store_true",
236                    help="Build a compiler with -g3 -O0 appended to both"
237                    " CFLAGS and CXXFLAGS.")
238  parser.add_option("-m",
239                    "--mount_only",
240                    dest="mount_only",
241                    default=False,
242                    action="store_true",
243                    help="Just mount the tool directories.")
244  parser.add_option("-u",
245                    "--unmount_only",
246                    dest="unmount_only",
247                    default=False,
248                    action="store_true",
249                    help="Just unmount the tool directories.")
250  parser.add_option("--extra_use_flags",
251                    dest="extra_use_flags",
252                    default="",
253                    help="Extra flag for USE, to be passed to the ebuild. "
254                    "('multislot' and 'mounted_<tool>' are always passed.)")
255  parser.add_option("--gcc_enable_ccache",
256                    dest="gcc_enable_ccache",
257                    default=False,
258                    action="store_true",
259                    help="Enable ccache for the gcc invocations")
260
261  options, _ = parser.parse_args(argv)
262
263  chromeos_root = misc.CanonicalizePath(options.chromeos_root)
264  if options.gcc_dir:
265    gcc_dir = misc.CanonicalizePath(options.gcc_dir)
266    assert gcc_dir and os.path.isdir(gcc_dir), "gcc_dir does not exist!"
267  if options.binutils_dir:
268    binutils_dir = misc.CanonicalizePath(options.binutils_dir)
269    assert os.path.isdir(binutils_dir), "binutils_dir does not exist!"
270  if options.gdb_dir:
271    gdb_dir = misc.CanonicalizePath(options.gdb_dir)
272    assert os.path.isdir(gdb_dir), "gdb_dir does not exist!"
273  if options.unmount_only:
274    options.mount_only = False
275  elif options.mount_only:
276    options.unmount_only = False
277  build_env = {}
278  if options.cflags:
279    build_env["CFLAGS"] = "`portageq envvar CFLAGS` " + options.cflags
280  if options.cxxflags:
281    build_env["CXXFLAGS"] = "`portageq envvar CXXFLAGS` " + options.cxxflags
282  if options.cflags_for_target:
283    build_env["CFLAGS_FOR_TARGET"] = options.cflags_for_target
284  if options.cxxflags_for_target:
285    build_env["CXXFLAGS_FOR_TARGET"] = options.cxxflags_for_target
286  if options.ldflags:
287    build_env["LDFLAGS"] = options.ldflags
288  if options.debug:
289    debug_flags = "-g3 -O0"
290    if "CFLAGS" in build_env:
291      build_env["CFLAGS"] += " %s" % (debug_flags)
292    else:
293      build_env["CFLAGS"] = debug_flags
294    if "CXXFLAGS" in build_env:
295      build_env["CXXFLAGS"] += " %s" % (debug_flags)
296    else:
297      build_env["CXXFLAGS"] = debug_flags
298  if options.extra_use_flags:
299    build_env["USE"] = options.extra_use_flags
300
301  # Create toolchain parts
302  toolchain_parts = {}
303  for board in options.board.split(","):
304    if options.gcc_dir:
305      tp = ToolchainPart("gcc", gcc_dir, chromeos_root, board,
306                         not options.noincremental, build_env,
307                         options.gcc_enable_ccache)
308      toolchain_parts[tp.tag] = tp
309      tp.RunSetupBoardIfNecessary()
310    if options.binutils_dir:
311      tp = ToolchainPart("binutils", binutils_dir, chromeos_root, board,
312                         not options.noincremental, build_env)
313      toolchain_parts[tp.tag] = tp
314      tp.RunSetupBoardIfNecessary()
315    if options.gdb_dir:
316      tp = ToolchainPart("gdb", gdb_dir, chromeos_root, board,
317                         not options.noincremental, build_env)
318      toolchain_parts[tp.tag] = tp
319      tp.RunSetupBoardIfNecessary()
320
321  rv = 0
322  try:
323    for tag in toolchain_parts:
324      tp = toolchain_parts[tag]
325      if options.mount_only or options.unmount_only:
326        tp.MountSources(options.unmount_only)
327      else:
328        rv = rv + tp.Build()
329  finally:
330    print "Exiting..."
331  return rv
332
333if __name__ == "__main__":
334  retval = Main(sys.argv)
335  sys.exit(retval)
336