update.py revision f3b273f5e6ffd2f6ba1c18a27a17db41dfb113c3 
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This script is used to download prebuilt clang binaries.
7
8It is also used by package.py to build the prebuilt clang binaries."""
9
10import argparse
11import distutils.spawn
12import glob
13import os
14import pipes
15import re
16import shutil
17import subprocess
18import stat
19import sys
20import tarfile
21import tempfile
22import time
23import urllib2
24import zipfile
25
26
27# Do NOT CHANGE this if you don't know what you're doing -- see
28# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
29# Reverting problematic clang rolls is safe, though.
30CLANG_REVISION = '282487'
31
32use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
33if use_head_revision:
34  CLANG_REVISION = 'HEAD'
35
36# This is incremented when pushing a new build of Clang at the same revision.
37CLANG_SUB_REVISION=1
38
39PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
40
41# Path constants. (All of these should be absolute paths.)
42THIS_DIR = os.path.abspath(os.path.dirname(__file__))
43CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
44THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
45LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
46LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
47LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
48                                          'llvm-bootstrap-install')
49LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
50CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
51LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
52                              'Release+Asserts')
53COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
54CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
55LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
56# compiler-rt is built as part of the regular LLVM build on Windows to get
57# the 64-bit runtime, and out-of-tree elsewhere.
58# TODO(thakis): Try to unify this.
59if sys.platform == 'win32':
60  COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
61else:
62  COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt')
63LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
64LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
65LLVM_BUILD_TOOLS_DIR = os.path.abspath(
66    os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
67STAMP_FILE = os.path.normpath(
68    os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
69BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
70BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, BINUTILS_DIR,
71                                'Linux_x64', 'Release', 'bin')
72BFD_PLUGINS_DIR = os.path.join(BINUTILS_DIR, 'Linux_x64', 'Release',
73                               'lib', 'bfd-plugins')
74VERSION = '4.0.0'
75ANDROID_NDK_DIR = os.path.join(
76    CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
77
78# URL for pre-built binaries.
79CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
80    'https://commondatastorage.googleapis.com/chromium-browser-clang')
81
82LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
83if 'LLVM_REPO_URL' in os.environ:
84  LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
85
86# Bump after VC updates.
87DIA_DLL = {
88  '2013': 'msdia120.dll',
89  '2015': 'msdia140.dll',
90}
91
92
93def DownloadUrl(url, output_file):
94  """Download url into output_file."""
95  CHUNK_SIZE = 4096
96  TOTAL_DOTS = 10
97  num_retries = 3
98  retry_wait_s = 5  # Doubled at each retry.
99
100  while True:
101    try:
102      sys.stdout.write('Downloading %s ' % url)
103      sys.stdout.flush()
104      response = urllib2.urlopen(url)
105      total_size = int(response.info().getheader('Content-Length').strip())
106      bytes_done = 0
107      dots_printed = 0
108      while True:
109        chunk = response.read(CHUNK_SIZE)
110        if not chunk:
111          break
112        output_file.write(chunk)
113        bytes_done += len(chunk)
114        num_dots = TOTAL_DOTS * bytes_done / total_size
115        sys.stdout.write('.' * (num_dots - dots_printed))
116        sys.stdout.flush()
117        dots_printed = num_dots
118      if bytes_done != total_size:
119        raise urllib2.URLError("only got %d of %d bytes" %
120                               (bytes_done, total_size))
121      print ' Done.'
122      return
123    except urllib2.URLError as e:
124      sys.stdout.write('\n')
125      print e
126      if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
127        raise e
128      num_retries -= 1
129      print 'Retrying in %d s ...' % retry_wait_s
130      time.sleep(retry_wait_s)
131      retry_wait_s *= 2
132
133
134def EnsureDirExists(path):
135  if not os.path.exists(path):
136    print "Creating directory %s" % path
137    os.makedirs(path)
138
139
140def DownloadAndUnpack(url, output_dir):
141  with tempfile.TemporaryFile() as f:
142    DownloadUrl(url, f)
143    f.seek(0)
144    EnsureDirExists(output_dir)
145    if url.endswith('.zip'):
146      zipfile.ZipFile(f).extractall(path=output_dir)
147    else:
148      tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
149
150
151def ReadStampFile(path=STAMP_FILE):
152  """Return the contents of the stamp file, or '' if it doesn't exist."""
153  try:
154    with open(path, 'r') as f:
155      return f.read().rstrip()
156  except IOError:
157    return ''
158
159
160def WriteStampFile(s, path=STAMP_FILE):
161  """Write s to the stamp file."""
162  EnsureDirExists(os.path.dirname(path))
163  with open(path, 'w') as f:
164    f.write(s)
165    f.write('\n')
166
167
168def GetSvnRevision(svn_repo):
169  """Returns current revision of the svn repo at svn_repo."""
170  if sys.platform == 'darwin':
171    # mac_files toolchain must be set for hermetic builds.
172    root = os.path.dirname(os.path.dirname(os.path.dirname(
173        os.path.dirname(__file__))))
174    sys.path.append(os.path.join(root, 'build'))
175    import mac_toolchain
176
177    mac_toolchain.SetToolchainEnvironment()
178  svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
179  m = re.search(r'Revision: (\d+)', svn_info)
180  return m.group(1)
181
182
183def RmTree(dir):
184  """Delete dir."""
185  def ChmodAndRetry(func, path, _):
186    # Subversion can leave read-only files around.
187    if not os.access(path, os.W_OK):
188      os.chmod(path, stat.S_IWUSR)
189      return func(path)
190    raise
191
192  shutil.rmtree(dir, onerror=ChmodAndRetry)
193
194
195def RmCmakeCache(dir):
196  """Delete CMake cache related files from dir."""
197  for dirpath, dirs, files in os.walk(dir):
198    if 'CMakeCache.txt' in files:
199      os.remove(os.path.join(dirpath, 'CMakeCache.txt'))
200    if 'CMakeFiles' in dirs:
201      RmTree(os.path.join(dirpath, 'CMakeFiles'))
202
203
204def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
205  """Run command and return success (True) or failure; or if fail_hard is
206     True, exit on failure.  If msvc_arch is set, runs the command in a
207     shell with the msvc tools for that architecture."""
208
209  if msvc_arch and sys.platform == 'win32':
210    command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
211
212  # https://docs.python.org/2/library/subprocess.html:
213  # "On Unix with shell=True [...] if args is a sequence, the first item
214  # specifies the command string, and any additional items will be treated as
215  # additional arguments to the shell itself.  That is to say, Popen does the
216  # equivalent of:
217  #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
218  #
219  # We want to pass additional arguments to command[0], not to the shell,
220  # so manually join everything into a single string.
221  # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
222  # quote c:\path but svn can't handle quoted paths on Windows.  Since on
223  # Windows follow-on args are passed to args[0] instead of the shell, don't
224  # do the single-string transformation there.
225  if sys.platform != 'win32':
226    command = ' '.join([pipes.quote(c) for c in command])
227  print 'Running', command
228  if subprocess.call(command, env=env, shell=True) == 0:
229    return True
230  print 'Failed.'
231  if fail_hard:
232    sys.exit(1)
233  return False
234
235
236def CopyFile(src, dst):
237  """Copy a file from src to dst."""
238  print "Copying %s to %s" % (src, dst)
239  shutil.copy(src, dst)
240
241
242def CopyDirectoryContents(src, dst, filename_filter=None):
243  """Copy the files from directory src to dst
244  with an optional filename filter."""
245  dst = os.path.realpath(dst)  # realpath() in case dst ends in /..
246  EnsureDirExists(dst)
247  for root, _, files in os.walk(src):
248    for f in files:
249      if filename_filter and not re.match(filename_filter, f):
250        continue
251      CopyFile(os.path.join(root, f), dst)
252
253
254def Checkout(name, url, dir):
255  """Checkout the SVN module at url into dir. Use name for the log message."""
256  print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir)
257
258  command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir]
259  if RunCommand(command, fail_hard=False):
260    return
261
262  if os.path.isdir(dir):
263    print "Removing %s." % (dir)
264    RmTree(dir)
265
266  print "Retrying."
267  RunCommand(command)
268
269
270def DeleteChromeToolsShim():
271  OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
272  shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
273  shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
274
275
276def CreateChromeToolsShim():
277  """Hooks the Chrome tools into the LLVM build.
278
279  Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
280  detects implicit tools in the tools subdirectory, so this helper install a
281  shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
282
283  Note that the shim directory name intentionally has no - or _. The implicit
284  tool detection logic munges them in a weird way."""
285  assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
286  os.mkdir(CHROME_TOOLS_SHIM_DIR)
287  with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
288    f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
289            'Do not edit.\n')
290    f.write('# Since tools/clang is located in another directory, use the \n')
291    f.write('# two arg version to specify where build artifacts go. CMake\n')
292    f.write('# disallows reuse of the same binary dir for multiple source\n')
293    f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
294    f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
295    f.write('if (CHROMIUM_TOOLS_SRC)\n')
296    f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
297              '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
298    f.write('endif (CHROMIUM_TOOLS_SRC)\n')
299
300
301def DownloadHostGcc(args):
302  """Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set."""
303  if not sys.platform.startswith('linux') or args.gcc_toolchain:
304    return
305  # Unconditionally download a prebuilt gcc to guarantee the included libstdc++
306  # works on Ubuntu Precise.
307  gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc485precise')
308  if not os.path.exists(gcc_dir):
309    print 'Downloading pre-built GCC 4.8.5...'
310    DownloadAndUnpack(
311        CDS_URL + '/tools/gcc485precise.tgz', LLVM_BUILD_TOOLS_DIR)
312  args.gcc_toolchain = gcc_dir
313
314
315def AddCMakeToPath():
316  """Download CMake and add it to PATH."""
317  if sys.platform == 'win32':
318    zip_name = 'cmake-3.4.3-win32-x86.zip'
319    cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
320                             'cmake-3.4.3-win32-x86', 'bin')
321  else:
322    suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
323    zip_name = 'cmake343_%s.tgz' % suffix
324    cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake343', 'bin')
325  if not os.path.exists(cmake_dir):
326    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
327  os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
328
329
330def AddGnuWinToPath():
331  """Download some GNU win tools and add them to PATH."""
332  if sys.platform != 'win32':
333    return
334
335  gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
336  GNUWIN_VERSION = '5'
337  GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
338  if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
339    print 'GNU Win tools already up to date.'
340  else:
341    zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION
342    DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
343    WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP)
344
345  os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '')
346
347
348vs_version = None
349def GetVSVersion():
350  global vs_version
351  if vs_version:
352    return vs_version
353
354  # Try using the toolchain in depot_tools.
355  # This sets environment variables used by SelectVisualStudioVersion below.
356  sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
357  import vs_toolchain
358  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
359
360  # Use gyp to find the MSVS installation, either in depot_tools as per above,
361  # or a system-wide installation otherwise.
362  sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
363  import gyp.MSVSVersion
364  vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
365      vs_toolchain.GetVisualStudioVersion())
366  return vs_version
367
368
369def CopyDiaDllTo(target_dir):
370  # This script always wants to use the 64-bit msdia*.dll.
371  dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
372  dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
373  CopyFile(dia_dll, target_dir)
374
375
376def VeryifyVersionOfBuiltClangMatchesVERSION():
377  """Checks that `clang --version` outputs VERSION.  If this fails, VERSION
378  in this file is out-of-date and needs to be updated (possibly in an
379  `if use_head_revision:` block in main() first)."""
380  clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
381  if sys.platform == 'win32':
382    # TODO: Parse `clang-cl /?` output for built clang's version and check that
383    # to check the binary we're actually shipping? But clang-cl.exe is just
384    # a copy of clang.exe, so this does check the same thing.
385    clang += '.exe'
386  version_out = subprocess.check_output([clang, '--version'])
387  version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1)
388  if version_out != VERSION:
389    print ('unexpected clang version %s (not %s), update VERSION in update.py'
390           % (version_out, VERSION))
391    sys.exit(1)
392
393
394def UpdateClang(args):
395  print 'Updating Clang to %s...' % PACKAGE_VERSION
396
397  need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or (
398      sys.platform.startswith('linux') and
399      'buildtype=Official' in os.environ.get('GYP_DEFINES', '') and
400      'branding=Chrome' in os.environ.get('GYP_DEFINES', ''))
401
402  if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build:
403    print 'Clang is already up to date.'
404    if not need_gold_plugin or os.path.exists(
405        os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")):
406      return 0
407
408  # Reset the stamp file in case the build is unsuccessful.
409  WriteStampFile('')
410
411  if not args.force_local_build:
412    cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
413    if sys.platform == 'win32' or sys.platform == 'cygwin':
414      cds_full_url = CDS_URL + '/Win/' + cds_file
415    elif sys.platform == 'darwin':
416      cds_full_url = CDS_URL + '/Mac/' + cds_file
417    else:
418      assert sys.platform.startswith('linux')
419      cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
420
421    print 'Downloading prebuilt clang'
422    if os.path.exists(LLVM_BUILD_DIR):
423      RmTree(LLVM_BUILD_DIR)
424    try:
425      DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
426      print 'clang %s unpacked' % PACKAGE_VERSION
427      if sys.platform == 'win32':
428        CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
429      # Download the gold plugin if requested to by an environment variable.
430      # This is used by the CFI ClusterFuzz bot, and it's required for official
431      # builds on linux.
432      if need_gold_plugin:
433        RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
434      WriteStampFile(PACKAGE_VERSION)
435      return 0
436    except urllib2.URLError:
437      print 'Failed to download prebuilt clang %s' % cds_file
438      print 'Use --force-local-build if you want to build locally.'
439      print 'Exiting.'
440      return 1
441
442  if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
443    print 'Android NDK not found at ' + ANDROID_NDK_DIR
444    print 'The Android NDK is needed to build a Clang whose -fsanitize=address'
445    print 'works on Android. See '
446    print 'https://www.chromium.org/developers/how-tos/android-build-instructions'
447    print 'for how to install the NDK, or pass --without-android.'
448    return 1
449
450  DownloadHostGcc(args)
451  AddCMakeToPath()
452  AddGnuWinToPath()
453
454  DeleteChromeToolsShim()
455
456  Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
457  Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
458  if sys.platform == 'win32' or use_head_revision:
459    Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
460  Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
461  if sys.platform == 'darwin':
462    # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
463    # (i.e. this is needed for bootstrap builds).
464    Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
465    # We used to check out libcxxabi on OS X; we no longer need that.
466    if os.path.exists(LIBCXXABI_DIR):
467      RmTree(LIBCXXABI_DIR)
468
469  cc, cxx = None, None
470  libstdcpp = None
471  if args.gcc_toolchain:  # This option is only used on Linux.
472    # Use the specified gcc installation for building.
473    cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
474    cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
475
476    if not os.access(cc, os.X_OK):
477      print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
478      print '"%s" does not appear to be valid.' % cc
479      return 1
480
481    # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
482    # compiler, etc.) find the .so.
483    libstdcpp = subprocess.check_output(
484        [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
485    os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
486
487  cflags = []
488  cxxflags = []
489  ldflags = []
490
491  base_cmake_args = ['-GNinja',
492                     '-DCMAKE_BUILD_TYPE=Release',
493                     '-DLLVM_ENABLE_ASSERTIONS=ON',
494                     '-DLLVM_ENABLE_THREADS=OFF',
495                     '-DLLVM_ENABLE_TIMESTAMPS=OFF',
496                     # Statically link MSVCRT to avoid DLL dependencies.
497                     '-DLLVM_USE_CRT_RELEASE=MT',
498                     ]
499
500  binutils_incdir = ''
501  if sys.platform.startswith('linux'):
502    binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
503
504  if args.bootstrap:
505    print 'Building bootstrap compiler'
506    EnsureDirExists(LLVM_BOOTSTRAP_DIR)
507    os.chdir(LLVM_BOOTSTRAP_DIR)
508    bootstrap_args = base_cmake_args + [
509        '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
510        '-DLLVM_TARGETS_TO_BUILD=host',
511        '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
512        '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
513        '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
514        ]
515    if cc is not None:  bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
516    if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
517    RmCmakeCache('.')
518    RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
519    RunCommand(['ninja'], msvc_arch='x64')
520    if args.run_tests:
521      if sys.platform == 'win32':
522        CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
523      RunCommand(['ninja', 'check-all'], msvc_arch='x64')
524    RunCommand(['ninja', 'install'], msvc_arch='x64')
525    if args.gcc_toolchain:
526      # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
527      # compiler can start.
528      CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
529
530    if sys.platform == 'win32':
531      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
532      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
533      # CMake has a hard time with backslashes in compiler paths:
534      # https://stackoverflow.com/questions/13050827
535      cc = cc.replace('\\', '/')
536      cxx = cxx.replace('\\', '/')
537    else:
538      cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
539      cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
540
541    if args.gcc_toolchain:
542      # Tell the bootstrap compiler to use a specific gcc prefix to search
543      # for standard library headers and shared object files.
544      cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
545      cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
546    print 'Building final compiler'
547
548  # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
549  # We only use LTO for Linux now.
550  if args.bootstrap and args.lto_gold_plugin:
551    print 'Building LTO LLVM Gold plugin'
552    if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
553      RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
554    EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
555    os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
556
557    # Create a symlink to LLVMgold.so build in the previous step so that ar
558    # and ranlib could find it while linking LLVMgold.so with LTO.
559    EnsureDirExists(BFD_PLUGINS_DIR)
560    RunCommand(['ln', '-sf',
561                os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
562                os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
563
564    lto_cflags = ['-flto']
565    lto_ldflags = ['-fuse-ld=gold']
566    if args.gcc_toolchain:
567      # Tell the bootstrap compiler to use a specific gcc prefix to search
568      # for standard library headers and shared object files.
569      lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
570    lto_cmake_args = base_cmake_args + [
571        '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
572        '-DCMAKE_C_COMPILER=' + cc,
573        '-DCMAKE_CXX_COMPILER=' + cxx,
574        '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
575        '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
576        '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
577        '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
578        '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
579
580    # We need to use the proper binutils which support LLVM Gold plugin.
581    lto_env = os.environ.copy()
582    lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
583
584    RmCmakeCache('.')
585    RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
586    RunCommand(['ninja', 'LLVMgold'], env=lto_env)
587
588
589  # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
590  # needed, on OS X it requires libc++. clang only automatically links to libc++
591  # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
592  # on OS X versions as old as 10.7.
593  deployment_target = ''
594
595  if sys.platform == 'darwin' and args.bootstrap:
596    # When building on 10.9, /usr/include usually doesn't exist, and while
597    # Xcode's clang automatically sets a sysroot, self-built clangs don't.
598    cflags = ['-isysroot', subprocess.check_output(
599        ['xcrun', '--show-sdk-path']).rstrip()]
600    cxxflags = ['-stdlib=libc++'] + cflags
601    ldflags += ['-stdlib=libc++']
602    deployment_target = '10.7'
603    # Running libc++ tests takes a long time. Since it was only needed for
604    # the install step above, don't build it as part of the main build.
605    # This makes running package.py over 10% faster (30 min instead of 34 min)
606    RmTree(LIBCXX_DIR)
607
608  # Build clang.
609
610  # If building at head, define a macro that plugins can use for #ifdefing
611  # out code that builds at head, but not at CLANG_REVISION or vice versa.
612  if use_head_revision:
613    cflags += ['-DLLVM_FORCE_HEAD_REVISION']
614    cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
615
616  CreateChromeToolsShim()
617
618  deployment_env = None
619  if deployment_target:
620    deployment_env = os.environ.copy()
621    deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
622
623  cmake_args = []
624  # TODO(thakis): Unconditionally append this to base_cmake_args instead once
625  # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
626  cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
627  if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
628  if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
629  cmake_args += base_cmake_args + [
630      '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
631      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
632      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
633      '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
634      '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
635      '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
636      '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
637      # TODO(thakis): Remove this once official builds pass -Wl,--build-id
638      # explicitly, https://crbug.com/622775
639      '-DENABLE_LINKER_BUILD_ID=ON',
640      '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
641      '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
642
643  EnsureDirExists(LLVM_BUILD_DIR)
644  os.chdir(LLVM_BUILD_DIR)
645  RmCmakeCache('.')
646  RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
647             msvc_arch='x64', env=deployment_env)
648
649  if args.gcc_toolchain:
650    # Copy in the right stdlibc++.so.6 so clang can start.
651    if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
652      os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
653    libstdcpp = subprocess.check_output(
654        [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
655    CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
656
657  RunCommand(['ninja'], msvc_arch='x64')
658
659  if args.tools:
660    # If any Chromium tools were built, install those now.
661    RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
662
663  if sys.platform == 'darwin':
664    # See http://crbug.com/256342
665    RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
666  elif sys.platform.startswith('linux'):
667    RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
668
669  VeryifyVersionOfBuiltClangMatchesVERSION()
670
671  # Do an out-of-tree build of compiler-rt.
672  # On Windows, this is used to get the 32-bit ASan run-time.
673  # TODO(hans): Remove once the regular build above produces this.
674  # On Mac and Linux, this is used to get the regular 64-bit run-time.
675  # Do a clobbered build due to cmake changes.
676  if os.path.isdir(COMPILER_RT_BUILD_DIR):
677    RmTree(COMPILER_RT_BUILD_DIR)
678  os.makedirs(COMPILER_RT_BUILD_DIR)
679  os.chdir(COMPILER_RT_BUILD_DIR)
680  # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
681  # above).
682  #if args.bootstrap and sys.platform == 'win32':
683    # The bootstrap compiler produces 64-bit binaries by default.
684    #cflags += ['-m32']
685    #cxxflags += ['-m32']
686  compiler_rt_args = base_cmake_args + [
687      '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
688      '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
689  if sys.platform == 'darwin':
690    compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON']
691  if sys.platform != 'win32':
692    compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
693                         os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
694                        '-DSANITIZER_MIN_OSX_VERSION="10.7"']
695  # compiler-rt is part of the llvm checkout on Windows but a stand-alone
696  # directory elsewhere, see the TODO above COMPILER_RT_DIR.
697  RmCmakeCache('.')
698  RunCommand(['cmake'] + compiler_rt_args +
699             [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR],
700             msvc_arch='x86', env=deployment_env)
701  RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
702
703  # Copy select output to the main tree.
704  # TODO(hans): Make this (and the .gypi and .isolate files) version number
705  # independent.
706  if sys.platform == 'win32':
707    platform = 'windows'
708  elif sys.platform == 'darwin':
709    platform = 'darwin'
710  else:
711    assert sys.platform.startswith('linux')
712    platform = 'linux'
713  asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform)
714  if sys.platform == 'win32':
715    # TODO(thakis): This too is due to compiler-rt being part of the checkout
716    # on Windows, see TODO above COMPILER_RT_DIR.
717    asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
718                                       VERSION, 'lib', platform)
719  asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
720                                     VERSION, 'lib', platform)
721  # Blacklists:
722  CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'),
723                        os.path.join(asan_rt_lib_dst_dir, '..', '..'),
724                        r'^.*blacklist\.txt$')
725  # Headers:
726  if sys.platform != 'win32':
727    CopyDirectoryContents(
728        os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'),
729        os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer'))
730  # Static and dynamic libraries:
731  CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir)
732  if sys.platform == 'darwin':
733    for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')):
734      # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
735      # @executable_path.
736      # TODO(glider): this is transitional. We'll need to fix the dylib
737      # name either in our build system, or in Clang. See also
738      # http://crbug.com/344836.
739      subprocess.call(['install_name_tool', '-id',
740                       '@executable_path/' + os.path.basename(dylib), dylib])
741
742
743  if sys.platform == 'win32':
744    # Make an extra copy of the sanitizer headers, to be put on the include path
745    # of the fallback compiler.
746    sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
747                                         VERSION, 'include', 'sanitizer')
748    aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
749                                             VERSION, 'include_sanitizer',
750                                             'sanitizer')
751    EnsureDirExists(aux_sanitizer_include_dir)
752    for _, _, files in os.walk(sanitizer_include_dir):
753      for f in files:
754        CopyFile(os.path.join(sanitizer_include_dir, f),
755                 aux_sanitizer_include_dir)
756
757  if args.with_android:
758    make_toolchain = os.path.join(
759        ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py')
760    for target_arch in ['aarch64', 'arm', 'i686']:
761      # Make standalone Android toolchain for target_arch.
762      toolchain_dir = os.path.join(
763          LLVM_BUILD_DIR, 'android-toolchain-' + target_arch)
764      RunCommand([
765          make_toolchain,
766          '--api=' + ('21' if target_arch == 'aarch64' else '19'),
767          '--force',
768          '--install-dir=%s' % toolchain_dir,
769          '--stl=stlport',
770          '--arch=' + {
771              'aarch64': 'arm64',
772              'arm': 'arm',
773              'i686': 'x86',
774          }[target_arch]])
775      # Android NDK r9d copies a broken unwind.h into the toolchain, see
776      # http://crbug.com/357890
777      for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')):
778        os.remove(f)
779
780      # Build ASan runtime for Android in a separate build tree.
781      build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
782      if not os.path.exists(build_dir):
783        os.mkdir(os.path.join(build_dir))
784      os.chdir(build_dir)
785      cflags = ['--target=%s-linux-androideabi' % target_arch,
786                '--sysroot=%s/sysroot' % toolchain_dir,
787                '-B%s' % toolchain_dir]
788      android_args = base_cmake_args + [
789        '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
790        '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
791        '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
792        '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
793        '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
794        '-DANDROID=1']
795      RmCmakeCache('.')
796      RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
797      RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch])
798
799      # And copy it into the main build tree.
800      runtime = 'libclang_rt.asan-%s-android.so' % target_arch
801      for root, _, files in os.walk(build_dir):
802        if runtime in files:
803          shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir)
804
805  # Run tests.
806  if args.run_tests or use_head_revision:
807    os.chdir(LLVM_BUILD_DIR)
808    RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64')
809  if args.run_tests:
810    if sys.platform == 'win32':
811      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
812    os.chdir(LLVM_BUILD_DIR)
813    RunCommand(['ninja', 'check-all'], msvc_arch='x64')
814
815  WriteStampFile(PACKAGE_VERSION)
816  print 'Clang update was successful.'
817  return 0
818
819
820def main():
821  parser = argparse.ArgumentParser(description='Build Clang.')
822  parser.add_argument('--bootstrap', action='store_true',
823                      help='first build clang with CC, then with itself.')
824  parser.add_argument('--if-needed', action='store_true',
825                      help="run only if the script thinks clang is needed")
826  parser.add_argument('--force-local-build', action='store_true',
827                      help="don't try to download prebuild binaries")
828  parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
829                      'version be used for building; --gcc-toolchain=/opt/foo '
830                      'picks /opt/foo/bin/gcc')
831  parser.add_argument('--lto-gold-plugin', action='store_true',
832                      help='build LLVM Gold plugin with LTO')
833  parser.add_argument('--llvm-force-head-revision', action='store_true',
834                      help=('use the revision in the repo when printing '
835                            'the revision'))
836  parser.add_argument('--print-revision', action='store_true',
837                      help='print current clang revision and exit.')
838  parser.add_argument('--print-clang-version', action='store_true',
839                      help='print current clang version (e.g. x.y.z) and exit.')
840  parser.add_argument('--run-tests', action='store_true',
841                      help='run tests after building; only for local builds')
842  parser.add_argument('--tools', nargs='*',
843                      help='select which chrome tools to build',
844                      default=['plugins', 'blink_gc_plugin'])
845  parser.add_argument('--without-android', action='store_false',
846                      help='don\'t build Android ASan runtime (linux only)',
847                      dest='with_android',
848                      default=sys.platform.startswith('linux'))
849  args = parser.parse_args()
850
851  if args.lto_gold_plugin and not args.bootstrap:
852    print '--lto-gold-plugin requires --bootstrap'
853    return 1
854  if args.lto_gold_plugin and not sys.platform.startswith('linux'):
855    print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
856    args.lto_gold_plugin = False
857
858  if args.if_needed:
859    is_clang_required = False
860    # clang is always used on Mac and Linux.
861    if sys.platform == 'darwin' or sys.platform.startswith('linux'):
862      is_clang_required = True
863    # clang requested via $GYP_DEFINES.
864    if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
865                 os.environ.get('GYP_DEFINES', '')):
866      is_clang_required = True
867    # clang previously downloaded, keep it up to date.
868    # If you don't want this, delete third_party/llvm-build on your machine.
869    if os.path.isdir(LLVM_BUILD_DIR):
870      is_clang_required = True
871    if not is_clang_required:
872      return 0
873    if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
874      print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
875      return 0
876
877  global CLANG_REVISION, PACKAGE_VERSION
878  if args.print_revision:
879    if use_head_revision or args.llvm_force_head_revision:
880      print GetSvnRevision(LLVM_DIR)
881    else:
882      print PACKAGE_VERSION
883    return 0
884
885  if args.print_clang_version:
886    sys.stdout.write(VERSION)
887    return 0
888
889  # Don't buffer stdout, so that print statements are immediately flushed.
890  # Do this only after --print-revision has been handled, else we'll get
891  # an error message when this script is run from gn for some reason.
892  sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
893
894  if use_head_revision:
895    # Use a real revision number rather than HEAD to make sure that the stamp
896    # file logic works.
897    CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL)
898    PACKAGE_VERSION = CLANG_REVISION + '-0'
899
900    args.force_local_build = True
901    if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
902      # Only build the Android ASan rt on ToT bots when targetting Android.
903      args.with_android = False
904
905  return UpdateClang(args)
906
907
908if __name__ == '__main__':
909  sys.exit(main())
910