update.sh revision e5d81f57cb97b3b6b7fccc9c5610d21eb81db09d
1#!/usr/bin/env bash 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# This script will check out llvm and clang into third_party/llvm and build it. 7 8# Do NOT CHANGE this if you don't know what you're doing -- see 9# https://code.google.com/p/chromium/wiki/UpdatingClang 10# Reverting problematic clang rolls is safe, though. 11CLANG_REVISION=204777 12 13THIS_DIR="$(dirname "${0}")" 14LLVM_DIR="${THIS_DIR}/../../../third_party/llvm" 15LLVM_BUILD_DIR="${LLVM_DIR}/../llvm-build" 16LLVM_BOOTSTRAP_DIR="${LLVM_DIR}/../llvm-bootstrap" 17LLVM_BOOTSTRAP_INSTALL_DIR="${LLVM_DIR}/../llvm-bootstrap-install" 18CLANG_DIR="${LLVM_DIR}/tools/clang" 19CLANG_TOOLS_EXTRA_DIR="${CLANG_DIR}/tools/extra" 20COMPILER_RT_DIR="${LLVM_DIR}/projects/compiler-rt" 21LIBCXX_DIR="${LLVM_DIR}/projects/libcxx" 22LIBCXXABI_DIR="${LLVM_DIR}/projects/libcxxabi" 23ANDROID_NDK_DIR="${LLVM_DIR}/../android_tools/ndk" 24STAMP_FILE="${LLVM_BUILD_DIR}/cr_build_revision" 25 26ABS_LIBCXX_DIR="${PWD}/${LIBCXX_DIR}" 27ABS_LIBCXXABI_DIR="${PWD}/${LIBCXXABI_DIR}" 28 29 30# Use both the clang revision and the plugin revisions to test for updates. 31BLINKGCPLUGIN_REVISION=\ 32$(grep LIBRARYNAME "$THIS_DIR"/../blink_gc_plugin/Makefile \ 33 | cut -d '_' -f 2) 34CLANG_AND_PLUGINS_REVISION="${CLANG_REVISION}-${BLINKGCPLUGIN_REVISION}" 35 36# ${A:-a} returns $A if it's set, a else. 37LLVM_REPO_URL=${LLVM_URL:-https://llvm.org/svn/llvm-project} 38 39if [[ -z "$GYP_DEFINES" ]]; then 40 GYP_DEFINES= 41fi 42if [[ -z "$GYP_GENERATORS" ]]; then 43 GYP_GENERATORS= 44fi 45 46 47# Die if any command dies, error on undefined variable expansions. 48set -eu 49 50OS="$(uname -s)" 51 52# Parse command line options. 53if_needed= 54force_local_build= 55run_tests= 56bootstrap= 57with_android=yes 58chrome_tools="plugins blink_gc_plugin" 59gcc_toolchain= 60 61if [[ "${OS}" = "Darwin" ]]; then 62 with_android= 63fi 64if [ "${OS}" = "FreeBSD" ]; then 65 MAKE=gmake 66else 67 MAKE=make 68fi 69 70while [[ $# > 0 ]]; do 71 case $1 in 72 --bootstrap) 73 bootstrap=yes 74 ;; 75 --if-needed) 76 if_needed=yes 77 ;; 78 --force-local-build) 79 force_local_build=yes 80 ;; 81 --run-tests) 82 run_tests=yes 83 ;; 84 --without-android) 85 with_android= 86 ;; 87 --with-chrome-tools) 88 shift 89 if [[ $# == 0 ]]; then 90 echo "--with-chrome-tools requires an argument." 91 exit 1 92 fi 93 chrome_tools=$1 94 ;; 95 --gcc-toolchain) 96 shift 97 if [[ $# == 0 ]]; then 98 echo "--gcc-toolchain requires an argument." 99 exit 1 100 fi 101 if [[ -x "$1/bin/gcc" ]]; then 102 gcc_toolchain=$1 103 else 104 echo "Invalid --gcc-toolchain: '$1'." 105 echo "'$1/bin/gcc' does not appear to be valid." 106 exit 1 107 fi 108 ;; 109 110 --help) 111 echo "usage: $0 [--force-local-build] [--if-needed] [--run-tests] " 112 echo "--bootstrap: First build clang with CC, then with itself." 113 echo "--force-local-build: Don't try to download prebuilt binaries." 114 echo "--if-needed: Download clang only if the script thinks it is needed." 115 echo "--run-tests: Run tests after building. Only for local builds." 116 echo "--without-android: Don't build ASan Android runtime library." 117 echo "--with-chrome-tools: Select which chrome tools to build." \ 118 "Defaults to plugins." 119 echo " Example: --with-chrome-tools 'plugins empty-string'" 120 echo "--gcc-toolchain: Set the prefix for which GCC version should" 121 echo " be used for building. For example, to use gcc in" 122 echo " /opt/foo/bin/gcc, use '--gcc-toolchain '/opt/foo" 123 echo 124 exit 1 125 ;; 126 *) 127 echo "Unknown argument: '$1'." 128 echo "Use --help for help." 129 exit 1 130 ;; 131 esac 132 shift 133done 134 135if [[ -n "$if_needed" ]]; then 136 if [[ "${OS}" == "Darwin" ]]; then 137 # clang is used on Mac. 138 true 139 elif [[ "$GYP_DEFINES" =~ .*(clang|tsan|asan|lsan|msan)=1.* ]]; then 140 # clang requested via $GYP_DEFINES. 141 true 142 elif [[ -d "${LLVM_BUILD_DIR}" ]]; then 143 # clang previously downloaded, remove third_party/llvm-build to prevent 144 # updating. 145 true 146 else 147 # clang wasn't needed, not doing anything. 148 exit 0 149 fi 150fi 151 152 153# Check if there's anything to be done, exit early if not. 154if [[ -f "${STAMP_FILE}" ]]; then 155 PREVIOUSLY_BUILT_REVISON=$(cat "${STAMP_FILE}") 156 if [[ -z "$force_local_build" ]] && \ 157 [[ "${PREVIOUSLY_BUILT_REVISON}" = \ 158 "${CLANG_AND_PLUGINS_REVISION}" ]]; then 159 echo "Clang already at ${CLANG_AND_PLUGINS_REVISION}" 160 exit 0 161 fi 162fi 163# To always force a new build if someone interrupts their build half way. 164rm -f "${STAMP_FILE}" 165 166 167# Clobber build files. PCH files only work with the compiler that created them. 168# We delete .o files to make sure all files are built with the new compiler. 169echo "Clobbering build files" 170MAKE_DIR="${THIS_DIR}/../../../out" 171XCODEBUILD_DIR="${THIS_DIR}/../../../xcodebuild" 172for DIR in "${XCODEBUILD_DIR}" "${MAKE_DIR}/Debug" "${MAKE_DIR}/Release"; do 173 if [[ -d "${DIR}" ]]; then 174 find "${DIR}" -name '*.o' -exec rm {} + 175 find "${DIR}" -name '*.o.d' -exec rm {} + 176 find "${DIR}" -name '*.gch' -exec rm {} + 177 find "${DIR}" -name '*.dylib' -exec rm -rf {} + 178 find "${DIR}" -name 'SharedPrecompiledHeaders' -exec rm -rf {} + 179 fi 180done 181 182# Clobber NaCl toolchain stamp files, see http://crbug.com/159793 183if [[ -d "${MAKE_DIR}" ]]; then 184 find "${MAKE_DIR}" -name 'stamp.untar' -exec rm {} + 185fi 186if [[ "${OS}" = "Darwin" ]]; then 187 if [[ -d "${XCODEBUILD_DIR}" ]]; then 188 find "${XCODEBUILD_DIR}" -name 'stamp.untar' -exec rm {} + 189 fi 190fi 191 192if [[ -z "$force_local_build" ]]; then 193 # Check if there's a prebuilt binary and if so just fetch that. That's faster, 194 # and goma relies on having matching binary hashes on client and server too. 195 CDS_URL=https://commondatastorage.googleapis.com/chromium-browser-clang 196 CDS_FILE="clang-${CLANG_REVISION}.tgz" 197 CDS_OUT_DIR=$(mktemp -d -t clang_download.XXXXXX) 198 CDS_OUTPUT="${CDS_OUT_DIR}/${CDS_FILE}" 199 if [ "${OS}" = "Linux" ]; then 200 CDS_FULL_URL="${CDS_URL}/Linux_x64/${CDS_FILE}" 201 elif [ "${OS}" = "Darwin" ]; then 202 CDS_FULL_URL="${CDS_URL}/Mac/${CDS_FILE}" 203 fi 204 echo Trying to download prebuilt clang 205 if which curl > /dev/null; then 206 curl -L --fail "${CDS_FULL_URL}" -o "${CDS_OUTPUT}" || \ 207 rm -rf "${CDS_OUT_DIR}" 208 elif which wget > /dev/null; then 209 wget "${CDS_FULL_URL}" -O "${CDS_OUTPUT}" || rm -rf "${CDS_OUT_DIR}" 210 else 211 echo "Neither curl nor wget found. Please install one of these." 212 exit 1 213 fi 214 if [ -f "${CDS_OUTPUT}" ]; then 215 rm -rf "${LLVM_BUILD_DIR}/Release+Asserts" 216 mkdir -p "${LLVM_BUILD_DIR}/Release+Asserts" 217 tar -xzf "${CDS_OUTPUT}" -C "${LLVM_BUILD_DIR}/Release+Asserts" 218 echo clang "${CLANG_REVISION}" unpacked 219 echo "${CLANG_AND_PLUGINS_REVISION}" > "${STAMP_FILE}" 220 rm -rf "${CDS_OUT_DIR}" 221 exit 0 222 else 223 echo Did not find prebuilt clang at r"${CLANG_REVISION}", building 224 fi 225fi 226 227if [[ -n "${with_android}" ]] && ! [[ -d "${ANDROID_NDK_DIR}" ]]; then 228 echo "Android NDK not found at ${ANDROID_NDK_DIR}" 229 echo "The Android NDK is needed to build a Clang whose -fsanitize=address" 230 echo "works on Android. See " 231 echo "http://code.google.com/p/chromium/wiki/AndroidBuildInstructions for how" 232 echo "to install the NDK, or pass --without-android." 233 exit 1 234fi 235 236# Revert previous temporary patches. 237if [[ -d "${COMPILER_RT_DIR}" ]]; then 238 pushd "${COMPILER_RT_DIR}" 239 svn revert lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc 240 svn revert make/platform/clang_linux.mk 241 popd 242fi 243if [[ -d "${LLVM_DIR}" ]]; then 244 pushd "${LLVM_DIR}" 245 svn revert lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp 246 svn revert test/CodeGen/ARM/debug-frame-large-stack.ll 247 svn revert test/CodeGen/ARM/debug-frame-vararg.ll 248 svn revert test/CodeGen/ARM/debug-frame.ll 249 popd 250fi 251 252echo Getting LLVM r"${CLANG_REVISION}" in "${LLVM_DIR}" 253if ! svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" \ 254 "${LLVM_DIR}"; then 255 echo Checkout failed, retrying 256 rm -rf "${LLVM_DIR}" 257 svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" "${LLVM_DIR}" 258fi 259 260echo Getting clang r"${CLANG_REVISION}" in "${CLANG_DIR}" 261svn co --force "${LLVM_REPO_URL}/cfe/trunk@${CLANG_REVISION}" "${CLANG_DIR}" 262 263echo Getting compiler-rt r"${CLANG_REVISION}" in "${COMPILER_RT_DIR}" 264svn co --force "${LLVM_REPO_URL}/compiler-rt/trunk@${CLANG_REVISION}" \ 265 "${COMPILER_RT_DIR}" 266 267# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes 268# (i.e. this is needed for bootstrap builds). 269if [ "${OS}" = "Darwin" ]; then 270 echo Getting libc++ r"${CLANG_REVISION}" in "${LIBCXX_DIR}" 271 svn co --force "${LLVM_REPO_URL}/libcxx/trunk@${CLANG_REVISION}" \ 272 "${LIBCXX_DIR}" 273fi 274 275# While we're bundling our own libc++ on OS X, we need to compile libc++abi 276# into it too (since OS X 10.6 doesn't have libc++abi.dylib either). 277if [ "${OS}" = "Darwin" ]; then 278 echo Getting libc++abi r"${CLANG_REVISION}" in "${LIBCXXABI_DIR}" 279 svn co --force "${LLVM_REPO_URL}/libcxxabi/trunk@${CLANG_REVISION}" \ 280 "${LIBCXXABI_DIR}" 281fi 282 283# Apply patch for test failing with --disable-pthreads (llvm.org/PR11974) 284pushd "${CLANG_DIR}" 285svn revert test/Index/crash-recovery-modules.m 286cat << 'EOF' | 287--- third_party/llvm/tools/clang/test/Index/crash-recovery-modules.m (revision 202554) 288+++ third_party/llvm/tools/clang/test/Index/crash-recovery-modules.m (working copy) 289@@ -12,6 +12,8 @@ 290 291 // REQUIRES: crash-recovery 292 // REQUIRES: shell 293+// XFAIL: * 294+// (PR11974) 295 296 @import Crash; 297EOF 298patch -p4 299popd 300 301# Echo all commands. 302set -x 303 304NUM_JOBS=3 305if [[ "${OS}" = "Linux" ]]; then 306 NUM_JOBS="$(grep -c "^processor" /proc/cpuinfo)" 307elif [ "${OS}" = "Darwin" -o "${OS}" = "FreeBSD" ]; then 308 NUM_JOBS="$(sysctl -n hw.ncpu)" 309fi 310 311if [[ -n "${gcc_toolchain}" ]]; then 312 # Use the specified gcc installation for building. 313 export CC="$gcc_toolchain/bin/gcc" 314 export CXX="$gcc_toolchain/bin/g++" 315fi 316 317export CFLAGS="" 318export CXXFLAGS="" 319# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is 320# needed, on OS X it requires libc++. clang only automatically links to libc++ 321# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run on 322# OS X versions as old as 10.7. 323# TODO(thakis): Some bots are still on 10.6, so for now bundle libc++.dylib. 324# Remove this once all bots are on 10.7+, then use --enable-libcpp=yes and 325# change all MACOSX_DEPLOYMENT_TARGET values to 10.7. 326if [ "${OS}" = "Darwin" ]; then 327 # When building on 10.9, /usr/include usually doesn't exist, and while 328 # Xcode's clang automatically sets a sysroot, self-built clangs don't. 329 export CFLAGS="-isysroot $(xcrun --show-sdk-path)" 330 export CPPFLAGS="${CFLAGS}" 331 export CXXFLAGS="-stdlib=libc++ -nostdinc++ -I${ABS_LIBCXX_DIR}/include ${CFLAGS}" 332fi 333 334# Build bootstrap clang if requested. 335if [[ -n "${bootstrap}" ]]; then 336 ABS_INSTALL_DIR="${PWD}/${LLVM_BOOTSTRAP_INSTALL_DIR}" 337 echo "Building bootstrap compiler" 338 mkdir -p "${LLVM_BOOTSTRAP_DIR}" 339 pushd "${LLVM_BOOTSTRAP_DIR}" 340 if [[ ! -f ./config.status ]]; then 341 # The bootstrap compiler only needs to be able to build the real compiler, 342 # so it needs no cross-compiler output support. In general, the host 343 # compiler should be as similar to the final compiler as possible, so do 344 # keep --disable-threads & co. 345 ../llvm/configure \ 346 --enable-optimized \ 347 --enable-targets=host-only \ 348 --enable-libedit=no \ 349 --disable-threads \ 350 --disable-pthreads \ 351 --without-llvmgcc \ 352 --without-llvmgxx \ 353 --prefix="${ABS_INSTALL_DIR}" 354 fi 355 356 ${MAKE} -j"${NUM_JOBS}" 357 if [[ -n "${run_tests}" ]]; then 358 ${MAKE} check-all 359 fi 360 361 ${MAKE} install 362 if [[ -n "${gcc_toolchain}" ]]; then 363 # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap 364 # compiler can start. 365 cp -v "$(${CXX} -print-file-name=libstdc++.so.6)" \ 366 "${ABS_INSTALL_DIR}/lib/" 367 fi 368 369 popd 370 export CC="${ABS_INSTALL_DIR}/bin/clang" 371 export CXX="${ABS_INSTALL_DIR}/bin/clang++" 372 373 if [[ -n "${gcc_toolchain}" ]]; then 374 # Tell the bootstrap compiler to use a specific gcc prefix to search 375 # for standard library headers and shared object file. 376 export CFLAGS="--gcc-toolchain=${gcc_toolchain}" 377 export CXXFLAGS="--gcc-toolchain=${gcc_toolchain}" 378 fi 379 380 echo "Building final compiler" 381fi 382 383# Build clang (in a separate directory). 384# The clang bots have this path hardcoded in built/scripts/slave/compile.py, 385# so if you change it you also need to change these links. 386mkdir -p "${LLVM_BUILD_DIR}" 387pushd "${LLVM_BUILD_DIR}" 388 389# Build libc++.dylib while some bots are still on OS X 10.6. 390if [ "${OS}" = "Darwin" ]; then 391 rm -rf libcxxbuild 392 LIBCXXFLAGS="-O3 -std=c++11 -fstrict-aliasing" 393 394 # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files 395 # into different subdirectories. 396 mkdir -p libcxxbuild/libcxx 397 pushd libcxxbuild/libcxx 398 ${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXX_DIR}"/src/*.cpp 399 popd 400 401 mkdir -p libcxxbuild/libcxxabi 402 pushd libcxxbuild/libcxxabi 403 ${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXXABI_DIR}"/src/*.cpp -I"${ABS_LIBCXXABI_DIR}/include" 404 popd 405 406 pushd libcxxbuild 407 ${CC:-cc} libcxx/*.o libcxxabi/*.o -o libc++.1.dylib -dynamiclib \ 408 -nodefaultlibs -current_version 1 -compatibility_version 1 \ 409 -lSystem -install_name @executable_path/libc++.dylib \ 410 -Wl,-unexported_symbols_list,${ABS_LIBCXX_DIR}/lib/libc++unexp.exp \ 411 -Wl,-force_symbols_not_weak_list,${ABS_LIBCXX_DIR}/lib/notweak.exp \ 412 -Wl,-force_symbols_weak_list,${ABS_LIBCXX_DIR}/lib/weak.exp 413 ln -sf libc++.1.dylib libc++.dylib 414 popd 415 export LDFLAGS+="-stdlib=libc++ -L${PWD}/libcxxbuild" 416fi 417 418if [[ ! -f ./config.status ]]; then 419 ../llvm/configure \ 420 --enable-optimized \ 421 --enable-libedit=no \ 422 --disable-threads \ 423 --disable-pthreads \ 424 --without-llvmgcc \ 425 --without-llvmgxx 426fi 427 428if [[ -n "${gcc_toolchain}" ]]; then 429 # Copy in the right stdlibc++.so.6 so clang can start. 430 mkdir -p Release+Asserts/lib 431 cp -v "$(${CXX} ${CXXFLAGS} -print-file-name=libstdc++.so.6)" \ 432 Release+Asserts/lib/ 433fi 434MACOSX_DEPLOYMENT_TARGET=10.5 ${MAKE} -j"${NUM_JOBS}" 435STRIP_FLAGS= 436if [ "${OS}" = "Darwin" ]; then 437 # See http://crbug.com/256342 438 STRIP_FLAGS=-x 439 440 cp libcxxbuild/libc++.1.dylib Release+Asserts/bin 441fi 442strip ${STRIP_FLAGS} Release+Asserts/bin/clang 443popd 444 445if [[ -n "${with_android}" ]]; then 446 # Make a standalone Android toolchain. 447 ${ANDROID_NDK_DIR}/build/tools/make-standalone-toolchain.sh \ 448 --platform=android-14 \ 449 --install-dir="${LLVM_BUILD_DIR}/android-toolchain" \ 450 --system=linux-x86_64 \ 451 --stl=stlport 452 453 # Android NDK r9d copies a broken unwind.h into the toolchain, see 454 # http://crbug.com/357890 455 rm -v "${LLVM_BUILD_DIR}"/android-toolchain/include/c++/*/unwind.h 456 457 # Build ASan runtime for Android. 458 # Note: LLVM_ANDROID_TOOLCHAIN_DIR is not relative to PWD, but to where we 459 # build the runtime, i.e. third_party/llvm/projects/compiler-rt. 460 pushd "${LLVM_BUILD_DIR}" 461 ${MAKE} -C tools/clang/runtime/ \ 462 LLVM_ANDROID_TOOLCHAIN_DIR="../../../llvm-build/android-toolchain" 463 popd 464fi 465 466# Build Chrome-specific clang tools. Paths in this list should be relative to 467# tools/clang. 468# For each tool directory, copy it into the clang tree and use clang's build 469# system to compile it. 470for CHROME_TOOL_DIR in ${chrome_tools}; do 471 TOOL_SRC_DIR="${THIS_DIR}/../${CHROME_TOOL_DIR}" 472 TOOL_DST_DIR="${LLVM_DIR}/tools/clang/tools/chrome-${CHROME_TOOL_DIR}" 473 TOOL_BUILD_DIR="${LLVM_BUILD_DIR}/tools/clang/tools/chrome-${CHROME_TOOL_DIR}" 474 rm -rf "${TOOL_DST_DIR}" 475 cp -R "${TOOL_SRC_DIR}" "${TOOL_DST_DIR}" 476 rm -rf "${TOOL_BUILD_DIR}" 477 mkdir -p "${TOOL_BUILD_DIR}" 478 cp "${TOOL_SRC_DIR}/Makefile" "${TOOL_BUILD_DIR}" 479 MACOSX_DEPLOYMENT_TARGET=10.5 ${MAKE} -j"${NUM_JOBS}" -C "${TOOL_BUILD_DIR}" 480done 481 482if [[ -n "$run_tests" ]]; then 483 # Run a few tests. 484 for CHROME_TOOL_DIR in ${chrome_tools}; do 485 TOOL_SRC_DIR="${THIS_DIR}/../${CHROME_TOOL_DIR}" 486 if [[ -f "${TOOL_SRC_DIR}/tests/test.sh" ]]; then 487 "${TOOL_SRC_DIR}/tests/test.sh" "${LLVM_BUILD_DIR}/Release+Asserts" 488 fi 489 done 490 pushd "${LLVM_BUILD_DIR}" 491 ${MAKE} check-all 492 popd 493fi 494 495# After everything is done, log success for this revision. 496echo "${CLANG_AND_PLUGINS_REVISION}" > "${STAMP_FILE}" 497