1#!/bin/sh
2#
3# Copyright (C) 2009 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17#  This shell script is used to download the sources of the Android NDK toolchain
18#  from the git server at android.googlesource.com and package them in a nice tarball
19#  that can later be used with the 'built-toolchain.sh' script.
20#
21
22# include common function and variable definitions
23. `dirname $0`/prebuilt-common.sh
24PROGDIR=`dirname $0`
25PROGDIR=`cd $PROGDIR && pwd`
26
27# the default branch to use
28BRANCH=master
29register_var_option "--branch=<name>" BRANCH "Specify release branch"
30
31# the default release name (use today's date)
32if [ "$TOOLCHAIN_GIT_DATE" -a "$TOOLCHAIN_GIT_DATE" != "now" ] ; then
33    RELEASE=`echo $TOOLCHAIN_GIT_DATE | sed -e 's!-!!g'`
34else
35    RELEASE=`date +%Y%m%d`
36fi
37register_var_option "--release=<name>" RELEASE "Specify release name"
38
39GIT_DATE=$TOOLCHAIN_GIT_DATE
40register_var_option "--git-date=<date>" GIT_DATE "Only sources that existed until specified <date>"
41
42GITCMD=git
43register_var_option "--git=<executable>" GITCMD "Use this version of the git tool"
44
45OPTION_GIT_BASE=https://android.googlesource.com/toolchain
46register_var_option "--git-base=<git-uri>" OPTION_GIT_BASE "Use specific git repository base"
47
48OPTION_GIT_REFERENCE=
49register_var_option "--git-reference=<path>" OPTION_GIT_REFERENCE "Use local git reference"
50
51OPTION_PACKAGE=no
52register_var_option "--package" OPTION_PACKAGE "Create source package in /tmp/ndk-$USER"
53
54OPTION_NO_PATCHES=no
55register_var_option "--no-patches" OPTION_NO_PATCHES "Do not patch sources"
56
57LLVM_VERSION_LIST=$DEFAULT_LLVM_VERSION_LIST
58register_var_option "--llvm-version-list=<vers>" LLVM_VERSION_LIST "List of LLVM release versions"
59
60LLVM_URL=$DEFAULT_LLVM_URL
61register_var_option "--llvm-url=<url>" LLVM_URL "URL to download LLVM tar archive"
62
63PROGRAM_PARAMETERS="<src-dir>"
64PROGRAM_DESCRIPTION=\
65"Download the NDK toolchain sources from android.googlesource.com into <src-dir>.
66You will need to run this script before being able to rebuilt the NDK toolchain
67binaries from scratch with build/tools/build-gcc.sh."
68
69if [ -n "$TOOLCHAIN_GIT_DATE" ] ; then
70  PROGRAM_DESCRIPTION="$PROGRAM_DESCRIPTION\
71
72
73By default, this script will download sources from android.googlesource.com that
74correspond to the date of $TOOLCHAIN_GIT_DATE. If you want to use the tip of
75tree, use '--git-date=now' instead.
76
77If you don't want to use the official servers, use --git-base=<path> to
78download the sources from another set of git repostories.
79
80You can also speed-up the download if you maintain a local copy of the
81toolchain repositories on your machine. Use --git-reference=<path> to
82specify the base path that contains all copies, and its subdirectories will
83be used as git clone shared references.
84"
85
86fi
87
88extract_parameters "$@"
89
90# Check that 'git' works
91$GITCMD --version > /dev/null 2>&1
92if [ $? != 0 ] ; then
93    echo "The git tool doesn't seem to work. Please check $GITCMD"
94    exit 1
95fi
96log "Git seems to work ok."
97
98SRC_DIR="$PARAMETERS"
99if [ -z "$SRC_DIR" -a "$OPTION_PACKAGE" = no ] ; then
100    echo "ERROR: You need to provide a <src-dir> parameter or use the --package option!"
101    exit 1
102fi
103
104if [ -n "$SRC_DIR" ] ; then
105    mkdir -p $SRC_DIR
106    SRC_DIR=`cd $SRC_DIR && pwd`
107    log "Using target source directory: $SRC_DIR"
108fi
109
110# Normalize the parameters
111LLVM_VERSION_LIST=$(commas_to_spaces $LLVM_VERSION_LIST)
112
113# Create temp directory where everything will be copied first
114#
115PKGNAME=android-ndk-toolchain-$RELEASE
116TMPDIR=$NDK_TMPDIR/$PKGNAME
117log "Creating temporary directory $TMPDIR"
118rm -rf $TMPDIR && mkdir $TMPDIR
119fail_panic "Could not create temporary directory: $TMPDIR"
120
121# prefix used for all clone operations
122GITPREFIX="$OPTION_GIT_BASE"
123dump "Using git clone prefix: $GITPREFIX"
124
125GITREFERENCE=
126if [ -n "$OPTION_GIT_REFERENCE" ] ; then
127    GITREFERENCE=$OPTION_GIT_REFERENCE
128    if [ ! -d "$GITREFERENCE" -o ! -d "$GITREFERENCE/build" ]; then
129        echo "ERROR: Invalid reference repository directory path: $GITREFERENCE"
130        exit 1
131    fi
132    dump "Using git clone reference: $GITREFERENCE"
133fi
134
135# Clone a given toolchain git repository
136# $1: repository name, relative to $GITPREFIX (e.g. 'gcc')
137#
138toolchain_clone ()
139{
140    local GITFLAGS
141    GITFLAGS="--no-checkout"
142    if [ "$GITREFERENCE" ]; then
143        GITFLAGS=$GITFLAGS" --shared --reference $GITREFERENCE/$1"
144    fi
145    dump "Cloning git repository for toolchain/$1"
146    if [ -d "$GITPREFIX/$1" ]; then
147        run ln -s "$GITPREFIX/$1" $CLONE_DIR/$1.git
148    else
149        log "cloning $GITPREFIX/$1.git"
150        (cd $CLONE_DIR && run $GITCMD clone $GITFLAGS $GITPREFIX/$1.git)
151    fi
152    fail_panic "Could not clone $GITPREFIX/$1.git ?"
153}
154
155# Checkout sources from a git clone created with toolchain_clone
156# $1: sub-directory
157# $2: branch (e.g. 'master')
158# $3: repository/clone name (e.g. 'gcc')
159# $4+: sub-path to extract, relative to clone top-level (e.g. 'gcc-4.6')
160#
161toolchain_checkout ()
162{
163    local SUBDIR=$1
164    local BRANCH=$2
165    local NAME=$3
166    shift ; shift ; shift
167    local GITOPTS="--git-dir=$CLONE_DIR/$NAME/.git"
168    log "Checking out $BRANCH branch of $NAME.git: $@"
169    local REVISION=origin/$BRANCH
170    if [ -n "$GIT_DATE" ] ; then
171        REVISION=`$GITCMD $GITOPTS rev-list -n 1 --until="$GIT_DATE" $REVISION`
172    fi
173    (mkdir -p $TMPDIR/$SUBDIR/$NAME && cd $TMPDIR/$SUBDIR/$NAME && run $GITCMD $GITOPTS checkout $REVISION "$@")
174    fail_panic "Could not checkout $NAME / $@ ?"
175    if [ "$BRANCH" = "master" ]; then
176        BRANCH=
177    else
178        BRANCH="($BRANCH)"
179    fi
180    (printf "%-38s " "toolchain/$NAME.git $BRANCH"; $GITCMD $GITOPTS log -1 --format=oneline $REVISION) >> $SOURCES_LIST
181}
182
183cd $TMPDIR
184
185CLONE_DIR=$TMPDIR/git
186mkdir -p $CLONE_DIR
187
188SOURCES_LIST=$(pwd)/SOURCES
189rm -f $SOURCES_LIST && touch $SOURCES_LIST
190
191toolchain_clone build
192toolchain_clone gmp
193toolchain_clone mpfr
194toolchain_clone mpc
195toolchain_clone cloog
196toolchain_clone isl
197toolchain_clone ppl
198toolchain_clone expat
199toolchain_clone binutils
200toolchain_clone gcc
201toolchain_clone gdb
202toolchain_clone python
203toolchain_clone perl
204toolchain_clone clang
205toolchain_clone llvm
206toolchain_clone compiler-rt
207toolchain_clone mclinker
208toolchain_clone yasm
209
210toolchain_checkout "" $BRANCH build .
211toolchain_checkout "" $BRANCH gmp .
212toolchain_checkout "" $BRANCH mpfr .
213toolchain_checkout "" $BRANCH mpc .
214toolchain_checkout "" $BRANCH cloog .
215toolchain_checkout "" $BRANCH isl .
216toolchain_checkout "" $BRANCH ppl .
217toolchain_checkout "" $BRANCH expat .
218toolchain_checkout "" $BRANCH binutils binutils-2.21 binutils-2.23 binutils-2.24
219toolchain_checkout "" $BRANCH gcc gcc-4.6 gcc-4.8 gcc-4.9
220toolchain_checkout "" $BRANCH gdb gdb-7.3.x gdb-7.6 gdb-7.7
221toolchain_checkout "" $BRANCH python Python-2.7.5
222toolchain_checkout "" $BRANCH perl perl-5.16.2
223toolchain_checkout "" $BRANCH mclinker .
224toolchain_checkout "" $BRANCH yasm .
225
226for LLVM_VERSION in $LLVM_VERSION_LIST; do
227    # Check-out and Adjust directory structure a bit
228    # 1. Create symbolic link for clang which is always built
229    # 2. Create symbolic link for compiler-rt too
230    # 3. Move tools/polly up to be a sibling of llvm and clang.  Script build-llvm.sh
231    #    will only create symbolic link when --with-polly is specified.
232    LLVM_VERSION_NO_DOT=$(echo $LLVM_VERSION | sed -e 's!\.!!g')
233    LLVM_BRANCH="release_$LLVM_VERSION_NO_DOT"
234    toolchain_checkout "llvm-$LLVM_VERSION" $LLVM_BRANCH clang .
235    toolchain_checkout "llvm-$LLVM_VERSION" $LLVM_BRANCH llvm .
236    (cd "$TMPDIR/llvm-$LLVM_VERSION/llvm" && \
237        ln -s ../../clang tools && \
238        test -d tools/polly && mv tools/polly ..)
239    if [ "$LLVM_VERSION" != "3.1" ]; then
240        # compiler-rt only exists on and after 3.2
241        toolchain_checkout "llvm-$LLVM_VERSION" $LLVM_BRANCH compiler-rt .
242    fi
243    # In polly/utils/cloog_src, touch Makefile.in, aclocal.m4, and configure to
244    # make sure they are not regenerated.
245    (test -d "$TMPDIR/llvm-$LLVM_VERSION/polly" && \
246     cd "$TMPDIR/llvm-$LLVM_VERSION/polly" && \
247        find . -name "Makefile.in" -exec touch {} \; && \
248        find . -name "aclocal.m4" -exec touch {} \; && \
249        find . -name "configure" -exec touch {} \; )
250done
251
252# Patch the toolchain sources
253if [ "$OPTION_NO_PATCHES" != "yes" ]; then
254    PATCHES_DIR="$PROGDIR/toolchain-patches"
255    if [ -d "$PATCHES_DIR" ] ; then
256        dump "Patching toolchain sources"
257        run $PROGDIR/patch-sources.sh $FLAGS $TMPDIR $PATCHES_DIR
258        if [ $? != 0 ] ; then
259            dump "ERROR: Could not patch sources."
260            exit 1
261        fi
262    fi
263fi
264
265# remove all info files from the toolchain sources
266# they create countless little problems during the build
267# if you don't have exactly the configuration expected by
268# the scripts.
269#
270find $TMPDIR -type f -a -name "*.info" ! -name sysroff.info -print0 | xargs -0 rm -f
271
272if [ $OPTION_PACKAGE = "yes" ] ; then
273    # create the package
274    PACKAGE=/tmp/ndk-$USER/$PKGNAME.tar.bz2
275    dump "Creating package archive $PACKAGE"
276    pack_archive "$PACKAGE" "$TMPDIR" "."
277    fail_panic "Could not package toolchain source archive ?. See $TMPLOG"
278    dump "Toolchain sources downloaded and packaged succesfully at $PACKAGE"
279else
280    # copy sources to <src-dir>
281    SRC_DIR=`cd $SRC_DIR && pwd`
282    rm -rf $SRC_DIR && mkdir -p $SRC_DIR
283    fail_panic "Could not create target source directory: $SRC_DIR"
284    cp $SOURCES_LIST $SRC_DIR/SOURCES
285    fail_panic "Could not copy $SOURCES_LIST to $SRC_DIR"
286    mv "$TMPDIR"/* "$SRC_DIR"  #copy_directory "$TMPDIR" "$SRC_DIR"
287    fail_panic "Could not move to target source directory: $TMPDIR -> $SRC_DIR"
288    dump "Toolchain sources downloaded and copied to $SRC_DIR"
289fi
290
291dump "Cleaning up..."
292rm -rf $TMPDIR
293rm -f $TMPLOG
294dump "Done."
295