1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Verifies that the build is sane.
18
19Cleans old build artifacts, configures the required environment, determines
20build goals, and invokes the build scripts.
21"""
22from __future__ import print_function
23
24import argparse
25import collections
26import datetime
27import inspect
28import os
29import shutil
30import site
31import subprocess
32import sys
33import tempfile
34import textwrap
35
36site.addsitedir(os.path.join(os.path.dirname(__file__), 'build/lib'))
37
38import build_support  # pylint: disable=import-error
39
40
41ALL_MODULES = {
42    'binutils',
43    'build',
44    'clang',
45    'cpufeatures',
46    'gabi++',
47    'gcc',
48    'gcclibs',
49    'gdbserver',
50    'gnustl',
51    'gtest',
52    'host-tools',
53    'libandroid_support',
54    'libc++',
55    'libc++abi',
56    'native_app_glue',
57    'ndk_helper',
58    'platforms',
59    'python-packages',
60    'stlport',
61    'system-stl',
62}
63
64
65class ArgParser(argparse.ArgumentParser):
66    def __init__(self):
67        super(ArgParser, self).__init__(
68            description=inspect.getdoc(sys.modules[__name__]))
69
70        self.add_argument(
71            '--arch',
72            choices=('arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'),
73            help='Build for the given architecture. Build all by default.')
74
75        package_group = self.add_mutually_exclusive_group()
76        package_group.add_argument(
77            '--package', action='store_true', dest='package', default=True,
78            help='Package the NDK when done building (default).')
79        package_group.add_argument(
80            '--no-package', action='store_false', dest='package',
81            help='Do not package the NDK when done building.')
82
83        test_group = self.add_mutually_exclusive_group()
84        test_group.add_argument(
85            '--test', action='store_true', dest='test', default=True,
86            help=textwrap.dedent("""\
87            Run host tests when finished. --package is required. Not supported
88            when targeting Windows.
89            """))
90        test_group.add_argument(
91            '--no-test', action='store_false', dest='test',
92            help='Do not run host tests when finished.')
93
94        self.add_argument(
95            '--release',
96            help='Release name. Package will be named android-ndk-RELEASE.')
97
98        self.add_argument(
99            '--system', choices=('darwin', 'linux', 'windows', 'windows64'),
100            default=build_support.get_default_host(),
101            help='Build for the given OS.')
102
103        module_group = self.add_mutually_exclusive_group()
104
105        module_group.add_argument(
106            '--module', choices=sorted(ALL_MODULES),
107            help='NDK modules to build.')
108
109        module_group.add_argument(
110            '--host-only', action='store_true',
111            help='Skip building target components.')
112
113
114def _invoke_build(script, args):
115    if args is None:
116        args = []
117    subprocess.check_call([build_support.android_path(script)] + args)
118
119
120def invoke_build(script, args=None):
121    script_path = os.path.join('build/tools', script)
122    _invoke_build(build_support.ndk_path(script_path), args)
123
124
125def invoke_external_build(script, args=None):
126    _invoke_build(build_support.android_path(script), args)
127
128
129def package_ndk(out_dir, dist_dir, args):
130    package_args = common_build_args(out_dir, dist_dir, args)
131    package_args.append(dist_dir)
132
133    if args.release is not None:
134        package_args.append('--release={}'.format(args.release))
135
136    if args.arch is not None:
137        package_args.append('--arch={}'.format(args.arch))
138
139    invoke_build('package.py', package_args)
140
141
142def test_ndk(out_dir, args):
143    release = args.release
144    if args.release is None:
145        release = datetime.date.today().strftime('%Y%m%d')
146
147    # The packaging step extracts all the modules to a known directory for
148    # packaging. This directory is not cleaned up after packaging, so we can
149    # reuse that for testing.
150    test_dir = os.path.join(out_dir, 'android-ndk-{}'.format(release))
151
152    test_env = dict(os.environ)
153    test_env['NDK'] = test_dir
154
155    abis = build_support.ALL_ABIS
156    if args.arch is not None:
157        abis = build_support.arch_to_abis(args.arch)
158
159    results = {}
160    for abi in abis:
161        cmd = [
162            'python', build_support.ndk_path('tests/run-all.py'),
163            '--abi', abi, '--suite', 'build'
164        ]
165        print('Running tests: {}'.format(' '.join(cmd)))
166        result = subprocess.call(cmd, env=test_env)
167        results[abi] = result == 0
168
169    print('Results:')
170    for abi, result in results.iteritems():
171        print('{}: {}'.format(abi, 'PASS' if result else 'FAIL'))
172    return all(results.values())
173
174
175def common_build_args(out_dir, dist_dir, args):
176    build_args = ['--out-dir={}'.format(out_dir)]
177    build_args = ['--dist-dir={}'.format(dist_dir)]
178    build_args.append('--host={}'.format(args.system))
179    return build_args
180
181
182def fixup_toolchain_triple(toolchain):
183    """Maps toolchain names to their proper triple.
184
185    The x86 toolchains are named stupidly and aren't a proper triple.
186    """
187    return {
188        'x86': 'i686-linux-android',
189        'x86_64': 'x86_64-linux-android',
190    }.get(toolchain, toolchain)
191
192
193def get_binutils_files(triple, has_gold, is_windows):
194    files = [
195        'ld.bfd',
196        'nm',
197        'as',
198        'objcopy',
199        'strip',
200        'objdump',
201        'ld',
202        'ar',
203        'ranlib',
204    ]
205
206    if has_gold:
207        files.append('ld.gold')
208
209    if is_windows:
210        files = [f + '.exe' for f in files]
211
212    # binutils programs get installed to two locations:
213    # 1: $INSTALL_DIR/bin/$TRIPLE-$PROGRAM
214    # 2: $INSTALL_DIR/$TRIPLE/bin/$PROGRAM
215    #
216    # We need to copy both.
217
218    prefixed_files = []
219    for file_name in files:
220        prefixed_name = '-'.join([triple, file_name])
221        prefixed_files.append(os.path.join('bin', prefixed_name))
222
223    dir_prefixed_files = []
224    for file_name in files:
225        dir_prefixed_files.append(os.path.join(triple, 'bin', file_name))
226
227    ldscripts_dir = os.path.join(triple, 'lib/ldscripts')
228    return prefixed_files + dir_prefixed_files + [ldscripts_dir]
229
230
231def install_file(file_name, src_dir, dst_dir):
232    src_file = os.path.join(src_dir, file_name)
233    dst_file = os.path.join(dst_dir, file_name)
234
235    print('Copying {} to {}...'.format(src_file, dst_file))
236    if os.path.isdir(src_file):
237        _install_dir(src_file, dst_file)
238    elif os.path.islink(src_file):
239        _install_symlink(src_file, dst_file)
240    else:
241        _install_file(src_file, dst_file)
242
243
244def _install_dir(src_dir, dst_dir):
245    parent_dir = os.path.normpath(os.path.join(dst_dir, '..'))
246    if not os.path.exists(parent_dir):
247        os.makedirs(parent_dir)
248    shutil.copytree(src_dir, dst_dir, symlinks=True)
249
250
251def _install_symlink(src_file, dst_file):
252    dirname = os.path.dirname(dst_file)
253    if not os.path.exists(dirname):
254        os.makedirs(dirname)
255    link_target = os.readlink(src_file)
256    os.symlink(link_target, dst_file)
257
258
259def _install_file(src_file, dst_file):
260    dirname = os.path.dirname(dst_file)
261    if not os.path.exists(dirname):
262        os.makedirs(dirname)
263    # copy2 is just copy followed by copystat (preserves file metadata).
264    shutil.copy2(src_file, dst_file)
265
266
267def pack_binutils(arch, host_tag, out_dir, binutils_path):
268    archive_name = '-'.join(['binutils', arch, host_tag])
269    build_support.make_package(archive_name, binutils_path, out_dir)
270
271
272def get_prebuilt_gcc(host, arch):
273    tag = build_support.host_to_tag(host)
274    system_subdir = 'prebuilts/ndk/current/toolchains/{}'.format(tag)
275    system_path = build_support.android_path(system_subdir)
276    toolchain = build_support.arch_to_toolchain(arch)
277    toolchain_dir = toolchain + '-4.9'
278    return os.path.join(system_path, toolchain_dir)
279
280
281def build_binutils(out_dir, dist_dir, args):
282    print('Extracting binutils package from GCC...')
283
284    arches = build_support.ALL_ARCHITECTURES
285    if args.arch is not None:
286        arches = [args.arch]
287
288    host_tag = build_support.host_to_tag(args.system)
289
290    for arch in arches:
291        toolchain = build_support.arch_to_toolchain(arch)
292        toolchain_path = get_prebuilt_gcc(args.system, arch)
293
294        triple = fixup_toolchain_triple(toolchain)
295
296        install_dir = os.path.join(out_dir, 'binutils', triple)
297        if os.path.exists(install_dir):
298            shutil.rmtree(install_dir)
299        os.makedirs(install_dir)
300
301        has_gold = True
302        if host_tag == 'windows':
303            # Note: 64-bit Windows is fine.
304            has_gold = False
305        if arch in ('mips', 'mips64'):
306            has_gold = False
307
308        is_windows = host_tag.startswith('windows')
309        for file_name in get_binutils_files(triple, has_gold, is_windows):
310            install_file(file_name, toolchain_path, install_dir)
311
312        license_path = build_support.android_path(
313            'toolchain/binutils/binutils-2.25/COPYING')
314        shutil.copy2(license_path, os.path.join(install_dir, 'NOTICE'))
315
316        pack_binutils(arch, host_tag, dist_dir, install_dir)
317
318
319def build_clang(out_dir, dist_dir, args):
320    print('Building Clang...')
321    invoke_build('build-llvm.py', common_build_args(out_dir, dist_dir, args))
322
323
324def build_gcc(out_dir, dist_dir, args):
325    print('Building GCC...')
326    build_args = common_build_args(out_dir, dist_dir, args)
327    if args.arch is not None:
328        build_args.append('--arch={}'.format(args.arch))
329    invoke_build('build-gcc.py', build_args)
330
331
332def build_gcc_libs(out_dir, dist_dir, args):
333    print('Packaging GCC libs...')
334
335    arches = build_support.ALL_ARCHITECTURES
336    if args.arch is not None:
337        arches = [args.arch]
338
339    for arch in arches:
340        toolchain = build_support.arch_to_toolchain(arch)
341        triple = fixup_toolchain_triple(toolchain)
342        libgcc_subdir = 'lib/gcc/{}/4.9'.format(triple)
343        is64 = arch.endswith('64')
344        libatomic_subdir = '{}/lib{}'.format(triple, '64' if is64 else '')
345
346        lib_names = [
347            (libatomic_subdir, 'libatomic.a'),
348            (libgcc_subdir, 'libgcc.a'),
349        ]
350
351        lib_dirs = ['']
352        if arch == 'arm':
353            lib_dirs += [
354                'armv7-a',
355                'armv7-a/hard',
356                'armv7-a/thumb',
357                'armv7-a/thumb/hard',
358                'thumb',
359            ]
360
361        libs = []
362        for lib_dir in lib_dirs:
363            for subdir, lib in lib_names:
364                libs.append((subdir, os.path.join(lib_dir, lib)))
365
366        install_dir = os.path.join(out_dir, 'gcclibs', triple)
367        if os.path.exists(install_dir):
368            shutil.rmtree(install_dir)
369        os.makedirs(install_dir)
370
371        # These are target libraries, so the OS we use here is not
372        # important. We explicitly use Linux because for whatever reason
373        # the Windows aarch64 toolchain doesn't include libatomic.
374        gcc_path = get_prebuilt_gcc('linux', arch)
375        for gcc_subdir, lib in libs:
376            src = os.path.join(gcc_path, gcc_subdir, lib)
377            dst = os.path.join(install_dir, lib)
378            dst_dir = os.path.dirname(dst)
379            if not os.path.exists(dst_dir):
380                os.makedirs(dst_dir)
381            shutil.copy2(src, dst)
382
383            shutil.copy2(
384                os.path.join(gcc_path, 'NOTICE'),
385                os.path.join(install_dir, 'NOTICE'))
386
387        archive_name = os.path.join('gcclibs-' + arch)
388        build_support.make_package(archive_name, install_dir, dist_dir)
389
390
391def build_host_tools(out_dir, dist_dir, args):
392    build_args = common_build_args(out_dir, dist_dir, args)
393
394    print('Building ndk-stack...')
395    invoke_external_build(
396        'ndk/sources/host-tools/ndk-stack/build.py', build_args)
397
398    print('Building ndk-depends...')
399    invoke_external_build(
400        'ndk/sources/host-tools/ndk-depends/build.py', build_args)
401
402    print('Building awk...')
403    invoke_external_build(
404        'ndk/sources/host-tools/nawk-20071023/build.py', build_args)
405
406    print('Building make...')
407    invoke_external_build(
408        'ndk/sources/host-tools/make-3.81/build.py', build_args)
409
410    if args.system in ('windows', 'windows64'):
411        print('Building toolbox...')
412        invoke_external_build(
413            'ndk/sources/host-tools/toolbox/build.py', build_args)
414
415    print('Building Python...')
416    invoke_external_build('toolchain/python/build.py', build_args)
417
418    print('Building GDB...')
419    invoke_external_build('toolchain/gdb/build.py', build_args)
420
421    print('Building YASM...')
422    invoke_external_build('toolchain/yasm/build.py', build_args)
423
424    package_host_tools(out_dir, dist_dir, args.system)
425
426
427def merge_license_files(output_path, files):
428    licenses = []
429    for license_path in files:
430        with open(license_path) as license_file:
431            licenses.append(license_file.read())
432
433    with open(output_path, 'w') as output_file:
434        output_file.write('\n'.join(licenses))
435
436
437def package_host_tools(out_dir, dist_dir, host):
438    packages = [
439        'gdb-multiarch-7.10',
440        'ndk-awk',
441        'ndk-depends',
442        'ndk-make',
443        'ndk-python',
444        'ndk-stack',
445        'ndk-yasm',
446    ]
447
448    files = [
449        'ndk-gdb',
450        'ndk-gdb.cmd',
451        'ndk-gdb.py',
452    ]
453
454    if host in ('windows', 'windows64'):
455        packages.append('toolbox')
456
457    host_tag = build_support.host_to_tag(host)
458
459    package_names = [p + '-' + host_tag + '.tar.bz2' for p in packages]
460    for package_name in package_names:
461        package_path = os.path.join(out_dir, package_name)
462        subprocess.check_call(['tar', 'xf', package_path, '-C', out_dir])
463
464    for f in files:
465        shutil.copy2(f, os.path.join(out_dir, 'host-tools/bin'))
466
467    merge_license_files(os.path.join(out_dir, 'host-tools/NOTICE'), [
468        build_support.android_path('toolchain/gdb/gdb-7.10/COPYING'),
469        build_support.ndk_path('sources/host-tools/nawk-20071023/NOTICE'),
470        build_support.ndk_path('sources/host-tools/ndk-depends/NOTICE'),
471        build_support.ndk_path('sources/host-tools/make-3.81/COPYING'),
472        build_support.android_path(
473            'toolchain/python/Python-2.7.5/LICENSE'),
474        build_support.ndk_path('sources/host-tools/ndk-stack/NOTICE'),
475        build_support.ndk_path('sources/host-tools/toolbox/NOTICE'),
476        build_support.android_path('toolchain/yasm/COPYING'),
477        build_support.android_path('toolchain/yasm/BSD.txt'),
478        build_support.android_path('toolchain/yasm/Artistic.txt'),
479        build_support.android_path('toolchain/yasm/GNU_GPL-2.0'),
480        build_support.android_path('toolchain/yasm/GNU_LGPL-2.0'),
481    ])
482
483    package_name = 'host-tools-' + host_tag
484    path = os.path.join(out_dir, 'host-tools')
485    build_support.make_package(package_name, path, dist_dir)
486
487
488def build_gdbserver(out_dir, dist_dir, args):
489    print('Building gdbserver...')
490    build_args = common_build_args(out_dir, dist_dir, args)
491    if args.arch is not None:
492        build_args.append('--arch={}'.format(args.arch))
493    invoke_build('build-gdbserver.py', build_args)
494
495
496def _build_stl(out_dir, dist_dir, args, stl):
497    build_args = common_build_args(out_dir, dist_dir, args)
498    if args.arch is not None:
499        build_args.append('--arch={}'.format(args.arch))
500    script = 'ndk/sources/cxx-stl/{}/build.py'.format(stl)
501    invoke_external_build(script, build_args)
502
503
504def build_gnustl(out_dir, dist_dir, args):
505    print('Building gnustl...')
506    _build_stl(out_dir, dist_dir, args, 'gnu-libstdc++')
507
508
509def build_libcxx(out_dir, dist_dir, args):
510    print('Building libc++...')
511    _build_stl(out_dir, dist_dir, args, 'llvm-libc++')
512
513
514def build_stlport(out_dir, dist_dir, args):
515    print('Building stlport...')
516    _build_stl(out_dir, dist_dir, args, 'stlport')
517
518
519def build_platforms(out_dir, dist_dir, args):
520    print('Building platforms...')
521    build_args = common_build_args(out_dir, dist_dir, args)
522    invoke_build('build-platforms.py', build_args)
523
524
525def build_cpufeatures(_, dist_dir, __):
526    path = build_support.ndk_path('sources/android/cpufeatures')
527    build_support.make_package('cpufeatures', path, dist_dir)
528
529
530def build_native_app_glue(_, dist_dir, __):
531    path = build_support.android_path(
532        'development/ndk/sources/android/native_app_glue')
533    build_support.make_package('native_app_glue', path, dist_dir)
534
535
536def build_ndk_helper(_, dist_dir, __):
537    path = build_support.android_path(
538        'development/ndk/sources/android/ndk_helper')
539    build_support.make_package('ndk_helper', path, dist_dir)
540
541
542def build_gtest(_, dist_dir, __):
543    path = build_support.ndk_path('sources/third_party/googletest')
544    build_support.make_package('gtest', path, dist_dir)
545
546
547def build_build(_, dist_dir, __):
548    path = build_support.ndk_path('build')
549    build_support.make_package('build', path, dist_dir)
550
551
552def build_python_packages(_, dist_dir, __):
553    # Stage the files in a temporary directory to make things easier.
554    temp_dir = tempfile.mkdtemp()
555    try:
556        path = os.path.join(temp_dir, 'python-packages')
557        shutil.copytree(
558            build_support.android_path('development/python-packages'), path)
559        build_support.make_package('python-packages', path, dist_dir)
560    finally:
561        shutil.rmtree(temp_dir)
562
563
564def build_gabixx(_out_dir, dist_dir, _args):
565    print('Building gabi++...')
566    path = build_support.ndk_path('sources/cxx-stl/gabi++')
567    build_support.make_package('gabixx', path, dist_dir)
568
569
570def build_system_stl(_out_dir, dist_dir, _args):
571    print('Building system-stl...')
572    path = build_support.ndk_path('sources/cxx-stl/system')
573    build_support.make_package('system-stl', path, dist_dir)
574
575
576def build_libandroid_support(_out_dir, dist_dir, _args):
577    print('Building libandroid_support...')
578    path = build_support.ndk_path('sources/android/support')
579    build_support.make_package('libandroid_support', path, dist_dir)
580
581
582def build_libcxxabi(_out_dir, dist_dir, _args):
583    print('Building libc++abi...')
584    path = build_support.ndk_path('sources/cxx-stl/llvm-libc++abi')
585    build_support.make_package('libcxxabi', path, dist_dir)
586
587
588def main():
589    parser = ArgParser()
590    args = parser.parse_args()
591
592    if args.module is None:
593        modules = ALL_MODULES
594    else:
595        modules = {args.module}
596
597    if args.host_only:
598        modules = {
599            'clang',
600            'gcc',
601            'host-tools',
602        }
603
604    required_package_modules = ALL_MODULES
605    if args.package and required_package_modules <= modules:
606        do_package = True
607    else:
608        do_package = False
609
610    # TODO(danalbert): wine?
611    # We're building the Windows packages from Linux, so we can't actually run
612    # any of the tests from here.
613    if args.system.startswith('windows') or not do_package:
614        args.test = False
615
616    # Disable buffering on stdout so the build output doesn't hide all of our
617    # "Building..." messages.
618    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
619
620    os.chdir(os.path.dirname(os.path.realpath(__file__)))
621
622    # Set ANDROID_BUILD_TOP.
623    if 'ANDROID_BUILD_TOP' not in os.environ:
624        os.environ['ANDROID_BUILD_TOP'] = os.path.realpath('..')
625
626    out_dir = build_support.get_out_dir()
627    dist_dir = build_support.get_dist_dir(out_dir)
628
629    print('Cleaning up...')
630    invoke_build('dev-cleanup.sh')
631
632    module_builds = collections.OrderedDict([
633        ('binutils', build_binutils),
634        ('build', build_build),
635        ('clang', build_clang),
636        ('cpufeatures', build_cpufeatures),
637        ('gabi++', build_gabixx),
638        ('gcc', build_gcc),
639        ('gcclibs', build_gcc_libs),
640        ('gdbserver', build_gdbserver),
641        ('gnustl', build_gnustl),
642        ('gtest', build_gtest),
643        ('host-tools', build_host_tools),
644        ('libandroid_support', build_libandroid_support),
645        ('libc++', build_libcxx),
646        ('libc++abi', build_libcxxabi),
647        ('native_app_glue', build_native_app_glue),
648        ('ndk_helper', build_ndk_helper),
649        ('platforms', build_platforms),
650        ('python-packages', build_python_packages),
651        ('stlport', build_stlport),
652        ('system-stl', build_system_stl),
653    ])
654
655    print('Building modules: {}'.format(' '.join(modules)))
656    for module in modules:
657        module_builds[module](out_dir, dist_dir, args)
658
659    if do_package:
660        package_ndk(out_dir, dist_dir, args)
661
662    if args.test:
663        result = test_ndk(out_dir, args)
664        sys.exit(0 if result else 1)
665
666
667if __name__ == '__main__':
668    main()
669