1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Entry point for both build and try bots. 7 8This script is invoked from XXX, usually without arguments 9to package an SDK. It automatically determines whether 10this SDK is for mac, win, linux. 11 12The script inspects the following environment variables: 13 14BUILDBOT_BUILDERNAME to determine whether the script is run locally 15and whether it should upload an SDK to file storage (GSTORE) 16""" 17 18# pylint: disable=W0621 19 20# std python includes 21import datetime 22import glob 23import optparse 24import os 25import re 26import sys 27 28if sys.version_info < (2, 6, 0): 29 sys.stderr.write("python 2.6 or later is required run this script\n") 30 sys.exit(1) 31 32# local includes 33import buildbot_common 34import build_projects 35import build_updater 36import build_version 37import generate_notice 38import manifest_util 39import parse_dsc 40import verify_filelist 41 42from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR 43from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR 44 45# Add SDK make tools scripts to the python path. 46sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 47sys.path.append(os.path.join(NACL_DIR, 'build')) 48 49import getos 50import oshelpers 51 52BUILD_DIR = os.path.join(NACL_DIR, 'build') 53NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain') 54NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars') 55 56CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py') 57PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py') 58 59NACLPORTS_URL = 'https://chromium.googlesource.com/external/naclports.git' 60NACLPORTS_REV = '99f2417' 61 62GYPBUILD_DIR = 'gypbuild' 63 64options = None 65 66# Map of: ToolchainName: (PackageName, SDKDir). 67TOOLCHAIN_PACKAGE_MAP = { 68 'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'), 69 'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'), 70 'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'), 71 'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'), 72 'pnacl': ('pnacl_newlib', '%(platform)s_pnacl') 73 } 74 75 76def GetToolchainNaClInclude(tcname, tcpath, arch): 77 if arch == 'x86': 78 if tcname == 'pnacl': 79 return os.path.join(tcpath, 'le32-nacl', 'include') 80 return os.path.join(tcpath, 'x86_64-nacl', 'include') 81 elif arch == 'arm': 82 return os.path.join(tcpath, 'arm-nacl', 'include') 83 else: 84 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 85 86 87def GetGypGenDir(xarch): 88 if xarch == 'arm': 89 build_dir = GYPBUILD_DIR + '-arm' 90 else: 91 build_dir = GYPBUILD_DIR 92 return os.path.join(OUT_DIR, build_dir, 'Release', 'gen') 93 94 95def GetGypBuiltLib(tcname, xarch=None): 96 if tcname == 'pnacl': 97 tcname = 'pnacl_newlib' 98 if not xarch: 99 xarch = '' 100 return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch) 101 102 103def GetToolchainNaClLib(tcname, tcpath, xarch): 104 if tcname == 'pnacl': 105 return os.path.join(tcpath, 'le32-nacl', 'lib') 106 elif xarch == '32': 107 return os.path.join(tcpath, 'x86_64-nacl', 'lib32') 108 elif xarch == '64': 109 return os.path.join(tcpath, 'x86_64-nacl', 'lib') 110 elif xarch == 'arm': 111 return os.path.join(tcpath, 'arm-nacl', 'lib') 112 113 114def GetToolchainDirName(tcname, xarch): 115 if tcname == 'pnacl': 116 return '%s_%s' % (getos.GetPlatform(), tcname) 117 elif xarch == 'arm': 118 return '%s_arm_%s' % (getos.GetPlatform(), tcname) 119 else: 120 return '%s_x86_%s' % (getos.GetPlatform(), tcname) 121 122 123def GetGypToolchainLib(tcname, xarch): 124 if xarch == 'arm': 125 toolchain = xarch 126 else: 127 toolchain = tcname 128 129 tcpath = os.path.join(GetGypGenDir(xarch), 'sdk', 130 '%s_x86' % getos.GetPlatform(), 131 TOOLCHAIN_PACKAGE_MAP[toolchain][0]) 132 return GetToolchainNaClLib(tcname, tcpath, xarch) 133 134 135def GetOutputToolchainLib(pepperdir, tcname, xarch): 136 tcpath = os.path.join(pepperdir, 'toolchain', 137 GetToolchainDirName(tcname, xarch)) 138 return GetToolchainNaClLib(tcname, tcpath, xarch) 139 140 141def GetPNaClTranslatorLib(tcpath, arch): 142 if arch not in ['arm', 'x86-32', 'x86-64']: 143 buildbot_common.ErrorExit('Unknown architecture %s.' % arch) 144 return os.path.join(tcpath, 'translator', arch, 'lib') 145 146 147def BuildStepDownloadToolchains(toolchains): 148 buildbot_common.BuildStep('Running package_version.py') 149 args = [sys.executable, PKGVER, '--exclude', 'arm_trusted'] 150 if 'bionic' in toolchains: 151 build_platform = '%s_x86' % getos.GetPlatform() 152 args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')]) 153 args.append('sync') 154 buildbot_common.Run(args, cwd=NACL_DIR) 155 156 157def BuildStepCleanPepperDirs(pepperdir, pepperdir_old): 158 buildbot_common.BuildStep('Clean Pepper Dirs') 159 buildbot_common.RemoveDir(pepperdir_old) 160 buildbot_common.RemoveDir(pepperdir) 161 buildbot_common.MakeDir(pepperdir) 162 163 164def BuildStepMakePepperDirs(pepperdir, subdirs): 165 for subdir in subdirs: 166 buildbot_common.MakeDir(os.path.join(pepperdir, subdir)) 167 168TEXT_FILES = [ 169 'AUTHORS', 170 'COPYING', 171 'LICENSE', 172 'README.Makefiles', 173 'getting_started/README', 174] 175 176def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, 177 nacl_revision): 178 buildbot_common.BuildStep('Add Text Files') 179 InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES) 180 181 # Replace a few placeholders in README 182 readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read() 183 readme_text = readme_text.replace('${VERSION}', pepper_ver) 184 readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision) 185 readme_text = readme_text.replace('${CHROME_COMMIT_POSITION}', 186 build_version.ChromeCommitPosition()) 187 readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision) 188 189 # Year/Month/Day Hour:Minute:Second 190 time_format = '%Y/%m/%d %H:%M:%S' 191 readme_text = readme_text.replace('${DATE}', 192 datetime.datetime.now().strftime(time_format)) 193 194 open(os.path.join(pepperdir, 'README'), 'w').write(readme_text) 195 196 197def BuildStepUntarToolchains(pepperdir, toolchains): 198 buildbot_common.BuildStep('Untar Toolchains') 199 platform = getos.GetPlatform() 200 build_platform = '%s_x86' % platform 201 tmpdir = os.path.join(OUT_DIR, 'tc_temp') 202 buildbot_common.RemoveDir(tmpdir) 203 buildbot_common.MakeDir(tmpdir) 204 205 # Create a list of extract packages tuples, the first part should be 206 # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination 207 # directory relative to pepperdir/toolchain. 208 extract_packages = [] 209 for toolchain in toolchains: 210 toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None) 211 if toolchain_map: 212 package_name, tcname = toolchain_map 213 package_tuple = (os.path.join(build_platform, package_name), 214 tcname % {'platform': platform}) 215 extract_packages.append(package_tuple) 216 217 if extract_packages: 218 # Extract all of the packages into the temp directory. 219 package_names = [package_tuple[0] for package_tuple in extract_packages] 220 buildbot_common.Run([sys.executable, PKGVER, 221 '--packages', ','.join(package_names), 222 '--tar-dir', NACL_TOOLCHAINTARS_DIR, 223 '--dest-dir', tmpdir, 224 'extract']) 225 226 # Move all the packages we extracted to the correct destination. 227 for package_name, dest_dir in extract_packages: 228 full_src_dir = os.path.join(tmpdir, package_name) 229 full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir) 230 buildbot_common.Move(full_src_dir, full_dst_dir) 231 232 # Cleanup the temporary directory we are no longer using. 233 buildbot_common.RemoveDir(tmpdir) 234 235 236# List of toolchain headers to install. 237# Source is relative to top of Chromium tree, destination is relative 238# to the toolchain header directory. 239NACL_HEADER_MAP = { 240 'newlib': [ 241 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'), 242 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'), 243 ('native_client/src/untrusted/irt/irt.h', ''), 244 ('native_client/src/untrusted/irt/irt_dev.h', ''), 245 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 246 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'), 247 ('native_client/src/untrusted/pthread/pthread.h', ''), 248 ('native_client/src/untrusted/pthread/semaphore.h', ''), 249 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 250 ('ppapi/nacl_irt/public/irt_ppapi.h', ''), 251 ], 252 'glibc': [ 253 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'), 254 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'), 255 ('native_client/src/untrusted/irt/irt.h', ''), 256 ('native_client/src/untrusted/irt/irt_dev.h', ''), 257 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 258 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'), 259 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 260 ('ppapi/nacl_irt/public/irt_ppapi.h', ''), 261 ], 262 'bionic': [ 263 ('ppapi/nacl_irt/public/irt_ppapi.h', ''), 264 ], 265 'host': [] 266} 267 268def InstallFiles(src_root, dest_root, file_list): 269 """Copy a set of files from src_root to dest_root according 270 to the given mapping. This allows files to be copied from 271 to a location in the destination tree that is different to the 272 location in the source tree. 273 274 If the destination mapping ends with a '/' then the destination 275 basename is inherited from the the source file. 276 277 Wildcards can be used in the source list but it is not recommended 278 as this can end up adding things to the SDK unintentionally. 279 """ 280 for file_spec in file_list: 281 # The list of files to install can be a simple list of 282 # strings or a list of pairs, where each pair corresponds 283 # to a mapping from source to destination names. 284 if type(file_spec) == str: 285 src_file = dest_file = file_spec 286 else: 287 src_file, dest_file = file_spec 288 289 src_file = os.path.join(src_root, src_file) 290 291 # Expand sources files using glob. 292 sources = glob.glob(src_file) 293 if not sources: 294 sources = [src_file] 295 296 if len(sources) > 1 and not dest_file.endswith('/'): 297 buildbot_common.ErrorExit("Target file must end in '/' when " 298 "using globbing to install multiple files") 299 300 for source in sources: 301 if dest_file.endswith('/'): 302 dest = os.path.join(dest_file, os.path.basename(source)) 303 else: 304 dest = dest_file 305 dest = os.path.join(dest_root, dest) 306 if not os.path.isdir(os.path.dirname(dest)): 307 buildbot_common.MakeDir(os.path.dirname(dest)) 308 buildbot_common.CopyFile(source, dest) 309 310 311def InstallNaClHeaders(tc_dst_inc, tc_name): 312 """Copies NaCl headers to expected locations in the toolchain.""" 313 if tc_name == 'arm': 314 # arm toolchain header should be the same as the x86 newlib 315 # ones 316 tc_name = 'newlib' 317 318 InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name]) 319 320 321def MakeNinjaRelPath(path): 322 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path) 323 324 325TOOLCHAIN_LIBS = { 326 'bionic' : [ 327 'libminidump_generator.a', 328 'libnacl_dyncode.a', 329 'libnacl_exception.a', 330 'libnacl_list_mappings.a', 331 'libppapi.a', 332 ], 333 'newlib' : [ 334 'crti.o', 335 'crtn.o', 336 'libminidump_generator.a', 337 'libnacl.a', 338 'libnacl_dyncode.a', 339 'libnacl_exception.a', 340 'libnacl_list_mappings.a', 341 'libnosys.a', 342 'libppapi.a', 343 'libppapi_stub.a', 344 'libpthread.a', 345 ], 346 'glibc': [ 347 'libminidump_generator.a', 348 'libminidump_generator.so', 349 'libnacl.a', 350 'libnacl_dyncode.a', 351 'libnacl_dyncode.so', 352 'libnacl_exception.a', 353 'libnacl_exception.so', 354 'libnacl_list_mappings.a', 355 'libnacl_list_mappings.so', 356 'libppapi.a', 357 'libppapi.so', 358 'libppapi_stub.a', 359 ], 360 'pnacl': [ 361 'libminidump_generator.a', 362 'libnacl.a', 363 'libnacl_dyncode.a', 364 'libnacl_exception.a', 365 'libnacl_list_mappings.a', 366 'libnosys.a', 367 'libppapi.a', 368 'libppapi_stub.a', 369 'libpthread.a', 370 ] 371} 372 373 374def GypNinjaInstall(pepperdir, toolchains): 375 build_dir = GYPBUILD_DIR 376 ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release') 377 tools_files = [ 378 ['sel_ldr', 'sel_ldr_x86_32'], 379 ['ncval_new', 'ncval'], 380 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'], 381 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'], 382 ] 383 384 platform = getos.GetPlatform() 385 386 # TODO(binji): dump_syms doesn't currently build on Windows. See 387 # http://crbug.com/245456 388 if platform != 'win': 389 tools_files += [ 390 ['dump_syms', 'dump_syms'], 391 ['minidump_dump', 'minidump_dump'], 392 ['minidump_stackwalk', 'minidump_stackwalk'] 393 ] 394 395 tools_files.append(['sel_ldr64', 'sel_ldr_x86_64']) 396 397 if platform == 'linux': 398 tools_files.append(['nacl_helper_bootstrap', 399 'nacl_helper_bootstrap_x86_32']) 400 tools_files.append(['nacl_helper_bootstrap64', 401 'nacl_helper_bootstrap_x86_64']) 402 403 buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) 404 405 # Add .exe extensions to all windows tools 406 for pair in tools_files: 407 if platform == 'win' and not pair[0].endswith('.nexe'): 408 pair[0] += '.exe' 409 pair[1] += '.exe' 410 411 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files) 412 413 # Add ARM binaries 414 if platform == 'linux' and not options.no_arm_trusted: 415 tools_files = [ 416 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'], 417 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'], 418 ['sel_ldr', 'sel_ldr_arm'], 419 ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm'] 420 ] 421 ninja_out_dir = os.path.join(OUT_DIR, build_dir + '-arm', 'Release') 422 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files) 423 424 for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']): 425 if tc == 'pnacl': 426 xarches = (None,) 427 else: 428 xarches = ('arm', '32', '64') 429 430 for xarch in xarches: 431 if tc == 'glibc' and xarch == 'arm': 432 continue 433 434 src_dir = GetGypBuiltLib(tc, xarch) 435 dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch) 436 InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc]) 437 438 # Copy ARM newlib components to bionic 439 if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains: 440 bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch) 441 InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic']) 442 443 if tc != 'pnacl': 444 src_dir = GetGypToolchainLib(tc, xarch) 445 InstallFiles(src_dir, dst_dir, ['crt1.o']) 446 447 448def GypNinjaBuild_NaCl(rel_out_dir): 449 gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl') 450 nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp') 451 all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp') 452 453 out_dir = MakeNinjaRelPath(rel_out_dir) 454 out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm') 455 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir) 456 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm) 457 GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir) 458 459 platform = getos.GetPlatform() 460 if platform == 'win': 461 NinjaBuild('sel_ldr64', out_dir) 462 else: 463 out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64') 464 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64) 465 466 # We only need sel_ldr from the 64-bit out directory. 467 # sel_ldr needs to be renamed, so we'll call it sel_ldr64. 468 files_to_copy = [('sel_ldr', 'sel_ldr64')] 469 if platform == 'linux': 470 files_to_copy.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap64')) 471 472 for src, dst in files_to_copy: 473 buildbot_common.CopyFile( 474 os.path.join(SRC_DIR, out_dir_64, 'Release', src), 475 os.path.join(SRC_DIR, out_dir, 'Release', dst)) 476 477 478def GypNinjaBuild_Breakpad(rel_out_dir): 479 # TODO(binji): dump_syms doesn't currently build on Windows. See 480 # http://crbug.com/245456 481 if getos.GetPlatform() == 'win': 482 return 483 484 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 485 out_dir = MakeNinjaRelPath(rel_out_dir) 486 gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp') 487 build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk'] 488 GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir) 489 490 491def GypNinjaBuild_PPAPI(arch, rel_out_dir): 492 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 493 out_dir = MakeNinjaRelPath(rel_out_dir) 494 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 495 'native_client.gyp') 496 GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir) 497 498 499def GypNinjaBuild_Pnacl(rel_out_dir, target_arch): 500 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the 501 # Chromium build, and once here. When we move more of the SDK build process 502 # to gyp, we can remove this. 503 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 504 505 out_dir = MakeNinjaRelPath(rel_out_dir) 506 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src', 507 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp') 508 targets = ['aot'] 509 GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False) 510 511 512def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, 513 out_dir, force_arm_gcc=True): 514 gyp_env = dict(os.environ) 515 gyp_env['GYP_GENERATORS'] = 'ninja' 516 gyp_defines = [] 517 if options.mac_sdk: 518 gyp_defines.append('mac_sdk=%s' % options.mac_sdk) 519 if arch: 520 gyp_defines.append('target_arch=%s' % arch) 521 if arch == 'arm': 522 if getos.GetPlatform() == 'linux': 523 gyp_env['CC'] = 'arm-linux-gnueabihf-gcc' 524 gyp_env['CXX'] = 'arm-linux-gnueabihf-g++' 525 gyp_env['AR'] = 'arm-linux-gnueabihf-ar' 526 gyp_env['AS'] = 'arm-linux-gnueabihf-as' 527 gyp_env['CC_host'] = 'cc' 528 gyp_env['CXX_host'] = 'c++' 529 gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1', 530 'arm_float_abi=hard'] 531 if force_arm_gcc: 532 gyp_defines.append('nacl_enable_arm_gcc=1') 533 if options.no_arm_trusted: 534 gyp_defines.append('disable_cross_trusted=1') 535 if getos.GetPlatform() == 'mac': 536 gyp_defines.append('clang=1') 537 538 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines) 539 for key in ['GYP_GENERATORS', 'GYP_DEFINES', 'CC']: 540 value = gyp_env.get(key) 541 if value is not None: 542 print '%s="%s"' % (key, value) 543 gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)] 544 gyp_depth = '--depth=.' 545 buildbot_common.Run( 546 [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \ 547 gyp_generator_flags, 548 cwd=SRC_DIR, 549 env=gyp_env) 550 NinjaBuild(targets, out_dir) 551 552 553def NinjaBuild(targets, out_dir): 554 if type(targets) is not list: 555 targets = [targets] 556 out_config_dir = os.path.join(out_dir, 'Release') 557 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR) 558 559 560def BuildStepBuildToolchains(pepperdir, toolchains): 561 buildbot_common.BuildStep('SDK Items') 562 563 GypNinjaBuild_NaCl(GYPBUILD_DIR) 564 GypNinjaBuild_Breakpad(GYPBUILD_DIR) 565 566 platform = getos.GetPlatform() 567 newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib') 568 glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc') 569 armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib') 570 pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl') 571 bionicdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_bionic') 572 573 if set(toolchains) & set(['glibc', 'newlib']): 574 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR) 575 GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR) 576 577 if 'arm' in toolchains: 578 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm') 579 580 GypNinjaInstall(pepperdir, toolchains) 581 582 if 'newlib' in toolchains: 583 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'), 584 'newlib') 585 586 if 'glibc' in toolchains: 587 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'), 588 'glibc') 589 590 if 'arm' in toolchains: 591 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'), 592 'arm') 593 594 if 'bionic' in toolchains: 595 InstallNaClHeaders(GetToolchainNaClInclude('bionic', bionicdir, 'arm'), 596 'bionic') 597 598 if 'pnacl' in toolchains: 599 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default. 600 for arch in ('ia32', 'arm'): 601 # Fill in the latest native pnacl shim library from the chrome build. 602 build_dir = GYPBUILD_DIR + '-pnacl-' + arch 603 GypNinjaBuild_Pnacl(build_dir, arch) 604 if arch == 'ia32': 605 nacl_arches = ['x86-32', 'x86-64'] 606 elif arch == 'arm': 607 nacl_arches = ['arm'] 608 else: 609 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 610 for nacl_arch in nacl_arches: 611 release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release', 612 'gen', 'tc_pnacl_translate', 613 'lib-' + nacl_arch) 614 615 pnacl_translator_lib_dir = GetPNaClTranslatorLib(pnacldir, nacl_arch) 616 if not os.path.isdir(pnacl_translator_lib_dir): 617 buildbot_common.ErrorExit('Expected %s directory to exist.' % 618 pnacl_translator_lib_dir) 619 620 buildbot_common.CopyFile( 621 os.path.join(release_build_dir, 'libpnacl_irt_shim.a'), 622 pnacl_translator_lib_dir) 623 624 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'), 625 'newlib') 626 627 628def MakeDirectoryOrClobber(pepperdir, dirname, clobber): 629 dirpath = os.path.join(pepperdir, dirname) 630 if clobber: 631 buildbot_common.RemoveDir(dirpath) 632 buildbot_common.MakeDir(dirpath) 633 634 return dirpath 635 636 637def BuildStepUpdateHelpers(pepperdir, clobber): 638 buildbot_common.BuildStep('Update project helpers') 639 build_projects.UpdateHelpers(pepperdir, clobber=clobber) 640 641 642def BuildStepUpdateUserProjects(pepperdir, toolchains, 643 build_experimental, clobber): 644 buildbot_common.BuildStep('Update examples and libraries') 645 646 filters = {} 647 if not build_experimental: 648 filters['EXPERIMENTAL'] = False 649 if toolchains: 650 toolchains = toolchains[:] 651 652 # arm isn't a valid toolchain for build_projects 653 if 'arm' in toolchains: 654 toolchains.remove('arm') 655 656 if 'host' in toolchains: 657 toolchains.remove('host') 658 toolchains.append(getos.GetPlatform()) 659 660 filters['TOOLS'] = toolchains 661 662 # Update examples and libraries 663 filters['DEST'] = [ 664 'getting_started', 665 'examples/api', 666 'examples/demo', 667 'examples/tutorial', 668 'src' 669 ] 670 671 tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters) 672 build_projects.UpdateProjects(pepperdir, tree, clobber=clobber, 673 toolchains=toolchains) 674 675 676def BuildStepMakeAll(pepperdir, directory, step_name, 677 deps=True, clean=False, config='Debug', args=None): 678 buildbot_common.BuildStep(step_name) 679 build_projects.BuildProjectsBranch(pepperdir, directory, clean, 680 deps, config, args) 681 682 683def BuildStepBuildLibraries(pepperdir, directory): 684 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug', 685 clean=True, config='Debug') 686 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release', 687 clean=True, config='Release') 688 689 # Cleanup .pyc file generated while building libraries. Without 690 # this we would end up shipping the pyc in the SDK tarball. 691 buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc')) 692 693 694def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None): 695 # Look for LICENSE files 696 license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT') 697 698 license_files = [] 699 for root, _, files in os.walk(fileroot): 700 for filename in files: 701 if license_filenames_re.match(filename): 702 path = os.path.join(root, filename) 703 license_files.append(path) 704 705 if extra_files: 706 license_files += [os.path.join(fileroot, f) for f in extra_files] 707 print '\n'.join(license_files) 708 709 if not os.path.isabs(output_filename): 710 output_filename = os.path.join(fileroot, output_filename) 711 generate_notice.Generate(output_filename, fileroot, license_files) 712 713 714def BuildStepVerifyFilelist(pepperdir): 715 buildbot_common.BuildStep('Verify SDK Files') 716 file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list') 717 try: 718 verify_filelist.Verify(file_list_path, pepperdir) 719 print 'OK' 720 except verify_filelist.ParseException, e: 721 buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e) 722 except verify_filelist.VerifyException, e: 723 file_list_rel = os.path.relpath(file_list_path) 724 verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py' 725 verify_filelist_py = os.path.relpath(verify_filelist_py) 726 pepperdir_rel = os.path.relpath(pepperdir) 727 728 msg = """\ 729SDK verification failed: 730 731%s 732Add/remove files from %s to fix. 733 734Run: 735 ./%s %s %s 736to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel, 737 pepperdir_rel) 738 buildbot_common.ErrorExit(msg) 739 740 741def BuildStepTarBundle(pepper_ver, tarfile): 742 buildbot_common.BuildStep('Tar Pepper Bundle') 743 buildbot_common.MakeDir(os.path.dirname(tarfile)) 744 buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 745 'pepper_' + pepper_ver], cwd=NACL_DIR) 746 747 748def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile, 749 archive_url): 750 with open(tarfile, 'rb') as tarfile_stream: 751 archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( 752 tarfile_stream) 753 754 archive = manifest_util.Archive(manifest_util.GetHostOS()) 755 archive.url = archive_url 756 archive.size = archive_size 757 archive.checksum = archive_sha1 758 759 bundle = manifest_util.Bundle('pepper_' + pepper_ver) 760 bundle.revision = int(chrome_revision) 761 bundle.repath = 'pepper_' + pepper_ver 762 bundle.version = int(pepper_ver) 763 bundle.description = ( 764 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % ( 765 pepper_ver, chrome_revision, nacl_revision)) 766 bundle.stability = 'dev' 767 bundle.recommended = 'no' 768 bundle.archives = [archive] 769 return bundle 770 771 772def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision, 773 tarfile): 774 buildbot_common.BuildStep('Archive %s' % name) 775 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 776 build_version.ChromeVersion(),) 777 tarname = os.path.basename(tarfile) 778 tarfile_dir = os.path.dirname(tarfile) 779 buildbot_common.Archive(tarname, bucket_path, tarfile_dir) 780 781 # generate "manifest snippet" for this archive. 782 archive_url = GSTORE + 'nacl_sdk/%s/%s' % ( 783 build_version.ChromeVersion(), tarname) 784 bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, 785 tarfile, archive_url) 786 787 manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') 788 with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: 789 manifest_snippet_stream.write(bundle.GetDataAsString()) 790 791 buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, 792 step_link=False) 793 794 795def BuildStepArchiveSDKTools(): 796 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot. 797 builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') 798 if builder_name == 'linux-sdk-multi': 799 buildbot_common.BuildStep('Build SDK Tools') 800 build_updater.BuildUpdater(OUT_DIR) 801 802 buildbot_common.BuildStep('Archive SDK Tools') 803 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 804 build_version.ChromeVersion(),) 805 buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, 806 step_link=False) 807 buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR, 808 step_link=False) 809 810 811def BuildStepSyncNaClPorts(): 812 """Pull the pinned revision of naclports from SVN.""" 813 buildbot_common.BuildStep('Sync naclports') 814 815 # In case a previous svn checkout exists, remove it. 816 # TODO(sbc): remove this once all the build machines 817 # have removed the old checkout 818 if (os.path.exists(NACLPORTS_DIR) and 819 not os.path.exists(os.path.join(NACLPORTS_DIR, '.git'))): 820 buildbot_common.RemoveDir(NACLPORTS_DIR) 821 822 if not os.path.exists(NACLPORTS_DIR): 823 # checkout new copy of naclports 824 cmd = ['git', 'clone', NACLPORTS_URL, 'naclports'] 825 buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR)) 826 else: 827 # checkout new copy of naclports 828 buildbot_common.Run(['git', 'fetch'], cwd=NACLPORTS_DIR) 829 830 # sync to required revision 831 cmd = ['git', 'checkout', str(NACLPORTS_REV)] 832 buildbot_common.Run(cmd, cwd=NACLPORTS_DIR) 833 834 835def BuildStepBuildNaClPorts(pepper_ver, pepperdir): 836 """Build selected naclports in all configurations.""" 837 # TODO(sbc): currently naclports doesn't know anything about 838 # Debug builds so the Debug subfolders are all empty. 839 840 env = dict(os.environ) 841 env['NACL_SDK_ROOT'] = pepperdir 842 env['PEPPER_DIR'] = os.path.basename(pepperdir) # pepper_NN 843 env['NACLPORTS_NO_ANNOTATE'] = "1" 844 env['NACLPORTS_NO_UPLOAD'] = "1" 845 env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV) 846 847 build_script = 'build_tools/buildbot_sdk_bundle.sh' 848 buildbot_common.BuildStep('Build naclports') 849 850 bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 851 out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver) 852 853 # Remove the sdk_bundle directory to remove stale files from previous builds. 854 buildbot_common.RemoveDir(bundle_dir) 855 856 buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR) 857 858 # Some naclports do not include a standalone LICENSE/COPYING file 859 # so we explicitly list those here for inclusion. 860 extra_licenses = ('tinyxml/readme.txt', 861 'jpeg-8d/README', 862 'zlib-1.2.3/README') 863 src_root = os.path.join(NACLPORTS_DIR, 'out', 'build') 864 output_license = os.path.join(out_dir, 'ports', 'LICENSE') 865 GenerateNotice(src_root , output_license, extra_licenses) 866 readme = os.path.join(out_dir, 'ports', 'README') 867 oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme]) 868 869 870def BuildStepTarNaClPorts(pepper_ver, tarfile): 871 """Create tar archive containing headers and libs from naclports build.""" 872 buildbot_common.BuildStep('Tar naclports Bundle') 873 buildbot_common.MakeDir(os.path.dirname(tarfile)) 874 pepper_dir = 'pepper_%s' % pepper_ver 875 archive_dirs = [os.path.join(pepper_dir, 'ports')] 876 877 ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 878 cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile] 879 cmd += archive_dirs 880 buildbot_common.Run(cmd, cwd=NACL_DIR) 881 882 883def BuildStepBuildAppEngine(pepperdir, chrome_revision): 884 """Build the projects found in src/gonacl_appengine/src""" 885 buildbot_common.BuildStep('Build GoNaCl AppEngine Projects') 886 cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision] 887 env = dict(os.environ) 888 env['NACL_SDK_ROOT'] = pepperdir 889 env['NACLPORTS_NO_ANNOTATE'] = "1" 890 buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR) 891 892 893def main(args): 894 parser = optparse.OptionParser(description=__doc__) 895 parser.add_option('--nacl-tree-path', 896 help='Path to native client tree for bionic build.', 897 dest='nacl_tree_path') 898 parser.add_option('--qemu', help='Add qemu for ARM.', 899 action='store_true') 900 parser.add_option('--bionic', help='Add bionic build.', 901 action='store_true') 902 parser.add_option('--tar', help='Force the tar step.', 903 action='store_true') 904 parser.add_option('--archive', help='Force the archive step.', 905 action='store_true') 906 parser.add_option('--release', help='PPAPI release version.', 907 dest='release', default=None) 908 parser.add_option('--build-ports', 909 help='Build naclport bundle.', action='store_true') 910 parser.add_option('--build-app-engine', 911 help='Build AppEngine demos.', action='store_true') 912 parser.add_option('--experimental', 913 help='build experimental examples and libraries', action='store_true', 914 dest='build_experimental') 915 parser.add_option('--skip-toolchain', help='Skip toolchain untar', 916 action='store_true') 917 parser.add_option('--mac-sdk', 918 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.') 919 parser.add_option('--no-arm-trusted', action='store_true', 920 help='Disable building of ARM trusted components (sel_ldr, etc).') 921 922 # To setup bash completion for this command first install optcomplete 923 # and then add this line to your .bashrc: 924 # complete -F _optcomplete build_sdk.py 925 try: 926 import optcomplete 927 optcomplete.autocomplete(parser) 928 except ImportError: 929 pass 930 931 global options 932 options, args = parser.parse_args(args[1:]) 933 if args: 934 parser.error("Unexpected arguments: %s" % str(args)) 935 936 if options.nacl_tree_path: 937 options.bionic = True 938 toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build') 939 print 'WARNING: Building bionic toolchain from NaCl checkout.' 940 print 'This option builds bionic from the sources currently in the' 941 print 'provided NativeClient checkout, and the results instead of ' 942 print 'downloading a toolchain from the builder. This may result in a' 943 print 'NaCl SDK that can not run on ToT chrome.' 944 print 'NOTE: To clobber you will need to run toolchain_build_bionic.py' 945 print 'directly from the NativeClient checkout.' 946 print '' 947 response = raw_input("Type 'y' and hit enter to continue.\n") 948 if response != 'y' and response != 'Y': 949 print 'Aborting.' 950 return 1 951 952 # Get head version of NativeClient tree 953 buildbot_common.BuildStep('Build bionic toolchain.') 954 buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'], 955 cwd=toolchain_build) 956 else: 957 toolchain_build = None 958 959 if buildbot_common.IsSDKBuilder(): 960 options.archive = True 961 options.build_ports = True 962 # TODO(binji): re-enable app_engine build when the linux builder stops 963 # breaking when trying to git clone from github. 964 # See http://crbug.com/412969. 965 options.build_app_engine = False 966 options.tar = True 967 968 # NOTE: order matters here. This will be the order that is specified in the 969 # Makefiles; the first toolchain will be the default. 970 toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'host'] 971 972 # Changes for experimental bionic builder 973 if options.bionic: 974 toolchains.append('bionic') 975 options.build_ports = False 976 options.build_app_engine = False 977 978 print 'Building: ' + ' '.join(toolchains) 979 980 if options.archive and not options.tar: 981 parser.error('Incompatible arguments with archive.') 982 983 chrome_version = int(build_version.ChromeMajorVersion()) 984 chrome_revision = build_version.ChromeRevision() 985 nacl_revision = build_version.NaClRevision() 986 pepper_ver = str(chrome_version) 987 pepper_old = str(chrome_version - 1) 988 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 989 pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old) 990 if options.bionic: 991 tarname = 'naclsdk_bionic.tar.bz2' 992 else: 993 tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2' 994 tarfile = os.path.join(OUT_DIR, tarname) 995 996 if options.release: 997 pepper_ver = options.release 998 print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision) 999 1000 if 'NACL_SDK_ROOT' in os.environ: 1001 # We don't want the currently configured NACL_SDK_ROOT to have any effect 1002 # of the build. 1003 del os.environ['NACL_SDK_ROOT'] 1004 1005 if not options.skip_toolchain: 1006 BuildStepCleanPepperDirs(pepperdir, pepperdir_old) 1007 BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools']) 1008 BuildStepDownloadToolchains(toolchains) 1009 if options.nacl_tree_path: 1010 # Instead of untarring, copy the raw bionic toolchain 1011 not_bionic = [i for i in toolchains if i != 'bionic'] 1012 BuildStepUntarToolchains(pepperdir, not_bionic) 1013 tcname = GetToolchainDirName('bionic', 'arm') 1014 srcdir = os.path.join(toolchain_build, 'out', tcname) 1015 bionicdir = os.path.join(pepperdir, 'toolchain', tcname) 1016 oshelpers.Copy(['-r', srcdir, bionicdir]) 1017 else: 1018 BuildStepUntarToolchains(pepperdir, toolchains) 1019 1020 BuildStepBuildToolchains(pepperdir, toolchains) 1021 1022 BuildStepUpdateHelpers(pepperdir, True) 1023 BuildStepUpdateUserProjects(pepperdir, toolchains, 1024 options.build_experimental, True) 1025 1026 BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision) 1027 1028 # Ship with libraries prebuilt, so run that first. 1029 BuildStepBuildLibraries(pepperdir, 'src') 1030 GenerateNotice(pepperdir) 1031 1032 # Verify the SDK contains what we expect. 1033 if not options.bionic: 1034 BuildStepVerifyFilelist(pepperdir) 1035 1036 if options.tar: 1037 BuildStepTarBundle(pepper_ver, tarfile) 1038 1039 if options.build_ports and getos.GetPlatform() == 'linux': 1040 ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2') 1041 BuildStepSyncNaClPorts() 1042 BuildStepBuildNaClPorts(pepper_ver, pepperdir) 1043 if options.tar: 1044 BuildStepTarNaClPorts(pepper_ver, ports_tarfile) 1045 1046 if options.build_app_engine and getos.GetPlatform() == 'linux': 1047 BuildStepBuildAppEngine(pepperdir, chrome_revision) 1048 1049 if options.qemu: 1050 qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted') 1051 oshelpers.Copy(['-r', qemudir, pepperdir]) 1052 1053 # Archive on non-trybots. 1054 if options.archive: 1055 BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision, 1056 tarfile) 1057 if options.build_ports and getos.GetPlatform() == 'linux': 1058 BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision, 1059 nacl_revision, ports_tarfile) 1060 BuildStepArchiveSDKTools() 1061 1062 return 0 1063 1064 1065if __name__ == '__main__': 1066 try: 1067 sys.exit(main(sys.argv)) 1068 except KeyboardInterrupt: 1069 buildbot_common.ErrorExit('build_sdk: interrupted') 1070