build_sdk.py revision 5c02ac1a9c1b504631c0a3d2b6e737b5d738bae1
1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Entry point for both build and try bots. 7 8This script is invoked from XXX, usually without arguments 9to package an SDK. It automatically determines whether 10this SDK is for mac, win, linux. 11 12The script inspects the following environment variables: 13 14BUILDBOT_BUILDERNAME to determine whether the script is run locally 15and whether it should upload an SDK to file storage (GSTORE) 16""" 17 18# pylint: disable=W0621 19 20# std python includes 21import datetime 22import glob 23import optparse 24import os 25import re 26import sys 27 28if sys.version_info < (2, 6, 0): 29 sys.stderr.write("python 2.6 or later is required run this script\n") 30 sys.exit(1) 31 32# local includes 33import buildbot_common 34import build_projects 35import build_updater 36import build_version 37import generate_notice 38import manifest_util 39import parse_dsc 40import verify_filelist 41 42from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR 43from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR 44 45# Add SDK make tools scripts to the python path. 46sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 47sys.path.append(os.path.join(NACL_DIR, 'build')) 48 49import getos 50import oshelpers 51 52CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') 53 54NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src' 55NACLPORTS_REV = 1226 56 57GYPBUILD_DIR = 'gypbuild' 58 59options = None 60 61 62def GetGlibcToolchain(): 63 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 64 tcname = 'toolchain_%s_x86.tar.bz2' % getos.GetPlatform() 65 return os.path.join(tcdir, tcname) 66 67 68def GetNewlibToolchain(): 69 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 70 tcname = 'naclsdk_%s_x86.tgz' % getos.GetPlatform() 71 return os.path.join(tcdir, tcname) 72 73 74def GetBionicToolchain(): 75 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 76 tcname = 'naclsdk_%s_arm_bionic.tgz' % getos.GetPlatform() 77 return os.path.join(tcdir, tcname) 78 79 80def GetPNaClToolchain(): 81 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 82 tcname = 'naclsdk_pnacl_%s_x86.tgz' % getos.GetPlatform() 83 return os.path.join(tcdir, tcname) 84 85 86def GetToolchainNaClInclude(tcname, tcpath, arch): 87 if arch == 'x86': 88 if tcname == 'pnacl': 89 return os.path.join(tcpath, 'sdk', 'include') 90 return os.path.join(tcpath, 'x86_64-nacl', 'include') 91 elif arch == 'arm': 92 return os.path.join(tcpath, 'arm-nacl', 'include') 93 else: 94 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 95 96 97def GetGypGenDir(xarch): 98 if xarch == 'arm': 99 build_dir = GYPBUILD_DIR + '-arm' 100 else: 101 build_dir = GYPBUILD_DIR 102 return os.path.join(OUT_DIR, build_dir, 'Release', 'gen') 103 104 105def GetGypBuiltLib(tcname, xarch=None): 106 if tcname == 'pnacl': 107 tcname = 'pnacl_newlib' 108 if not xarch: 109 xarch = '' 110 return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch) 111 112 113def GetToolchainNaClLib(tcname, tcpath, xarch): 114 if tcname == 'pnacl': 115 return os.path.join(tcpath, 'sdk', 'lib') 116 elif xarch == '32': 117 return os.path.join(tcpath, 'x86_64-nacl', 'lib32') 118 elif xarch == '64': 119 return os.path.join(tcpath, 'x86_64-nacl', 'lib') 120 elif xarch == 'arm': 121 return os.path.join(tcpath, 'arm-nacl', 'lib') 122 123 124def GetToolchainDirName(tcname, xarch): 125 if tcname == 'pnacl': 126 return '%s_%s' % (getos.GetPlatform(), tcname) 127 elif xarch == 'arm': 128 return '%s_arm_%s' % (getos.GetPlatform(), tcname) 129 else: 130 return '%s_x86_%s' % (getos.GetPlatform(), tcname) 131 132 133def GetGypToolchainLib(tcname, xarch): 134 tcpath = os.path.join(GetGypGenDir(xarch), 'sdk', 'toolchain', 135 GetToolchainDirName(tcname, xarch)) 136 return GetToolchainNaClLib(tcname, tcpath, xarch) 137 138 139def GetOutputToolchainLib(pepperdir, tcname, xarch): 140 tcpath = os.path.join(pepperdir, 'toolchain', 141 GetToolchainDirName(tcname, xarch)) 142 return GetToolchainNaClLib(tcname, tcpath, xarch) 143 144 145def GetPNaClNativeLib(tcpath, arch): 146 if arch not in ['arm', 'x86-32', 'x86-64']: 147 buildbot_common.ErrorExit('Unknown architecture %s.' % arch) 148 return os.path.join(tcpath, 'lib-' + arch) 149 150 151def BuildStepDownloadToolchains(toolchains): 152 buildbot_common.BuildStep('Running download_toolchains.py') 153 download_script = os.path.join('build', 'download_toolchains.py') 154 args = [sys.executable, download_script, '--no-arm-trusted', 155 '--arm-untrusted', '--keep'] 156 if 'bionic' in toolchains: 157 args.append('--allow-bionic') 158 buildbot_common.Run(args, cwd=NACL_DIR) 159 160 161def BuildStepCleanPepperDirs(pepperdir, pepperdir_old): 162 buildbot_common.BuildStep('Clean Pepper Dirs') 163 buildbot_common.RemoveDir(pepperdir_old) 164 buildbot_common.RemoveDir(pepperdir) 165 buildbot_common.MakeDir(pepperdir) 166 167 168def BuildStepMakePepperDirs(pepperdir, subdirs): 169 for subdir in subdirs: 170 buildbot_common.MakeDir(os.path.join(pepperdir, subdir)) 171 172TEXT_FILES = [ 173 'AUTHORS', 174 'COPYING', 175 'LICENSE', 176 'README.Makefiles', 177 'getting_started/README', 178] 179 180def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, 181 nacl_revision): 182 buildbot_common.BuildStep('Add Text Files') 183 InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES) 184 185 # Replace a few placeholders in README 186 readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read() 187 readme_text = readme_text.replace('${VERSION}', pepper_ver) 188 readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision) 189 readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision) 190 191 # Year/Month/Day Hour:Minute:Second 192 time_format = '%Y/%m/%d %H:%M:%S' 193 readme_text = readme_text.replace('${DATE}', 194 datetime.datetime.now().strftime(time_format)) 195 196 open(os.path.join(pepperdir, 'README'), 'w').write(readme_text) 197 198 199def BuildStepUntarToolchains(pepperdir, toolchains): 200 buildbot_common.BuildStep('Untar Toolchains') 201 platform = getos.GetPlatform() 202 tmpdir = os.path.join(OUT_DIR, 'tc_temp') 203 buildbot_common.RemoveDir(tmpdir) 204 buildbot_common.MakeDir(tmpdir) 205 206 if 'newlib' in toolchains: 207 # Untar the newlib toolchains 208 tarfile = GetNewlibToolchain() 209 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 210 cwd=NACL_DIR) 211 212 # Then rename/move it to the pepper toolchain directory 213 srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk') 214 tcname = platform + '_x86_newlib' 215 newlibdir = os.path.join(pepperdir, 'toolchain', tcname) 216 buildbot_common.Move(srcdir, newlibdir) 217 218 if 'bionic' in toolchains: 219 # Untar the bionic toolchains 220 tarfile = GetBionicToolchain() 221 tcname = platform + '_arm_bionic' 222 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 223 cwd=NACL_DIR) 224 srcdir = os.path.join(tmpdir, tcname) 225 bionicdir = os.path.join(pepperdir, 'toolchain', tcname) 226 buildbot_common.Move(srcdir, bionicdir) 227 228 if 'arm' in toolchains: 229 # Copy the existing arm toolchain from native_client tree 230 tcname = platform + '_arm_newlib' 231 arm_toolchain = os.path.join(NACL_DIR, 'toolchain', '%s_x86' % platform, 232 'nacl_arm_newlib' ) 233 arm_toolchain_sdk = os.path.join(pepperdir, 'toolchain', tcname) 234 buildbot_common.CopyDir(arm_toolchain, arm_toolchain_sdk) 235 236 if 'glibc' in toolchains: 237 # Untar the glibc toolchains 238 tarfile = GetGlibcToolchain() 239 tcname = platform + '_x86_glibc' 240 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 241 cwd=NACL_DIR) 242 243 # Then rename/move it to the pepper toolchain directory 244 srcdir = os.path.join(tmpdir, 'toolchain', platform + '_x86') 245 glibcdir = os.path.join(pepperdir, 'toolchain', tcname) 246 buildbot_common.Move(srcdir, glibcdir) 247 248 # Untar the pnacl toolchains 249 if 'pnacl' in toolchains: 250 tmpdir = os.path.join(tmpdir, 'pnacl') 251 buildbot_common.RemoveDir(tmpdir) 252 buildbot_common.MakeDir(tmpdir) 253 tarfile = GetPNaClToolchain() 254 tcname = platform + '_pnacl' 255 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 256 cwd=NACL_DIR) 257 258 # Then rename/move it to the pepper toolchain directory 259 pnacldir = os.path.join(pepperdir, 'toolchain', tcname) 260 buildbot_common.Move(tmpdir, pnacldir) 261 262 buildbot_common.RemoveDir(tmpdir) 263 264 265# List of toolchain headers to install. 266# Source is relative to top of Chromium tree, destination is relative 267# to the toolchain header directory. 268NACL_HEADER_MAP = { 269 'newlib': [ 270 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'), 271 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'), 272 ('native_client/src/untrusted/irt/irt.h', ''), 273 ('native_client/src/untrusted/irt/irt_dev.h', ''), 274 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 275 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'), 276 ('native_client/src/untrusted/nacl/nacl_thread.h', 'nacl/'), 277 ('native_client/src/untrusted/pthread/pthread.h', ''), 278 ('native_client/src/untrusted/pthread/semaphore.h', ''), 279 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 280 ('ppapi/nacl_irt/public/irt_ppapi.h', ''), 281 ], 282 'glibc': [ 283 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'), 284 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'), 285 ('native_client/src/untrusted/irt/irt.h', ''), 286 ('native_client/src/untrusted/irt/irt_dev.h', ''), 287 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 288 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'), 289 ('native_client/src/untrusted/nacl/nacl_thread.h', 'nacl/'), 290 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 291 ('ppapi/nacl_irt/public/irt_ppapi.h', ''), 292 ], 293 'host': [] 294} 295 296def InstallFiles(src_root, dest_root, file_list): 297 """Copy a set of files from src_root to dest_root according 298 to the given mapping. This allows files to be copied from 299 to a location in the destination tree that is different to the 300 location in the source tree. 301 302 If the destination mapping ends with a '/' then the destination 303 basename is inherited from the the source file. 304 305 Wildcards can be used in the source list but it is not recommended 306 as this can end up adding things to the SDK unintentionally. 307 """ 308 for file_spec in file_list: 309 # The list of files to install can be a simple list of 310 # strings or a list of pairs, where each pair corresponds 311 # to a mapping from source to destination names. 312 if type(file_spec) == str: 313 src_file = dest_file = file_spec 314 else: 315 src_file, dest_file = file_spec 316 317 src_file = os.path.join(src_root, src_file) 318 319 # Expand sources files using glob. 320 sources = glob.glob(src_file) 321 if not sources: 322 sources = [src_file] 323 324 if len(sources) > 1 and not dest_file.endswith('/'): 325 buildbot_common.ErrorExit("Target file must end in '/' when " 326 "using globbing to install multiple files") 327 328 for source in sources: 329 if dest_file.endswith('/'): 330 dest = os.path.join(dest_file, os.path.basename(source)) 331 else: 332 dest = dest_file 333 dest = os.path.join(dest_root, dest) 334 if not os.path.isdir(os.path.dirname(dest)): 335 buildbot_common.MakeDir(os.path.dirname(dest)) 336 buildbot_common.CopyFile(source, dest) 337 338 339def InstallNaClHeaders(tc_dst_inc, tc_name): 340 """Copies NaCl headers to expected locations in the toolchain.""" 341 if tc_name == 'arm': 342 # arm toolchain header should be the same as the x86 newlib 343 # ones 344 tc_name = 'newlib' 345 346 InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name]) 347 348 349def MakeNinjaRelPath(path): 350 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path) 351 352 353TOOLCHAIN_LIBS = { 354 'bionic' : [ 355 'libminidump_generator.a', 356 'libnacl_dyncode.a', 357 'libnacl_exception.a', 358 'libnacl_list_mappings.a', 359 'libppapi.a', 360 ], 361 'newlib' : [ 362 'crti.o', 363 'crtn.o', 364 'libminidump_generator.a', 365 'libnacl.a', 366 'libnacl_dyncode.a', 367 'libnacl_exception.a', 368 'libnacl_list_mappings.a', 369 'libnosys.a', 370 'libppapi.a', 371 'libppapi_stub.a', 372 'libpthread.a', 373 ], 374 'glibc': [ 375 'libminidump_generator.a', 376 'libminidump_generator.so', 377 'libnacl.a', 378 'libnacl_dyncode.a', 379 'libnacl_dyncode.so', 380 'libnacl_exception.a', 381 'libnacl_exception.so', 382 'libnacl_list_mappings.a', 383 'libnacl_list_mappings.so', 384 'libppapi.a', 385 'libppapi.so', 386 'libppapi_stub.a', 387 ], 388 'pnacl': [ 389 'libminidump_generator.a', 390 'libnacl.a', 391 'libnacl_dyncode.a', 392 'libnacl_exception.a', 393 'libnacl_list_mappings.a', 394 'libnosys.a', 395 'libppapi.a', 396 'libppapi_stub.a', 397 'libpthread.a', 398 ] 399} 400 401 402def GypNinjaInstall(pepperdir, toolchains): 403 build_dir = GYPBUILD_DIR 404 ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release') 405 tools_files = [ 406 ['sel_ldr', 'sel_ldr_x86_32'], 407 ['ncval_new', 'ncval'], 408 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'], 409 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'], 410 ] 411 412 platform = getos.GetPlatform() 413 414 # TODO(binji): dump_syms doesn't currently build on Windows. See 415 # http://crbug.com/245456 416 if platform != 'win': 417 tools_files += [ 418 ['dump_syms', 'dump_syms'], 419 ['minidump_dump', 'minidump_dump'], 420 ['minidump_stackwalk', 'minidump_stackwalk'] 421 ] 422 423 tools_files.append(['sel_ldr64', 'sel_ldr_x86_64']) 424 425 if platform == 'linux': 426 tools_files.append(['nacl_helper_bootstrap', 427 'nacl_helper_bootstrap_x86_32']) 428 tools_files.append(['nacl_helper_bootstrap64', 429 'nacl_helper_bootstrap_x86_64']) 430 431 buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) 432 433 # Add .exe extensions to all windows tools 434 for pair in tools_files: 435 if platform == 'win' and not pair[0].endswith('.nexe'): 436 pair[0] += '.exe' 437 pair[1] += '.exe' 438 439 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files) 440 441 # Add ARM binaries 442 if platform == 'linux': 443 tools_files = [ 444 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'], 445 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'], 446 ['sel_ldr', 'sel_ldr_arm'], 447 ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm'] 448 ] 449 ninja_out_dir = os.path.join(OUT_DIR, build_dir + '-arm', 'Release') 450 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files) 451 452 for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']): 453 if tc == 'pnacl': 454 xarches = (None,) 455 else: 456 xarches = ('arm', '32', '64') 457 458 for xarch in xarches: 459 if tc == 'glibc' and xarch == 'arm': 460 continue 461 462 src_dir = GetGypBuiltLib(tc, xarch) 463 dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch) 464 InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc]) 465 466 # Copy ARM newlib components to bionic 467 if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains: 468 bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch) 469 InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic']) 470 471 if tc != 'pnacl': 472 src_dir = GetGypToolchainLib(tc, xarch) 473 InstallFiles(src_dir, dst_dir, ['crt1.o']) 474 475 476def GypNinjaBuild_NaCl(rel_out_dir): 477 gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl') 478 nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp') 479 all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp') 480 481 out_dir = MakeNinjaRelPath(rel_out_dir) 482 out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm') 483 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir) 484 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm) 485 GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir) 486 487 platform = getos.GetPlatform() 488 if platform == 'win': 489 NinjaBuild('sel_ldr64', out_dir) 490 else: 491 out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64') 492 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64) 493 494 # We only need sel_ldr from the 64-bit out directory. 495 # sel_ldr needs to be renamed, so we'll call it sel_ldr64. 496 files_to_copy = [('sel_ldr', 'sel_ldr64')] 497 if platform == 'linux': 498 files_to_copy.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap64')) 499 500 for src, dst in files_to_copy: 501 buildbot_common.CopyFile( 502 os.path.join(SRC_DIR, out_dir_64, 'Release', src), 503 os.path.join(SRC_DIR, out_dir, 'Release', dst)) 504 505 506def GypNinjaBuild_Breakpad(rel_out_dir): 507 # TODO(binji): dump_syms doesn't currently build on Windows. See 508 # http://crbug.com/245456 509 if getos.GetPlatform() == 'win': 510 return 511 512 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 513 out_dir = MakeNinjaRelPath(rel_out_dir) 514 gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp') 515 build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk'] 516 GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir) 517 518 519def GypNinjaBuild_PPAPI(arch, rel_out_dir): 520 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 521 out_dir = MakeNinjaRelPath(rel_out_dir) 522 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 523 'native_client.gyp') 524 GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir) 525 526 527def GypNinjaBuild_Pnacl(rel_out_dir, target_arch): 528 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the 529 # Chromium build, and once here. When we move more of the SDK build process 530 # to gyp, we can remove this. 531 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 532 533 out_dir = MakeNinjaRelPath(rel_out_dir) 534 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src', 535 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp') 536 targets = ['pnacl_irt_shim_aot'] 537 GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False) 538 539 540def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, 541 out_dir, force_arm_gcc=True): 542 gyp_env = dict(os.environ) 543 gyp_env['GYP_GENERATORS'] = 'ninja' 544 gyp_defines = [] 545 if options.mac_sdk: 546 gyp_defines.append('mac_sdk=%s' % options.mac_sdk) 547 if arch: 548 gyp_defines.append('target_arch=%s' % arch) 549 if arch == 'arm': 550 if getos.GetPlatform() == 'linux': 551 gyp_env['CC'] = 'arm-linux-gnueabihf-gcc' 552 gyp_env['CXX'] = 'arm-linux-gnueabihf-g++' 553 gyp_env['AR'] = 'arm-linux-gnueabihf-ar' 554 gyp_env['AS'] = 'arm-linux-gnueabihf-as' 555 gyp_env['CC_host'] = 'cc' 556 gyp_env['CXX_host'] = 'c++' 557 gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1', 558 'arm_float_abi=hard'] 559 if force_arm_gcc: 560 gyp_defines.append('nacl_enable_arm_gcc=1') 561 if getos.GetPlatform() == 'mac': 562 gyp_defines.append('clang=1') 563 564 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines) 565 for key in ['GYP_GENERATORS', 'GYP_DEFINES', 'CC']: 566 value = gyp_env.get(key) 567 if value is not None: 568 print '%s="%s"' % (key, value) 569 gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)] 570 gyp_depth = '--depth=.' 571 buildbot_common.Run( 572 [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \ 573 gyp_generator_flags, 574 cwd=SRC_DIR, 575 env=gyp_env) 576 NinjaBuild(targets, out_dir) 577 578 579def NinjaBuild(targets, out_dir): 580 if type(targets) is not list: 581 targets = [targets] 582 out_config_dir = os.path.join(out_dir, 'Release') 583 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR) 584 585 586def BuildStepBuildToolchains(pepperdir, toolchains): 587 buildbot_common.BuildStep('SDK Items') 588 589 GypNinjaBuild_NaCl(GYPBUILD_DIR) 590 GypNinjaBuild_Breakpad(GYPBUILD_DIR) 591 592 platform = getos.GetPlatform() 593 newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib') 594 glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc') 595 armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib') 596 pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl') 597 598 if set(toolchains) & set(['glibc', 'newlib']): 599 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR) 600 601 if 'arm' in toolchains: 602 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm') 603 604 GypNinjaInstall(pepperdir, toolchains) 605 606 if 'newlib' in toolchains: 607 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'), 608 'newlib') 609 610 if 'glibc' in toolchains: 611 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'), 612 'glibc') 613 614 if 'arm' in toolchains: 615 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'), 616 'arm') 617 618 if 'pnacl' in toolchains: 619 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default. 620 for arch in ('ia32', 'arm'): 621 # Fill in the latest native pnacl shim library from the chrome build. 622 build_dir = GYPBUILD_DIR + '-pnacl-' + arch 623 GypNinjaBuild_Pnacl(build_dir, arch) 624 if arch == 'ia32': 625 nacl_arches = ['x86-32', 'x86-64'] 626 elif arch == 'arm': 627 nacl_arches = ['arm'] 628 else: 629 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 630 for nacl_arch in nacl_arches: 631 release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release', 632 'gen', 'tc_pnacl_translate', 633 'lib-' + nacl_arch) 634 635 buildbot_common.CopyFile( 636 os.path.join(release_build_dir, 'libpnacl_irt_shim.a'), 637 GetPNaClNativeLib(pnacldir, nacl_arch)) 638 639 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'), 640 'newlib') 641 642 643def MakeDirectoryOrClobber(pepperdir, dirname, clobber): 644 dirpath = os.path.join(pepperdir, dirname) 645 if clobber: 646 buildbot_common.RemoveDir(dirpath) 647 buildbot_common.MakeDir(dirpath) 648 649 return dirpath 650 651 652def BuildStepUpdateHelpers(pepperdir, clobber): 653 buildbot_common.BuildStep('Update project helpers') 654 build_projects.UpdateHelpers(pepperdir, clobber=clobber) 655 656 657def BuildStepUpdateUserProjects(pepperdir, toolchains, 658 build_experimental, clobber): 659 buildbot_common.BuildStep('Update examples and libraries') 660 661 filters = {} 662 if not build_experimental: 663 filters['EXPERIMENTAL'] = False 664 if toolchains: 665 toolchains = toolchains[:] 666 667 # arm isn't a valid toolchain for build_projects 668 if 'arm' in toolchains: 669 toolchains.remove('arm') 670 671 if 'host' in toolchains: 672 toolchains.remove('host') 673 toolchains.append(getos.GetPlatform()) 674 675 filters['TOOLS'] = toolchains 676 677 # Update examples and libraries 678 filters['DEST'] = [ 679 'getting_started', 680 'examples/api', 681 'examples/demo', 682 'examples/tutorial', 683 'src' 684 ] 685 686 tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters) 687 build_projects.UpdateProjects(pepperdir, tree, clobber=clobber, 688 toolchains=toolchains) 689 690 691def BuildStepMakeAll(pepperdir, directory, step_name, 692 deps=True, clean=False, config='Debug', args=None): 693 buildbot_common.BuildStep(step_name) 694 build_projects.BuildProjectsBranch(pepperdir, directory, clean, 695 deps, config, args) 696 697 698def BuildStepBuildLibraries(pepperdir, directory): 699 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug', 700 clean=True, config='Debug') 701 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release', 702 clean=True, config='Release') 703 704 # Cleanup .pyc file generated while building libraries. Without 705 # this we would end up shipping the pyc in the SDK tarball. 706 buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc')) 707 708 709def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None): 710 # Look for LICENSE files 711 license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT') 712 713 license_files = [] 714 for root, _, files in os.walk(fileroot): 715 for filename in files: 716 if license_filenames_re.match(filename): 717 path = os.path.join(root, filename) 718 license_files.append(path) 719 720 if extra_files: 721 license_files += [os.path.join(fileroot, f) for f in extra_files] 722 print '\n'.join(license_files) 723 724 if not os.path.isabs(output_filename): 725 output_filename = os.path.join(fileroot, output_filename) 726 generate_notice.Generate(output_filename, fileroot, license_files) 727 728 729def BuildStepVerifyFilelist(pepperdir): 730 buildbot_common.BuildStep('Verify SDK Files') 731 file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list') 732 try: 733 verify_filelist.Verify(file_list_path, pepperdir) 734 print 'OK' 735 except verify_filelist.ParseException, e: 736 buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e) 737 except verify_filelist.VerifyException, e: 738 file_list_rel = os.path.relpath(file_list_path) 739 verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py' 740 verify_filelist_py = os.path.relpath(verify_filelist_py) 741 pepperdir_rel = os.path.relpath(pepperdir) 742 743 msg = """\ 744SDK verification failed: 745 746%s 747Add/remove files from %s to fix. 748 749Run: 750 ./%s %s %s 751to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel, 752 pepperdir_rel) 753 buildbot_common.ErrorExit(msg) 754 755 756def BuildStepTarBundle(pepper_ver, tarfile): 757 buildbot_common.BuildStep('Tar Pepper Bundle') 758 buildbot_common.MakeDir(os.path.dirname(tarfile)) 759 buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 760 'pepper_' + pepper_ver], cwd=NACL_DIR) 761 762 763def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile, 764 archive_url): 765 with open(tarfile, 'rb') as tarfile_stream: 766 archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( 767 tarfile_stream) 768 769 archive = manifest_util.Archive(manifest_util.GetHostOS()) 770 archive.url = archive_url 771 archive.size = archive_size 772 archive.checksum = archive_sha1 773 774 bundle = manifest_util.Bundle('pepper_' + pepper_ver) 775 bundle.revision = int(chrome_revision) 776 bundle.repath = 'pepper_' + pepper_ver 777 bundle.version = int(pepper_ver) 778 bundle.description = ( 779 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % ( 780 pepper_ver, chrome_revision, nacl_revision)) 781 bundle.stability = 'dev' 782 bundle.recommended = 'no' 783 bundle.archives = [archive] 784 return bundle 785 786 787def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision, 788 tarfile): 789 buildbot_common.BuildStep('Archive %s' % name) 790 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 791 build_version.ChromeVersion(),) 792 tarname = os.path.basename(tarfile) 793 tarfile_dir = os.path.dirname(tarfile) 794 buildbot_common.Archive(tarname, bucket_path, tarfile_dir) 795 796 # generate "manifest snippet" for this archive. 797 archive_url = GSTORE + 'nacl_sdk/%s/%s' % ( 798 build_version.ChromeVersion(), tarname) 799 bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, 800 tarfile, archive_url) 801 802 manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') 803 with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: 804 manifest_snippet_stream.write(bundle.GetDataAsString()) 805 806 buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, 807 step_link=False) 808 809 810def BuildStepArchiveSDKTools(): 811 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot. 812 builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') 813 if builder_name == 'linux-sdk-multi': 814 buildbot_common.BuildStep('Build SDK Tools') 815 build_updater.BuildUpdater(OUT_DIR) 816 817 buildbot_common.BuildStep('Archive SDK Tools') 818 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 819 build_version.ChromeVersion(),) 820 buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, 821 step_link=False) 822 buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR, 823 step_link=False) 824 825 826def BuildStepSyncNaClPorts(): 827 """Pull the pinned revision of naclports from SVN.""" 828 buildbot_common.BuildStep('Sync naclports') 829 if not os.path.exists(NACLPORTS_DIR): 830 # checkout new copy of naclports 831 cmd = ['svn', 'checkout', '-q', '-r', str(NACLPORTS_REV), NACLPORTS_URL, 832 'naclports'] 833 buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR)) 834 else: 835 # sync existing copy to pinned revision. 836 cmd = ['svn', 'update', '-r', str(NACLPORTS_REV)] 837 buildbot_common.Run(cmd, cwd=NACLPORTS_DIR) 838 839 840def BuildStepBuildNaClPorts(pepper_ver, pepperdir): 841 """Build selected naclports in all configurations.""" 842 # TODO(sbc): currently naclports doesn't know anything about 843 # Debug builds so the Debug subfolders are all empty. 844 845 env = dict(os.environ) 846 env['NACL_SDK_ROOT'] = pepperdir 847 env['PEPPER_DIR'] = os.path.basename(pepperdir) # pepper_NN 848 env['NACLPORTS_NO_ANNOTATE'] = "1" 849 env['NACLPORTS_NO_UPLOAD'] = "1" 850 env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV) 851 852 build_script = 'build_tools/buildbot_sdk_bundle.sh' 853 buildbot_common.BuildStep('Build naclports') 854 855 bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 856 out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver) 857 858 # Remove the sdk_bundle directory to remove stale files from previous builds. 859 buildbot_common.RemoveDir(bundle_dir) 860 861 buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR) 862 863 # Some naclports do not include a standalone LICENSE/COPYING file 864 # so we explicitly list those here for inclusion. 865 extra_licenses = ('tinyxml/readme.txt', 866 'jpeg-8d/README', 867 'zlib-1.2.3/README') 868 src_root = os.path.join(NACLPORTS_DIR, 'out', 'build') 869 output_license = os.path.join(out_dir, 'ports', 'LICENSE') 870 GenerateNotice(src_root , output_license, extra_licenses) 871 readme = os.path.join(out_dir, 'ports', 'README') 872 oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme]) 873 874 875def BuildStepTarNaClPorts(pepper_ver, tarfile): 876 """Create tar archive containing headers and libs from naclports build.""" 877 buildbot_common.BuildStep('Tar naclports Bundle') 878 buildbot_common.MakeDir(os.path.dirname(tarfile)) 879 pepper_dir = 'pepper_%s' % pepper_ver 880 archive_dirs = [os.path.join(pepper_dir, 'ports')] 881 882 ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 883 cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile] 884 cmd += archive_dirs 885 buildbot_common.Run(cmd, cwd=NACL_DIR) 886 887 888def BuildStepBuildAppEngine(pepperdir, chrome_revision): 889 """Build the projects found in src/gonacl_appengine/src""" 890 buildbot_common.BuildStep('Build GoNaCl AppEngine Projects') 891 cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision] 892 env = dict(os.environ) 893 env['NACL_SDK_ROOT'] = pepperdir 894 env['NACLPORTS_NO_ANNOTATE'] = "1" 895 buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR) 896 897 898def main(args): 899 parser = optparse.OptionParser(description=__doc__) 900 parser.add_option('--nacl-tree-path', 901 help='Path to native client tree for bionic build.', 902 dest='nacl_tree_path') 903 parser.add_option('--qemu', help='Add qemu for ARM.', 904 action='store_true') 905 parser.add_option('--bionic', help='Add bionic build.', 906 action='store_true') 907 parser.add_option('--tar', help='Force the tar step.', 908 action='store_true') 909 parser.add_option('--archive', help='Force the archive step.', 910 action='store_true') 911 parser.add_option('--release', help='PPAPI release version.', 912 dest='release', default=None) 913 parser.add_option('--build-ports', 914 help='Build naclport bundle.', action='store_true') 915 parser.add_option('--build-app-engine', 916 help='Build AppEngine demos.', action='store_true') 917 parser.add_option('--experimental', 918 help='build experimental examples and libraries', action='store_true', 919 dest='build_experimental') 920 parser.add_option('--skip-toolchain', help='Skip toolchain untar', 921 action='store_true') 922 parser.add_option('--mac-sdk', 923 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.') 924 925 # To setup bash completion for this command first install optcomplete 926 # and then add this line to your .bashrc: 927 # complete -F _optcomplete build_sdk.py 928 try: 929 import optcomplete 930 optcomplete.autocomplete(parser) 931 except ImportError: 932 pass 933 934 global options 935 options, args = parser.parse_args(args[1:]) 936 if args: 937 parser.error("Unexpected arguments: %s" % str(args)) 938 939 if options.nacl_tree_path: 940 options.bionic = True 941 toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build') 942 print 'WARNING: Building bionic toolchain from NaCl checkout.' 943 print 'This option builds bionic in a NativeClient checkout and uses the' 944 print 'results instead of downloading a toolchain from the builder.' 945 print 'This may result in a NaCl SDK that can not run on ToT chrome.' 946 print '' 947 response = raw_input("Type 'y' and hit enter to continue.\n") 948 if response != 'y' and response != 'Y': 949 print 'Aborting.' 950 return 1 951 952 # Get head version of NativeClient tree 953 buildbot_common.BuildStep('Build bionic toolchain.') 954 buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py'], 955 cwd=toolchain_build) 956 else: 957 toolchain_build = None 958 959 if buildbot_common.IsSDKBuilder(): 960 options.archive = True 961 options.build_ports = True 962 options.build_app_engine = True 963 options.tar = True 964 965 toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host'] 966 967 # Changes for experimental bionic builder 968 if options.bionic: 969 toolchains.append('bionic') 970 options.build_ports = False 971 options.build_app_engine = False 972 973 print 'Building: ' + ' '.join(toolchains) 974 975 if options.archive and not options.tar: 976 parser.error('Incompatible arguments with archive.') 977 978 chrome_version = int(build_version.ChromeMajorVersion()) 979 chrome_revision = build_version.ChromeRevision() 980 nacl_revision = build_version.NaClRevision() 981 pepper_ver = str(chrome_version) 982 pepper_old = str(chrome_version - 1) 983 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 984 pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old) 985 if options.bionic: 986 tarname = 'naclsdk_bionic.tar.bz2' 987 else: 988 tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2' 989 tarfile = os.path.join(OUT_DIR, tarname) 990 991 if options.release: 992 pepper_ver = options.release 993 print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision) 994 995 if 'NACL_SDK_ROOT' in os.environ: 996 # We don't want the currently configured NACL_SDK_ROOT to have any effect 997 # of the build. 998 del os.environ['NACL_SDK_ROOT'] 999 1000 if not options.skip_toolchain: 1001 BuildStepCleanPepperDirs(pepperdir, pepperdir_old) 1002 BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools']) 1003 BuildStepDownloadToolchains(toolchains) 1004 if options.nacl_tree_path: 1005 # Instead of untarring, copy the raw bionic toolchain 1006 not_bionic = [i for i in toolchains if i != 'bionic'] 1007 BuildStepUntarToolchains(pepperdir, not_bionic) 1008 tcname = GetToolchainDirName('bionic', 'arm') 1009 srcdir = os.path.join(toolchain_build, 'out', tcname) 1010 bionicdir = os.path.join(pepperdir, 'toolchain', tcname) 1011 oshelpers.Copy(['-r', srcdir, bionicdir]) 1012 else: 1013 BuildStepUntarToolchains(pepperdir, toolchains) 1014 BuildStepBuildToolchains(pepperdir, toolchains) 1015 1016 BuildStepUpdateHelpers(pepperdir, True) 1017 BuildStepUpdateUserProjects(pepperdir, toolchains, 1018 options.build_experimental, True) 1019 1020 BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision) 1021 1022 # Ship with libraries prebuilt, so run that first. 1023 BuildStepBuildLibraries(pepperdir, 'src') 1024 GenerateNotice(pepperdir) 1025 1026 # Verify the SDK contains what we expect. 1027 if not options.bionic: 1028 BuildStepVerifyFilelist(pepperdir) 1029 1030 if options.tar: 1031 BuildStepTarBundle(pepper_ver, tarfile) 1032 1033 if options.build_ports and getos.GetPlatform() == 'linux': 1034 ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2') 1035 BuildStepSyncNaClPorts() 1036 BuildStepBuildNaClPorts(pepper_ver, pepperdir) 1037 if options.tar: 1038 BuildStepTarNaClPorts(pepper_ver, ports_tarfile) 1039 1040 if options.build_app_engine and getos.GetPlatform() == 'linux': 1041 BuildStepBuildAppEngine(pepperdir, chrome_revision) 1042 1043 if options.qemu: 1044 qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted') 1045 oshelpers.Copy(['-r', qemudir, pepperdir]) 1046 1047 # Archive on non-trybots. 1048 if options.archive: 1049 BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision, 1050 tarfile) 1051 if options.build_ports and getos.GetPlatform() == 'linux': 1052 BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision, 1053 nacl_revision, ports_tarfile) 1054 BuildStepArchiveSDKTools() 1055 1056 return 0 1057 1058 1059if __name__ == '__main__': 1060 try: 1061 sys.exit(main(sys.argv)) 1062 except KeyboardInterrupt: 1063 buildbot_common.ErrorExit('build_sdk: interrupted') 1064 1065