build_sdk.py revision 8bcbed890bc3ce4d7a057a8f32cab53fa534672e
1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Entry point for both build and try bots. 7 8This script is invoked from XXX, usually without arguments 9to package an SDK. It automatically determines whether 10this SDK is for mac, win, linux. 11 12The script inspects the following environment variables: 13 14BUILDBOT_BUILDERNAME to determine whether the script is run locally 15and whether it should upload an SDK to file storage (GSTORE) 16""" 17 18# pylint: disable=W0621 19 20# std python includes 21import copy 22import datetime 23import glob 24import optparse 25import os 26import re 27import sys 28 29if sys.version_info < (2, 6, 0): 30 sys.stderr.write("python 2.6 or later is required run this script\n") 31 sys.exit(1) 32 33# local includes 34import buildbot_common 35import build_projects 36import build_updater 37import build_version 38import generate_make 39import generate_notice 40import manifest_util 41import parse_dsc 42import verify_filelist 43 44from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR 45from build_paths import NACLPORTS_DIR, GSTORE 46 47# Add SDK make tools scripts to the python path. 48sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 49sys.path.append(os.path.join(NACL_DIR, 'build')) 50 51import getos 52import oshelpers 53 54CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') 55 56NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src' 57NACLPORTS_REV = 850 58 59GYPBUILD_DIR = 'gypbuild' 60 61options = None 62 63 64def GetGlibcToolchain(): 65 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 66 tcname = 'toolchain_%s_x86.tar.bz2' % getos.GetPlatform() 67 return os.path.join(tcdir, tcname) 68 69 70def GetNewlibToolchain(): 71 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 72 tcname = 'naclsdk_%s_x86.tgz' % getos.GetPlatform() 73 return os.path.join(tcdir, tcname) 74 75 76def GetPNaClToolchain(): 77 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 78 tcname = 'naclsdk_pnacl_%s_x86.tgz' % getos.GetPlatform() 79 return os.path.join(tcdir, tcname) 80 81 82def GetToolchainNaClInclude(tcname, tcpath, arch): 83 if arch == 'x86': 84 if tcname == 'pnacl': 85 return os.path.join(tcpath, 'sdk', 'include') 86 return os.path.join(tcpath, 'x86_64-nacl', 'include') 87 elif arch == 'arm': 88 return os.path.join(tcpath, 'arm-nacl', 'include') 89 else: 90 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 91 92 93def GetGypGenDir(xarch): 94 if xarch == 'arm': 95 build_dir = GYPBUILD_DIR + '-arm' 96 else: 97 build_dir = GYPBUILD_DIR 98 return os.path.join(OUT_DIR, build_dir, 'Release', 'gen') 99 100 101def GetGypBuiltLib(tcname, xarch=None): 102 if tcname == 'pnacl': 103 tcname = 'pnacl_newlib' 104 if not xarch: 105 xarch = '' 106 return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch) 107 108 109def GetToolchainNaClLib(tcname, tcpath, xarch): 110 if tcname == 'pnacl': 111 return os.path.join(tcpath, 'sdk', 'lib') 112 elif xarch == '32': 113 return os.path.join(tcpath, 'x86_64-nacl', 'lib32') 114 elif xarch == '64': 115 return os.path.join(tcpath, 'x86_64-nacl', 'lib') 116 elif xarch == 'arm': 117 return os.path.join(tcpath, 'arm-nacl', 'lib') 118 119 120def GetToolchainDirName(tcname, xarch): 121 if tcname == 'pnacl': 122 return '%s_%s' % (getos.GetPlatform(), tcname) 123 elif xarch == 'arm': 124 return '%s_arm_%s' % (getos.GetPlatform(), tcname) 125 else: 126 return '%s_x86_%s' % (getos.GetPlatform(), tcname) 127 128 129def GetGypToolchainLib(tcname, xarch): 130 tcpath = os.path.join(GetGypGenDir(xarch), 'sdk', 'toolchain', 131 GetToolchainDirName(tcname, xarch)) 132 return GetToolchainNaClLib(tcname, tcpath, xarch) 133 134 135def GetOutputToolchainLib(pepperdir, tcname, xarch): 136 tcpath = os.path.join(pepperdir, 'toolchain', 137 GetToolchainDirName(tcname, xarch)) 138 return GetToolchainNaClLib(tcname, tcpath, xarch) 139 140 141def GetPNaClNativeLib(tcpath, arch): 142 if arch not in ['arm', 'x86-32', 'x86-64']: 143 buildbot_common.ErrorExit('Unknown architecture %s.' % arch) 144 return os.path.join(tcpath, 'lib-' + arch) 145 146 147def BuildStepDownloadToolchains(): 148 buildbot_common.BuildStep('Running download_toolchains.py') 149 download_script = os.path.join('build', 'download_toolchains.py') 150 buildbot_common.Run([sys.executable, download_script, 151 '--no-arm-trusted', '--arm-untrusted', '--keep'], 152 cwd=NACL_DIR) 153 154 155def BuildStepCleanPepperDirs(pepperdir, pepperdir_old): 156 buildbot_common.BuildStep('Clean Pepper Dirs') 157 buildbot_common.RemoveDir(pepperdir_old) 158 buildbot_common.RemoveDir(pepperdir) 159 buildbot_common.MakeDir(pepperdir) 160 161 162def BuildStepMakePepperDirs(pepperdir, subdirs): 163 for subdir in subdirs: 164 buildbot_common.MakeDir(os.path.join(pepperdir, subdir)) 165 166TEXT_FILES = [ 167 'AUTHORS', 168 'COPYING', 169 'LICENSE', 170 'README.Makefiles', 171 'getting_started/README', 172] 173 174def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, 175 nacl_revision): 176 buildbot_common.BuildStep('Add Text Files') 177 InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES) 178 179 # Replace a few placeholders in README 180 readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read() 181 readme_text = readme_text.replace('${VERSION}', pepper_ver) 182 readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision) 183 readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision) 184 185 # Year/Month/Day Hour:Minute:Second 186 time_format = '%Y/%m/%d %H:%M:%S' 187 readme_text = readme_text.replace('${DATE}', 188 datetime.datetime.now().strftime(time_format)) 189 190 open(os.path.join(pepperdir, 'README'), 'w').write(readme_text) 191 192 193def PrunePNaClToolchain(root): 194 dirs_to_prune = [ 195 'lib-bc-x86-64', 196 'usr-bc-x86-64' 197 # TODO(sbc): remove this once its really not needed. 198 # Currently we seem to rely on it at least for <bits/stat.h> 199 #'sysroot', 200 ] 201 for dirname in dirs_to_prune: 202 buildbot_common.RemoveDir(os.path.join(root, dirname)) 203 204 205def BuildStepUntarToolchains(pepperdir, toolchains): 206 buildbot_common.BuildStep('Untar Toolchains') 207 platform = getos.GetPlatform() 208 tmpdir = os.path.join(OUT_DIR, 'tc_temp') 209 buildbot_common.RemoveDir(tmpdir) 210 buildbot_common.MakeDir(tmpdir) 211 212 if 'newlib' in toolchains: 213 # Untar the newlib toolchains 214 tarfile = GetNewlibToolchain() 215 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 216 cwd=NACL_DIR) 217 218 # Then rename/move it to the pepper toolchain directory 219 srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk') 220 tcname = platform + '_x86_newlib' 221 newlibdir = os.path.join(pepperdir, 'toolchain', tcname) 222 buildbot_common.Move(srcdir, newlibdir) 223 224 if 'arm' in toolchains: 225 # Copy the existing arm toolchain from native_client tree 226 tcname = platform + '_arm_newlib' 227 arm_toolchain = os.path.join(NACL_DIR, 'toolchain', tcname) 228 arm_toolchain_sdk = os.path.join(pepperdir, 'toolchain', 229 os.path.basename(arm_toolchain)) 230 buildbot_common.CopyDir(arm_toolchain, arm_toolchain_sdk) 231 232 if 'glibc' in toolchains: 233 # Untar the glibc toolchains 234 tarfile = GetGlibcToolchain() 235 tcname = platform + '_x86_glibc' 236 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 237 cwd=NACL_DIR) 238 239 # Then rename/move it to the pepper toolchain directory 240 srcdir = os.path.join(tmpdir, 'toolchain', platform + '_x86') 241 glibcdir = os.path.join(pepperdir, 'toolchain', tcname) 242 buildbot_common.Move(srcdir, glibcdir) 243 244 # Untar the pnacl toolchains 245 if 'pnacl' in toolchains: 246 tmpdir = os.path.join(tmpdir, 'pnacl') 247 buildbot_common.RemoveDir(tmpdir) 248 buildbot_common.MakeDir(tmpdir) 249 tarfile = GetPNaClToolchain() 250 tcname = platform + '_pnacl' 251 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 252 cwd=NACL_DIR) 253 254 # Then rename/move it to the pepper toolchain directory 255 pnacldir = os.path.join(pepperdir, 'toolchain', tcname) 256 buildbot_common.Move(tmpdir, pnacldir) 257 PrunePNaClToolchain(pnacldir) 258 259 buildbot_common.RemoveDir(tmpdir) 260 261 if options.gyp and platform != 'win': 262 # If the gyp options is specified we install a toolchain 263 # wrapper so that gyp can switch toolchains via a commandline 264 # option. 265 bindir = os.path.join(pepperdir, 'toolchain', tcname, 'bin') 266 wrapper = os.path.join(SDK_SRC_DIR, 'tools', 'compiler-wrapper.py') 267 buildbot_common.MakeDir(bindir) 268 buildbot_common.CopyFile(wrapper, bindir) 269 270 # Module 'os' has no 'symlink' member (on Windows). 271 # pylint: disable=E1101 272 273 os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-g++')) 274 os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-gcc')) 275 os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-ar')) 276 277 278# List of toolchain headers to install. 279# Source is relative to native_client tree, destination is relative 280# to the toolchain header directory. 281NACL_HEADER_MAP = { 282 'newlib': [ 283 ('src/include/nacl/nacl_exception.h', 'nacl/'), 284 ('src/include/nacl/nacl_minidump.h', 'nacl/'), 285 ('src/untrusted/irt/irt.h', ''), 286 ('src/untrusted/irt/irt_dev.h', ''), 287 ('src/untrusted/irt/irt_ppapi.h', ''), 288 ('src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 289 ('src/untrusted/nacl/nacl_startup.h', 'nacl/'), 290 ('src/untrusted/nacl/nacl_thread.h', 'nacl/'), 291 ('src/untrusted/pthread/pthread.h', ''), 292 ('src/untrusted/pthread/semaphore.h', ''), 293 ('src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 294 ], 295 'glibc': [ 296 ('src/include/nacl/nacl_exception.h', 'nacl/'), 297 ('src/include/nacl/nacl_minidump.h', 'nacl/'), 298 ('src/untrusted/irt/irt.h', ''), 299 ('src/untrusted/irt/irt_dev.h', ''), 300 ('src/untrusted/irt/irt_ppapi.h', ''), 301 ('src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 302 ('src/untrusted/nacl/nacl_startup.h', 'nacl/'), 303 ('src/untrusted/nacl/nacl_thread.h', 'nacl/'), 304 ('src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 305 ], 306 'host': [] 307} 308 309def InstallFiles(src_root, dest_root, file_list): 310 """Copy a set of files from src_root to dest_root according 311 to the given mapping. This allows files to be copied from 312 to a location in the destination tree that is different to the 313 location in the source tree. 314 315 If the destination mapping ends with a '/' then the destination 316 basename is inherited from the the source file. 317 318 Wildcards can be used in the source list but it is not recommended 319 as this can end up adding things to the SDK unintentionally. 320 """ 321 for file_spec in file_list: 322 # The list of files to install can be a simple list of 323 # strings or a list of pairs, where each pair corresponds 324 # to a mapping from source to destination names. 325 if type(file_spec) == str: 326 src_file = dest_file = file_spec 327 else: 328 src_file, dest_file = file_spec 329 330 src_file = os.path.join(src_root, src_file) 331 332 # Expand sources files using glob. 333 sources = glob.glob(src_file) 334 if not sources: 335 sources = [src_file] 336 337 if len(sources) > 1 and not dest_file.endswith('/'): 338 buildbot_common.ErrorExit("Target file must end in '/' when " 339 "using globbing to install multiple files") 340 341 for source in sources: 342 if dest_file.endswith('/'): 343 dest = os.path.join(dest_file, os.path.basename(source)) 344 else: 345 dest = dest_file 346 dest = os.path.join(dest_root, dest) 347 if not os.path.isdir(os.path.dirname(dest)): 348 buildbot_common.MakeDir(os.path.dirname(dest)) 349 buildbot_common.CopyFile(source, dest) 350 351 352def InstallNaClHeaders(tc_dst_inc, tc_name): 353 """Copies NaCl headers to expected locations in the toolchain.""" 354 if tc_name == 'arm': 355 # arm toolchain header should be the same as the x86 newlib 356 # ones 357 tc_name = 'newlib' 358 359 InstallFiles(NACL_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name]) 360 361 362def MakeNinjaRelPath(path): 363 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path) 364 365 366TOOLCHAIN_LIBS = { 367 'newlib' : [ 368 'crti.o', 369 'crtn.o', 370 'libminidump_generator.a', 371 'libnacl.a', 372 'libnacl_dyncode.a', 373 'libnacl_exception.a', 374 'libnacl_list_mappings.a', 375 'libnosys.a', 376 'libppapi.a', 377 'libppapi_stub.a', 378 'libpthread.a', 379 ], 380 'glibc': [ 381 'libminidump_generator.a', 382 'libminidump_generator.so', 383 'libnacl.a', 384 'libnacl_dyncode.a', 385 'libnacl_dyncode.so', 386 'libnacl_exception.a', 387 'libnacl_exception.so', 388 'libnacl_list_mappings.a', 389 'libnacl_list_mappings.so', 390 'libppapi.a', 391 'libppapi.so', 392 'libppapi_stub.a', 393 ], 394 'pnacl': [ 395 'libminidump_generator.a', 396 'libnacl.a', 397 'libnacl_dyncode.a', 398 'libnacl_exception.a', 399 'libnacl_list_mappings.a', 400 'libnosys.a', 401 'libppapi.a', 402 'libppapi_stub.a', 403 'libpthread.a', 404 ] 405} 406 407 408def GypNinjaInstall(pepperdir, toolchains): 409 build_dir = GYPBUILD_DIR 410 ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release') 411 tools_files = [ 412 ['sel_ldr', 'sel_ldr_x86_32'], 413 ['ncval_new', 'ncval'], 414 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'], 415 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'], 416 ] 417 418 platform = getos.GetPlatform() 419 420 # TODO(binji): dump_syms doesn't currently build on Windows. See 421 # http://crbug.com/245456 422 if platform != 'win': 423 tools_files += [ 424 ['dump_syms', 'dump_syms'], 425 ['minidump_dump', 'minidump_dump'], 426 ['minidump_stackwalk', 'minidump_stackwalk'] 427 ] 428 429 if platform != 'mac': 430 # Mac doesn't build 64-bit binaries. 431 tools_files.append(['sel_ldr64', 'sel_ldr_x86_64']) 432 433 if platform == 'linux': 434 tools_files.append(['nacl_helper_bootstrap', 435 'nacl_helper_bootstrap_x86_32']) 436 tools_files.append(['nacl_helper_bootstrap64', 437 'nacl_helper_bootstrap_x86_64']) 438 439 buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) 440 441 # Add .exe extensions to all windows tools 442 for pair in tools_files: 443 if platform == 'win' and not pair[0].endswith('.nexe'): 444 pair[0] += '.exe' 445 pair[1] += '.exe' 446 447 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files) 448 449 for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']): 450 if tc == 'pnacl': 451 xarches = (None,) 452 else: 453 xarches = ('arm', '32', '64') 454 455 for xarch in xarches: 456 if tc == 'glibc' and xarch == 'arm': 457 continue 458 459 src_dir = GetGypBuiltLib(tc, xarch) 460 dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch) 461 InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc]) 462 463 if tc != 'pnacl': 464 src_dir = GetGypToolchainLib(tc, xarch) 465 InstallFiles(src_dir, dst_dir, ['crt1.o']) 466 467 468def GypNinjaBuild_NaCl(rel_out_dir): 469 gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl') 470 nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp') 471 all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp') 472 473 out_dir = MakeNinjaRelPath(rel_out_dir) 474 out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm') 475 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir) 476 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm) 477 GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir) 478 479 platform = getos.GetPlatform() 480 if platform == 'win': 481 NinjaBuild('sel_ldr64', out_dir) 482 elif platform == 'linux': 483 out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64') 484 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64) 485 486 # We only need sel_ldr from the 64-bit out directory. 487 # sel_ldr needs to be renamed, so we'll call it sel_ldr64. 488 files_to_copy = [ 489 ('sel_ldr', 'sel_ldr64'), 490 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'), 491 ] 492 493 for src, dst in files_to_copy: 494 buildbot_common.CopyFile( 495 os.path.join(SRC_DIR, out_dir_64, 'Release', src), 496 os.path.join(SRC_DIR, out_dir, 'Release', dst)) 497 498 499def GypNinjaBuild_Breakpad(rel_out_dir): 500 # TODO(binji): dump_syms doesn't currently build on Windows. See 501 # http://crbug.com/245456 502 if getos.GetPlatform() == 'win': 503 return 504 505 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 506 out_dir = MakeNinjaRelPath(rel_out_dir) 507 gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp') 508 build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk'] 509 GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir) 510 511 512def GypNinjaBuild_PPAPI(arch, rel_out_dir): 513 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 514 out_dir = MakeNinjaRelPath(rel_out_dir) 515 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 516 'native_client.gyp') 517 GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir) 518 519 520def GypNinjaBuild_Pnacl(rel_out_dir, target_arch): 521 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the 522 # Chromium build, and once here. When we move more of the SDK build process 523 # to gyp, we can remove this. 524 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 525 526 out_dir = MakeNinjaRelPath(rel_out_dir) 527 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src', 528 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp') 529 targets = ['pnacl_irt_shim'] 530 GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False) 531 532 533def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, 534 out_dir, force_arm_gcc=True): 535 gyp_env = copy.copy(os.environ) 536 gyp_env['GYP_GENERATORS'] = 'ninja' 537 gyp_defines = [] 538 if options.mac_sdk: 539 gyp_defines.append('mac_sdk=%s' % options.mac_sdk) 540 if arch: 541 gyp_defines.append('target_arch=%s' % arch) 542 if arch == 'arm': 543 gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1'] 544 if force_arm_gcc: 545 gyp_defines += ['nacl_enable_arm_gcc=1'] 546 547 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines) 548 for key in ['GYP_GENERATORS', 'GYP_DEFINES']: 549 value = gyp_env[key] 550 print '%s="%s"' % (key, value) 551 gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)] 552 gyp_depth = '--depth=.' 553 buildbot_common.Run( 554 [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \ 555 gyp_generator_flags, 556 cwd=SRC_DIR, 557 env=gyp_env) 558 NinjaBuild(targets, out_dir) 559 560 561def NinjaBuild(targets, out_dir): 562 if type(targets) is not list: 563 targets = [targets] 564 out_config_dir = os.path.join(out_dir, 'Release') 565 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR) 566 567 568def BuildStepBuildToolchains(pepperdir, toolchains): 569 buildbot_common.BuildStep('SDK Items') 570 571 GypNinjaBuild_NaCl(GYPBUILD_DIR) 572 GypNinjaBuild_Breakpad(GYPBUILD_DIR) 573 574 platform = getos.GetPlatform() 575 newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib') 576 glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc') 577 armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib') 578 pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl') 579 580 if set(toolchains) & set(['glibc', 'newlib']): 581 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR) 582 583 if 'arm' in toolchains: 584 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm') 585 586 GypNinjaInstall(pepperdir, toolchains) 587 588 if 'newlib' in toolchains: 589 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'), 590 'newlib') 591 592 if 'glibc' in toolchains: 593 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'), 594 'glibc') 595 596 if 'arm' in toolchains: 597 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'), 598 'arm') 599 600 if 'pnacl' in toolchains: 601 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default. 602 for arch in ('ia32', 'arm'): 603 # Fill in the latest native pnacl shim library from the chrome build. 604 build_dir = GYPBUILD_DIR + '-pnacl-' + arch 605 GypNinjaBuild_Pnacl(build_dir, arch) 606 if arch == 'ia32': 607 nacl_arches = ['x86-32', 'x86-64'] 608 elif arch == 'arm': 609 nacl_arches = ['arm'] 610 else: 611 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 612 for nacl_arch in nacl_arches: 613 release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release', 614 'gen', 'tc_pnacl_translate', 615 'lib-' + nacl_arch) 616 617 buildbot_common.CopyFile( 618 os.path.join(release_build_dir, 'libpnacl_irt_shim.a'), 619 GetPNaClNativeLib(pnacldir, nacl_arch)) 620 621 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'), 622 'newlib') 623 624 625def MakeDirectoryOrClobber(pepperdir, dirname, clobber): 626 dirpath = os.path.join(pepperdir, dirname) 627 if clobber: 628 buildbot_common.RemoveDir(dirpath) 629 buildbot_common.MakeDir(dirpath) 630 631 return dirpath 632 633 634def BuildStepUpdateHelpers(pepperdir, clobber): 635 buildbot_common.BuildStep('Update project helpers') 636 build_projects.UpdateHelpers(pepperdir, clobber=clobber) 637 638 639def BuildStepUpdateUserProjects(pepperdir, toolchains, 640 build_experimental, clobber): 641 buildbot_common.BuildStep('Update examples and libraries') 642 643 filters = {} 644 if not build_experimental: 645 filters['EXPERIMENTAL'] = False 646 if toolchains: 647 toolchains = toolchains[:] 648 649 # arm isn't a valid toolchain for build_projects 650 if 'arm' in toolchains: 651 toolchains.remove('arm') 652 653 if 'host' in toolchains: 654 toolchains.remove('host') 655 toolchains.append(getos.GetPlatform()) 656 657 filters['TOOLS'] = toolchains 658 659 # Update examples and libraries 660 filters['DEST'] = [ 661 'getting_started', 662 'examples/api', 663 'examples/demo', 664 'examples/tutorial', 665 'src' 666 ] 667 668 tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters) 669 build_projects.UpdateProjects(pepperdir, tree, clobber=clobber, 670 toolchains=toolchains) 671 672 673def BuildStepMakeAll(pepperdir, directory, step_name, 674 deps=True, clean=False, config='Debug', args=None): 675 buildbot_common.BuildStep(step_name) 676 build_projects.BuildProjectsBranch(pepperdir, directory, clean, 677 deps, config, args) 678 679 680def BuildStepBuildLibraries(pepperdir, directory): 681 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug', 682 clean=True, config='Debug') 683 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release', 684 clean=True, config='Release') 685 686 # Cleanup .pyc file generated while building libraries. Without 687 # this we would end up shipping the pyc in the SDK tarball. 688 buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc')) 689 690 691def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None): 692 # Look for LICENSE files 693 license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT') 694 695 license_files = [] 696 for root, _, files in os.walk(fileroot): 697 for filename in files: 698 if license_filenames_re.match(filename): 699 path = os.path.join(root, filename) 700 license_files.append(path) 701 702 if extra_files: 703 license_files += [os.path.join(fileroot, f) for f in extra_files] 704 print '\n'.join(license_files) 705 706 if not os.path.isabs(output_filename): 707 output_filename = os.path.join(fileroot, output_filename) 708 generate_notice.Generate(output_filename, fileroot, license_files) 709 710 711def BuildStepVerifyFilelist(pepperdir): 712 buildbot_common.BuildStep('Verify SDK Files') 713 file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list') 714 try: 715 verify_filelist.Verify(file_list_path, pepperdir) 716 print 'OK' 717 except verify_filelist.ParseException, e: 718 buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e) 719 except verify_filelist.VerifyException, e: 720 file_list_rel = os.path.relpath(file_list_path) 721 verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py' 722 verify_filelist_py = os.path.relpath(verify_filelist_py) 723 pepperdir_rel = os.path.relpath(pepperdir) 724 725 msg = """\ 726SDK verification failed: 727 728%s 729Add/remove files from %s to fix. 730 731Run: 732 ./%s %s %s 733to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel, 734 pepperdir_rel) 735 buildbot_common.ErrorExit(msg) 736 737 738def BuildStepTarBundle(pepper_ver, tarfile): 739 buildbot_common.BuildStep('Tar Pepper Bundle') 740 buildbot_common.MakeDir(os.path.dirname(tarfile)) 741 buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 742 'pepper_' + pepper_ver], cwd=NACL_DIR) 743 744 745def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile, 746 archive_url): 747 with open(tarfile, 'rb') as tarfile_stream: 748 archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( 749 tarfile_stream) 750 751 archive = manifest_util.Archive(manifest_util.GetHostOS()) 752 archive.url = archive_url 753 archive.size = archive_size 754 archive.checksum = archive_sha1 755 756 bundle = manifest_util.Bundle('pepper_' + pepper_ver) 757 bundle.revision = int(chrome_revision) 758 bundle.repath = 'pepper_' + pepper_ver 759 bundle.version = int(pepper_ver) 760 bundle.description = ( 761 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % ( 762 pepper_ver, chrome_revision, nacl_revision)) 763 bundle.stability = 'dev' 764 bundle.recommended = 'no' 765 bundle.archives = [archive] 766 return bundle 767 768 769def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision, 770 tarfile): 771 buildbot_common.BuildStep('Archive %s' % name) 772 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 773 build_version.ChromeVersion(),) 774 tarname = os.path.basename(tarfile) 775 tarfile_dir = os.path.dirname(tarfile) 776 buildbot_common.Archive(tarname, bucket_path, tarfile_dir) 777 778 # generate "manifest snippet" for this archive. 779 archive_url = GSTORE + 'nacl_sdk/%s/%s' % ( 780 build_version.ChromeVersion(), tarname) 781 bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, 782 tarfile, archive_url) 783 784 manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') 785 with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: 786 manifest_snippet_stream.write(bundle.GetDataAsString()) 787 788 buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, 789 step_link=False) 790 791 792def BuildStepArchiveSDKTools(): 793 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot. 794 builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') 795 if builder_name == 'linux-sdk-multi': 796 buildbot_common.BuildStep('Build SDK Tools') 797 build_updater.BuildUpdater(OUT_DIR) 798 799 buildbot_common.BuildStep('Archive SDK Tools') 800 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 801 build_version.ChromeVersion(),) 802 buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, 803 step_link=False) 804 buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR, 805 step_link=False) 806 807 808def BuildStepSyncNaClPorts(): 809 """Pull the pinned revision of naclports from SVN.""" 810 buildbot_common.BuildStep('Sync naclports') 811 if not os.path.exists(NACLPORTS_DIR): 812 # checkout new copy of naclports 813 cmd = ['svn', 'checkout', '-q', '-r', str(NACLPORTS_REV), NACLPORTS_URL, 814 'naclports'] 815 buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR)) 816 else: 817 # sync existing copy to pinned revision. 818 cmd = ['svn', 'update', '-r', str(NACLPORTS_REV)] 819 buildbot_common.Run(cmd, cwd=NACLPORTS_DIR) 820 821 822def BuildStepBuildNaClPorts(pepper_ver, pepperdir): 823 """Build selected naclports in all configurations.""" 824 # TODO(sbc): currently naclports doesn't know anything about 825 # Debug builds so the Debug subfolders are all empty. 826 bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 827 828 env = dict(os.environ) 829 env['NACL_SDK_ROOT'] = pepperdir 830 env['NACLPORTS_NO_ANNOTATE'] = "1" 831 env['NACLPORTS_NO_UPLOAD'] = "1" 832 833 build_script = 'build_tools/bots/linux/naclports-linux-sdk-bundle.sh' 834 buildbot_common.BuildStep('Build naclports') 835 buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR) 836 837 out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver) 838 839 # Some naclports do not include a standalone LICENSE/COPYING file 840 # so we explicitly list those here for inclusion. 841 extra_licenses = ('tinyxml/readme.txt', 842 'jpeg-8d/README', 843 'zlib-1.2.3/README') 844 src_root = os.path.join(NACLPORTS_DIR, 'out', 'repository-i686') 845 output_license = os.path.join(out_dir, 'ports', 'LICENSE') 846 GenerateNotice(src_root , output_license, extra_licenses) 847 readme = os.path.join(out_dir, 'ports', 'README') 848 oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme]) 849 850 851def BuildStepTarNaClPorts(pepper_ver, tarfile): 852 """Create tar archive containing headers and libs from naclports build.""" 853 buildbot_common.BuildStep('Tar naclports Bundle') 854 buildbot_common.MakeDir(os.path.dirname(tarfile)) 855 pepper_dir = 'pepper_%s' % pepper_ver 856 archive_dirs = [os.path.join(pepper_dir, 'ports')] 857 858 ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 859 cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile] 860 cmd += archive_dirs 861 buildbot_common.Run(cmd, cwd=NACL_DIR) 862 863 864def main(args): 865 parser = optparse.OptionParser() 866 parser.add_option('--tar', help='Force the tar step.', 867 action='store_true') 868 parser.add_option('--archive', help='Force the archive step.', 869 action='store_true') 870 parser.add_option('--gyp', 871 help='Use gyp to build examples/libraries/Makefiles.', 872 action='store_true') 873 parser.add_option('--release', help='PPAPI release version.', 874 dest='release', default=None) 875 parser.add_option('--build-ports', 876 help='Build naclport bundle.', action='store_true') 877 parser.add_option('--experimental', 878 help='build experimental examples and libraries', action='store_true', 879 dest='build_experimental') 880 parser.add_option('--skip-toolchain', help='Skip toolchain untar', 881 action='store_true') 882 parser.add_option('--mac_sdk', 883 help='Set the mac_sdk (e.g. 10.6) to use when building with ninja.', 884 dest='mac_sdk') 885 886 global options 887 options, args = parser.parse_args(args[1:]) 888 889 generate_make.use_gyp = options.gyp 890 if buildbot_common.IsSDKBuilder(): 891 options.archive = True 892 options.build_ports = True 893 options.tar = True 894 895 toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host'] 896 print 'Building: ' + ' '.join(toolchains) 897 898 if options.archive and not options.tar: 899 parser.error('Incompatible arguments with archive.') 900 901 chrome_version = int(build_version.ChromeMajorVersion()) 902 chrome_revision = build_version.ChromeRevision() 903 nacl_revision = build_version.NaClRevision() 904 pepper_ver = str(chrome_version) 905 pepper_old = str(chrome_version - 1) 906 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 907 pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old) 908 tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2' 909 tarfile = os.path.join(OUT_DIR, tarname) 910 911 if options.release: 912 pepper_ver = options.release 913 print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision) 914 915 if 'NACL_SDK_ROOT' in os.environ: 916 # We don't want the currently configured NACL_SDK_ROOT to have any effect 917 # of the build. 918 del os.environ['NACL_SDK_ROOT'] 919 920 if not options.skip_toolchain: 921 BuildStepCleanPepperDirs(pepperdir, pepperdir_old) 922 BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools']) 923 BuildStepDownloadToolchains() 924 BuildStepUntarToolchains(pepperdir, toolchains) 925 926 BuildStepBuildToolchains(pepperdir, toolchains) 927 928 BuildStepUpdateHelpers(pepperdir, True) 929 BuildStepUpdateUserProjects(pepperdir, toolchains, 930 options.build_experimental, True) 931 932 BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision) 933 934 # Ship with libraries prebuilt, so run that first. 935 BuildStepBuildLibraries(pepperdir, 'src') 936 GenerateNotice(pepperdir) 937 938 # Verify the SDK contains what we expect. 939 BuildStepVerifyFilelist(pepperdir) 940 941 if options.tar: 942 BuildStepTarBundle(pepper_ver, tarfile) 943 944 if options.build_ports and getos.GetPlatform() == 'linux': 945 ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2') 946 BuildStepSyncNaClPorts() 947 BuildStepBuildNaClPorts(pepper_ver, pepperdir) 948 if options.tar: 949 BuildStepTarNaClPorts(pepper_ver, ports_tarfile) 950 951 # Archive on non-trybots. 952 if options.archive: 953 BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision, 954 tarfile) 955 if options.build_ports and getos.GetPlatform() == 'linux': 956 BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision, 957 nacl_revision, ports_tarfile) 958 BuildStepArchiveSDKTools() 959 960 return 0 961 962 963if __name__ == '__main__': 964 try: 965 sys.exit(main(sys.argv)) 966 except KeyboardInterrupt: 967 buildbot_common.ErrorExit('build_sdk: interrupted') 968