1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Entry point for both build and try bots. 7 8This script is invoked from XXX, usually without arguments 9to package an SDK. It automatically determines whether 10this SDK is for mac, win, linux. 11 12The script inspects the following environment variables: 13 14BUILDBOT_BUILDERNAME to determine whether the script is run locally 15and whether it should upload an SDK to file storage (GSTORE) 16""" 17 18# pylint: disable=W0621 19 20# std python includes 21import copy 22import datetime 23import glob 24import optparse 25import os 26import re 27import sys 28 29if sys.version_info < (2, 6, 0): 30 sys.stderr.write("python 2.6 or later is required run this script\n") 31 sys.exit(1) 32 33# local includes 34import buildbot_common 35import build_projects 36import build_updater 37import build_version 38import generate_make 39import generate_notice 40import manifest_util 41import parse_dsc 42import verify_filelist 43 44from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR 45from build_paths import NACLPORTS_DIR, GSTORE 46 47# Add SDK make tools scripts to the python path. 48sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 49sys.path.append(os.path.join(NACL_DIR, 'build')) 50 51import getos 52import oshelpers 53 54CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') 55 56NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src' 57NACLPORTS_REV = 850 58 59GYPBUILD_DIR = 'gypbuild' 60 61options = None 62 63 64def GetGlibcToolchain(host_arch): 65 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 66 tcname = 'toolchain_%s_%s.tar.bz2' % (getos.GetPlatform(), host_arch) 67 return os.path.join(tcdir, tcname) 68 69 70def GetNewlibToolchain(host_arch): 71 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 72 tcname = 'naclsdk_%s_%s.tgz' % (getos.GetPlatform(), host_arch) 73 return os.path.join(tcdir, tcname) 74 75 76def GetPNaClToolchain(host_arch): 77 tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') 78 tcname = 'naclsdk_pnacl_%s_%s.tgz' % (getos.GetPlatform(), host_arch) 79 return os.path.join(tcdir, tcname) 80 81 82def GetToolchainNaClInclude(tcname, tcpath, arch): 83 if arch == 'x86': 84 if tcname == 'pnacl': 85 return os.path.join(tcpath, 'newlib', 'sdk', 'include') 86 return os.path.join(tcpath, 'x86_64-nacl', 'include') 87 elif arch == 'arm': 88 return os.path.join(tcpath, 'arm-nacl', 'include') 89 else: 90 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 91 92 93def GetGypGenDir(xarch): 94 if xarch == 'arm': 95 build_dir = GYPBUILD_DIR + '-arm' 96 else: 97 build_dir = GYPBUILD_DIR 98 return os.path.join(OUT_DIR, build_dir, 'Release', 'gen') 99 100 101def GetGypBuiltLib(tcname, xarch=None): 102 if tcname == 'pnacl': 103 tcname = 'pnacl_newlib' 104 if not xarch: 105 xarch = '' 106 return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch) 107 108 109def GetToolchainNaClLib(tcname, tcpath, xarch): 110 if tcname == 'pnacl': 111 return os.path.join(tcpath, 'newlib', 'sdk', 'lib') 112 if xarch == '32': 113 return os.path.join(tcpath, 'x86_64-nacl', 'lib32') 114 if xarch == '64': 115 return os.path.join(tcpath, 'x86_64-nacl', 'lib') 116 if xarch == 'arm': 117 return os.path.join(tcpath, 'arm-nacl', 'lib') 118 119 120def GetToolchainDirName(tcname, xarch): 121 if xarch != 'arm': 122 xarch = 'x86' 123 return '%s_%s_%s' % (getos.GetPlatform(), xarch, tcname) 124 125 126def GetGypToolchainLib(tcname, xarch): 127 tcpath = os.path.join(GetGypGenDir(xarch), 'sdk', 'toolchain', 128 GetToolchainDirName(tcname, xarch)) 129 return GetToolchainNaClLib(tcname, tcpath, xarch) 130 131 132def GetOutputToolchainLib(pepperdir, tcname, xarch): 133 tcpath = os.path.join(pepperdir, 'toolchain', 134 GetToolchainDirName(tcname, xarch)) 135 return GetToolchainNaClLib(tcname, tcpath, xarch) 136 137 138def GetPNaClNativeLib(tcpath, arch): 139 if arch not in ['arm', 'x86-32', 'x86-64']: 140 buildbot_common.ErrorExit('Unknown architecture %s.' % arch) 141 return os.path.join(tcpath, 'lib-' + arch) 142 143 144def BuildStepDownloadToolchains(): 145 buildbot_common.BuildStep('Running download_toolchains.py') 146 download_script = os.path.join('build', 'download_toolchains.py') 147 buildbot_common.Run([sys.executable, download_script, 148 '--no-arm-trusted', '--arm-untrusted', '--keep'], 149 cwd=NACL_DIR) 150 151 152def BuildStepCleanPepperDirs(pepperdir, pepperdir_old): 153 buildbot_common.BuildStep('Clean Pepper Dirs') 154 buildbot_common.RemoveDir(pepperdir_old) 155 buildbot_common.RemoveDir(pepperdir) 156 buildbot_common.MakeDir(pepperdir) 157 158 159def BuildStepMakePepperDirs(pepperdir, subdirs): 160 for subdir in subdirs: 161 buildbot_common.MakeDir(os.path.join(pepperdir, subdir)) 162 163TEXT_FILES = [ 164 'AUTHORS', 165 'COPYING', 166 'LICENSE', 167 'README.Makefiles', 168] 169 170def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, 171 nacl_revision): 172 buildbot_common.BuildStep('Add Text Files') 173 InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES) 174 175 # Replace a few placeholders in README 176 readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read() 177 readme_text = readme_text.replace('${VERSION}', pepper_ver) 178 readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision) 179 readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision) 180 181 # Year/Month/Day Hour:Minute:Second 182 time_format = '%Y/%m/%d %H:%M:%S' 183 readme_text = readme_text.replace('${DATE}', 184 datetime.datetime.now().strftime(time_format)) 185 186 open(os.path.join(pepperdir, 'README'), 'w').write(readme_text) 187 188 189def PrunePNaClToolchain(root): 190 dirs_to_prune = [ 191 'newlib/lib-bc-x86-64', 192 'newlib/usr-bc-x86-64' 193 # TODO(sbc): remove this once its really not needed. 194 # Currently we seem to rely on it at least for <bits/stat.h> 195 #'newlib/sysroot', 196 ] 197 for dirname in dirs_to_prune: 198 buildbot_common.RemoveDir(os.path.join(root, dirname)) 199 200 201def BuildStepUntarToolchains(pepperdir, arch, toolchains): 202 buildbot_common.BuildStep('Untar Toolchains') 203 platform = getos.GetPlatform() 204 tcname = platform + '_' + arch 205 tmpdir = os.path.join(OUT_DIR, 'tc_temp') 206 buildbot_common.RemoveDir(tmpdir) 207 buildbot_common.MakeDir(tmpdir) 208 209 if 'newlib' in toolchains: 210 # Untar the newlib toolchains 211 tarfile = GetNewlibToolchain(arch) 212 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 213 cwd=NACL_DIR) 214 215 # Then rename/move it to the pepper toolchain directory 216 srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk') 217 newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib') 218 buildbot_common.Move(srcdir, newlibdir) 219 220 if 'arm' in toolchains: 221 # Copy the existing arm toolchain from native_client tree 222 arm_toolchain = os.path.join(NACL_DIR, 'toolchain', 223 platform + '_arm_newlib') 224 arm_toolchain_sdk = os.path.join(pepperdir, 'toolchain', 225 os.path.basename(arm_toolchain)) 226 buildbot_common.CopyDir(arm_toolchain, arm_toolchain_sdk) 227 228 if 'glibc' in toolchains: 229 # Untar the glibc toolchains 230 tarfile = GetGlibcToolchain(arch) 231 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 232 cwd=NACL_DIR) 233 234 # Then rename/move it to the pepper toolchain directory 235 srcdir = os.path.join(tmpdir, 'toolchain', tcname) 236 glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc') 237 buildbot_common.Move(srcdir, glibcdir) 238 239 # Untar the pnacl toolchains 240 if 'pnacl' in toolchains: 241 tmpdir = os.path.join(tmpdir, 'pnacl') 242 buildbot_common.RemoveDir(tmpdir) 243 buildbot_common.MakeDir(tmpdir) 244 tarfile = GetPNaClToolchain(arch) 245 buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], 246 cwd=NACL_DIR) 247 248 # Then rename/move it to the pepper toolchain directory 249 pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl') 250 buildbot_common.Move(tmpdir, pnacldir) 251 PrunePNaClToolchain(pnacldir) 252 253 buildbot_common.RemoveDir(tmpdir) 254 255 if options.gyp and platform != 'win': 256 # If the gyp options is specified we install a toolchain 257 # wrapper so that gyp can switch toolchains via a commandline 258 # option. 259 bindir = os.path.join(pepperdir, 'toolchain', tcname, 'bin') 260 wrapper = os.path.join(SDK_SRC_DIR, 'tools', 'compiler-wrapper.py') 261 buildbot_common.MakeDir(bindir) 262 buildbot_common.CopyFile(wrapper, bindir) 263 264 # Module 'os' has no 'symlink' member (on Windows). 265 # pylint: disable=E1101 266 267 os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-g++')) 268 os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-gcc')) 269 os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-ar')) 270 271 272# List of toolchain headers to install. 273# Source is relative to native_client tree, destination is relative 274# to the toolchain header directory. 275NACL_HEADER_MAP = { 276 'newlib': [ 277 ('src/include/nacl/nacl_exception.h', 'nacl/'), 278 ('src/include/nacl/nacl_minidump.h', 'nacl/'), 279 ('src/untrusted/irt/irt.h', ''), 280 ('src/untrusted/irt/irt_ppapi.h', ''), 281 ('src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 282 ('src/untrusted/nacl/nacl_startup.h', 'nacl/'), 283 ('src/untrusted/nacl/nacl_thread.h', 'nacl/'), 284 ('src/untrusted/nacl/pnacl.h', ''), 285 ('src/untrusted/pthread/pthread.h', ''), 286 ('src/untrusted/pthread/semaphore.h', ''), 287 ('src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 288 ], 289 'glibc': [ 290 ('src/include/nacl/nacl_exception.h', 'nacl/'), 291 ('src/include/nacl/nacl_minidump.h', 'nacl/'), 292 ('src/untrusted/irt/irt.h', ''), 293 ('src/untrusted/irt/irt_ppapi.h', ''), 294 ('src/untrusted/nacl/nacl_dyncode.h', 'nacl/'), 295 ('src/untrusted/nacl/nacl_startup.h', 'nacl/'), 296 ('src/untrusted/nacl/nacl_thread.h', 'nacl/'), 297 ('src/untrusted/nacl/pnacl.h', ''), 298 ('src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'), 299 ], 300 'host': [] 301} 302 303def InstallFiles(src_root, dest_root, file_list): 304 """Copy a set of files from src_root to dest_root according 305 to the given mapping. This allows files to be copied from 306 to a location in the destination tree that is different to the 307 location in the source tree. 308 309 If the destination mapping ends with a '/' then the destination 310 basename is inherited from the the source file. 311 312 Wildcards can be used in the source list but it is not recommended 313 as this can end up adding things to the SDK unintentionally. 314 """ 315 for file_spec in file_list: 316 # The list of files to install can be a simple list of 317 # strings or a list of pairs, where each pair corresponds 318 # to a mapping from source to destination names. 319 if type(file_spec) == str: 320 src_file = dest_file = file_spec 321 else: 322 src_file, dest_file = file_spec 323 324 src_file = os.path.join(src_root, src_file) 325 326 # Expand sources files using glob. 327 sources = glob.glob(src_file) 328 if not sources: 329 sources = [src_file] 330 331 if len(sources) > 1 and not dest_file.endswith('/'): 332 buildbot_common.ErrorExit("Target file must end in '/' when " 333 "using globbing to install multiple files") 334 335 for source in sources: 336 if dest_file.endswith('/'): 337 dest = os.path.join(dest_file, os.path.basename(source)) 338 else: 339 dest = dest_file 340 dest = os.path.join(dest_root, dest) 341 if not os.path.isdir(os.path.dirname(dest)): 342 buildbot_common.MakeDir(os.path.dirname(dest)) 343 buildbot_common.CopyFile(source, dest) 344 345 346def InstallNaClHeaders(tc_dst_inc, tc_name): 347 """Copies NaCl headers to expected locations in the toolchain.""" 348 if tc_name == 'arm': 349 # arm toolchain header should be the same as the x86 newlib 350 # ones 351 tc_name = 'newlib' 352 353 InstallFiles(NACL_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name]) 354 355 356def MakeNinjaRelPath(path): 357 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path) 358 359 360TOOLCHAIN_LIBS = { 361 'newlib' : [ 362 'crti.o', 363 'crtn.o', 364 'libminidump_generator.a', 365 'libnacl.a', 366 'libnacl_dyncode.a', 367 'libnacl_exception.a', 368 'libnacl_list_mappings.a', 369 'libnosys.a', 370 'libppapi.a', 371 'libppapi_stub.a', 372 'libpthread.a', 373 ], 374 'glibc': [ 375 'libminidump_generator.a', 376 'libminidump_generator.so', 377 'libnacl.a', 378 'libnacl_dyncode.a', 379 'libnacl_dyncode.so', 380 'libnacl_exception.a', 381 'libnacl_exception.so', 382 'libnacl_list_mappings.a', 383 'libnacl_list_mappings.so', 384 'libppapi.a', 385 'libppapi.so', 386 'libppapi_stub.a', 387 ], 388 'pnacl': [ 389 'libminidump_generator.a', 390 'libnacl.a', 391 'libnacl_dyncode.a', 392 'libnacl_exception.a', 393 'libnacl_list_mappings.a', 394 'libnosys.a', 395 'libppapi.a', 396 'libppapi_stub.a', 397 'libpthread.a', 398 ] 399} 400 401 402def GypNinjaInstall(pepperdir, toolchains): 403 build_dir = GYPBUILD_DIR 404 ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release') 405 tools_files = [ 406 ['sel_ldr', 'sel_ldr_x86_32'], 407 ['ncval_new', 'ncval'], 408 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'], 409 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'], 410 ] 411 412 platform = getos.GetPlatform() 413 414 # TODO(binji): dump_syms doesn't currently build on Windows. See 415 # http://crbug.com/245456 416 if platform != 'win': 417 tools_files += [ 418 ['dump_syms', 'dump_syms'], 419 ['minidump_dump', 'minidump_dump'], 420 ['minidump_stackwalk', 'minidump_stackwalk'] 421 ] 422 423 if platform != 'mac': 424 # Mac doesn't build 64-bit binaries. 425 tools_files.append(['sel_ldr64', 'sel_ldr_x86_64']) 426 427 if platform == 'linux': 428 tools_files.append(['nacl_helper_bootstrap', 429 'nacl_helper_bootstrap_x86_32']) 430 tools_files.append(['nacl_helper_bootstrap64', 431 'nacl_helper_bootstrap_x86_64']) 432 433 buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) 434 435 # Add .exe extensions to all windows tools 436 for pair in tools_files: 437 if platform == 'win' and not pair[0].endswith('.nexe'): 438 pair[0] += '.exe' 439 pair[1] += '.exe' 440 441 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files) 442 443 for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']): 444 if tc == 'pnacl': 445 xarches = (None,) 446 else: 447 xarches = ('arm', '32', '64') 448 449 for xarch in xarches: 450 if tc == 'glibc' and xarch == 'arm': 451 continue 452 453 src_dir = GetGypBuiltLib(tc, xarch) 454 dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch) 455 InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc]) 456 457 if tc != 'pnacl': 458 src_dir = GetGypToolchainLib(tc, xarch) 459 InstallFiles(src_dir, dst_dir, ['crt1.o']) 460 461 462def GypNinjaBuild_NaCl(rel_out_dir): 463 gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl') 464 nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp') 465 all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp') 466 467 out_dir = MakeNinjaRelPath(rel_out_dir) 468 out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm') 469 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir) 470 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm) 471 GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir) 472 473 platform = getos.GetPlatform() 474 if platform == 'win': 475 NinjaBuild('sel_ldr64', out_dir) 476 elif platform == 'linux': 477 out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64') 478 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64) 479 480 # We only need sel_ldr from the 64-bit out directory. 481 # sel_ldr needs to be renamed, so we'll call it sel_ldr64. 482 files_to_copy = [ 483 ('sel_ldr', 'sel_ldr64'), 484 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'), 485 ] 486 487 for src, dst in files_to_copy: 488 buildbot_common.CopyFile( 489 os.path.join(SRC_DIR, out_dir_64, 'Release', src), 490 os.path.join(SRC_DIR, out_dir, 'Release', dst)) 491 492 493def GypNinjaBuild_Breakpad(rel_out_dir): 494 # TODO(binji): dump_syms doesn't currently build on Windows. See 495 # http://crbug.com/245456 496 if getos.GetPlatform() == 'win': 497 return 498 499 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 500 out_dir = MakeNinjaRelPath(rel_out_dir) 501 gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp') 502 build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk'] 503 GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir) 504 505 506def GypNinjaBuild_PPAPI(arch, rel_out_dir): 507 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 508 out_dir = MakeNinjaRelPath(rel_out_dir) 509 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 510 'native_client.gyp') 511 GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir) 512 513 514def GypNinjaBuild_Pnacl(rel_out_dir, target_arch): 515 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the 516 # Chromium build, and once here. When we move more of the SDK build process 517 # to gyp, we can remove this. 518 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium') 519 520 out_dir = MakeNinjaRelPath(rel_out_dir) 521 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src', 522 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp') 523 targets = ['pnacl_irt_shim'] 524 GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False) 525 526 527def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, 528 out_dir, force_arm_gcc=True): 529 gyp_env = copy.copy(os.environ) 530 gyp_env['GYP_GENERATORS'] = 'ninja' 531 gyp_defines = [] 532 if options.mac_sdk: 533 gyp_defines.append('mac_sdk=%s' % options.mac_sdk) 534 if arch: 535 gyp_defines.append('target_arch=%s' % arch) 536 if arch == 'arm': 537 gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1'] 538 if force_arm_gcc: 539 gyp_defines += ['nacl_enable_arm_gcc=1'] 540 541 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines) 542 for key in ['GYP_GENERATORS', 'GYP_DEFINES']: 543 value = gyp_env[key] 544 print '%s="%s"' % (key, value) 545 gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)] 546 gyp_depth = '--depth=.' 547 buildbot_common.Run( 548 [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \ 549 gyp_generator_flags, 550 cwd=SRC_DIR, 551 env=gyp_env) 552 NinjaBuild(targets, out_dir) 553 554 555def NinjaBuild(targets, out_dir): 556 if type(targets) is not list: 557 targets = [targets] 558 out_config_dir = os.path.join(out_dir, 'Release') 559 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR) 560 561 562def BuildStepBuildToolchains(pepperdir, toolchains): 563 buildbot_common.BuildStep('SDK Items') 564 565 GypNinjaBuild_NaCl(GYPBUILD_DIR) 566 GypNinjaBuild_Breakpad(GYPBUILD_DIR) 567 568 platform = getos.GetPlatform() 569 tcname = platform + '_x86' 570 newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib') 571 glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc') 572 pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl') 573 574 if set(toolchains) & set(['glibc', 'newlib']): 575 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR) 576 577 if 'arm' in toolchains: 578 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm') 579 580 GypNinjaInstall(pepperdir, toolchains) 581 582 if 'newlib' in toolchains: 583 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'), 584 'newlib') 585 586 if 'glibc' in toolchains: 587 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'), 588 'glibc') 589 590 if 'arm' in toolchains: 591 tcname = platform + '_arm_newlib' 592 armdir = os.path.join(pepperdir, 'toolchain', tcname) 593 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'), 594 'arm') 595 596 if 'pnacl' in toolchains: 597 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default. 598 for arch in ('ia32', 'arm'): 599 # Fill in the latest native pnacl shim library from the chrome build. 600 build_dir = GYPBUILD_DIR + '-pnacl-' + arch 601 GypNinjaBuild_Pnacl(build_dir, arch) 602 if arch == 'ia32': 603 nacl_arches = ['x86-32', 'x86-64'] 604 elif arch == 'arm': 605 nacl_arches = ['arm'] 606 else: 607 buildbot_common.ErrorExit('Unknown architecture: %s' % arch) 608 for nacl_arch in nacl_arches: 609 release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release', 610 'gen', 'tc_pnacl_translate', 611 'lib-' + nacl_arch) 612 613 buildbot_common.CopyFile( 614 os.path.join(release_build_dir, 'libpnacl_irt_shim.a'), 615 GetPNaClNativeLib(pnacldir, nacl_arch)) 616 617 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'), 618 'newlib') 619 620 621def MakeDirectoryOrClobber(pepperdir, dirname, clobber): 622 dirpath = os.path.join(pepperdir, dirname) 623 if clobber: 624 buildbot_common.RemoveDir(dirpath) 625 buildbot_common.MakeDir(dirpath) 626 627 return dirpath 628 629 630def BuildStepUpdateHelpers(pepperdir, clobber): 631 buildbot_common.BuildStep('Update project helpers') 632 build_projects.UpdateHelpers(pepperdir, clobber=clobber) 633 634 635def BuildStepUpdateUserProjects(pepperdir, toolchains, 636 build_experimental, clobber): 637 buildbot_common.BuildStep('Update examples and libraries') 638 639 filters = {} 640 if not build_experimental: 641 filters['EXPERIMENTAL'] = False 642 if toolchains: 643 toolchains = toolchains[:] 644 645 # arm isn't a valid toolchain for build_projects 646 if 'arm' in toolchains: 647 toolchains.remove('arm') 648 649 if 'host' in toolchains: 650 toolchains.remove('host') 651 toolchains.append(getos.GetPlatform()) 652 653 filters['TOOLS'] = toolchains 654 655 # Update examples and libraries 656 filters['DEST'] = [ 657 'examples/api', 658 'examples/demo', 659 'examples/getting_started', 660 'examples/tutorial', 661 'src' 662 ] 663 664 tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters) 665 build_projects.UpdateProjects(pepperdir, tree, clobber=clobber, 666 toolchains=toolchains) 667 668 669def BuildStepMakeAll(pepperdir, directory, step_name, 670 deps=True, clean=False, config='Debug', args=None): 671 buildbot_common.BuildStep(step_name) 672 build_projects.BuildProjectsBranch(pepperdir, directory, clean, 673 deps, config, args) 674 675 676def BuildStepBuildLibraries(pepperdir, directory): 677 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug', 678 clean=True, config='Debug') 679 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release', 680 clean=True, config='Release') 681 682 683def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None): 684 # Look for LICENSE files 685 license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT') 686 687 license_files = [] 688 for root, _, files in os.walk(fileroot): 689 for filename in files: 690 if license_filenames_re.match(filename): 691 path = os.path.join(root, filename) 692 license_files.append(path) 693 694 if extra_files: 695 license_files += [os.path.join(fileroot, f) for f in extra_files] 696 print '\n'.join(license_files) 697 698 if not os.path.isabs(output_filename): 699 output_filename = os.path.join(fileroot, output_filename) 700 generate_notice.Generate(output_filename, fileroot, license_files) 701 702 703def BuildStepVerifyFilelist(pepperdir): 704 buildbot_common.BuildStep('Verify SDK Files') 705 file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list') 706 try: 707 verify_filelist.Verify(file_list_path, pepperdir) 708 print 'OK' 709 except verify_filelist.ParseException, e: 710 buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e) 711 except verify_filelist.VerifyException, e: 712 file_list_rel = os.path.relpath(file_list_path) 713 verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py' 714 verify_filelist_py = os.path.relpath(verify_filelist_py) 715 pepperdir_rel = os.path.relpath(pepperdir) 716 717 msg = """\ 718SDK verification failed: 719 720%s 721Add/remove files from %s to fix. 722 723Run: 724 ./%s %s %s 725to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel, 726 pepperdir_rel) 727 buildbot_common.ErrorExit(msg) 728 729 730 731def BuildStepTarBundle(pepper_ver, tarfile): 732 buildbot_common.BuildStep('Tar Pepper Bundle') 733 buildbot_common.MakeDir(os.path.dirname(tarfile)) 734 buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 735 'pepper_' + pepper_ver], cwd=NACL_DIR) 736 737 738 739def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile, 740 archive_url): 741 with open(tarfile, 'rb') as tarfile_stream: 742 archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( 743 tarfile_stream) 744 745 archive = manifest_util.Archive(manifest_util.GetHostOS()) 746 archive.url = archive_url 747 archive.size = archive_size 748 archive.checksum = archive_sha1 749 750 bundle = manifest_util.Bundle('pepper_' + pepper_ver) 751 bundle.revision = int(chrome_revision) 752 bundle.repath = 'pepper_' + pepper_ver 753 bundle.version = int(pepper_ver) 754 bundle.description = ( 755 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % ( 756 pepper_ver, chrome_revision, nacl_revision)) 757 bundle.stability = 'dev' 758 bundle.recommended = 'no' 759 bundle.archives = [archive] 760 return bundle 761 762 763def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision, 764 tarfile): 765 buildbot_common.BuildStep('Archive %s' % name) 766 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 767 build_version.ChromeVersion(),) 768 tarname = os.path.basename(tarfile) 769 tarfile_dir = os.path.dirname(tarfile) 770 buildbot_common.Archive(tarname, bucket_path, tarfile_dir) 771 772 # generate "manifest snippet" for this archive. 773 archive_url = GSTORE + 'nacl_sdk/%s/%s' % ( 774 build_version.ChromeVersion(), tarname) 775 bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, 776 tarfile, archive_url) 777 778 manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') 779 with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: 780 manifest_snippet_stream.write(bundle.GetDataAsString()) 781 782 buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, 783 step_link=False) 784 785 786def BuildStepArchiveSDKTools(): 787 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot. 788 builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') 789 if builder_name == 'linux-sdk-multi': 790 buildbot_common.BuildStep('Build SDK Tools') 791 build_updater.BuildUpdater(OUT_DIR) 792 793 buildbot_common.BuildStep('Archive SDK Tools') 794 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( 795 build_version.ChromeVersion(),) 796 buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, 797 step_link=False) 798 buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR, 799 step_link=False) 800 801 802def BuildStepSyncNaClPorts(): 803 """Pull the pinned revision of naclports from SVN.""" 804 buildbot_common.BuildStep('Sync naclports') 805 if not os.path.exists(NACLPORTS_DIR): 806 # checkout new copy of naclports 807 cmd = ['svn', 'checkout', '-q', '-r', str(NACLPORTS_REV), NACLPORTS_URL, 808 'naclports'] 809 buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR)) 810 else: 811 # sync existing copy to pinned revision. 812 cmd = ['svn', 'update', '-r', str(NACLPORTS_REV)] 813 buildbot_common.Run(cmd, cwd=NACLPORTS_DIR) 814 815 816def BuildStepBuildNaClPorts(pepper_ver, pepperdir): 817 """Build selected naclports in all configurations.""" 818 # TODO(sbc): currently naclports doesn't know anything about 819 # Debug builds so the Debug subfolders are all empty. 820 bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 821 822 env = dict(os.environ) 823 env['NACL_SDK_ROOT'] = pepperdir 824 env['NACLPORTS_NO_ANNOTATE'] = "1" 825 env['NACLPORTS_NO_UPLOAD'] = "1" 826 827 build_script = 'build_tools/bots/linux/naclports-linux-sdk-bundle.sh' 828 buildbot_common.BuildStep('Build naclports') 829 buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR) 830 831 out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver) 832 833 # Some naclports do not include a standalone LICENSE/COPYING file 834 # so we explicitly list those here for inclusion. 835 extra_licenses = ('tinyxml/readme.txt', 836 'jpeg-8d/README', 837 'zlib-1.2.3/README') 838 src_root = os.path.join(NACLPORTS_DIR, 'out', 'repository-i686') 839 output_license = os.path.join(out_dir, 'ports', 'LICENSE') 840 GenerateNotice(src_root , output_license, extra_licenses) 841 readme = os.path.join(out_dir, 'ports', 'README') 842 oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme]) 843 844 845def BuildStepTarNaClPorts(pepper_ver, tarfile): 846 """Create tar archive containing headers and libs from naclports build.""" 847 buildbot_common.BuildStep('Tar naclports Bundle') 848 buildbot_common.MakeDir(os.path.dirname(tarfile)) 849 pepper_dir = 'pepper_%s' % pepper_ver 850 archive_dirs = [os.path.join(pepper_dir, 'ports')] 851 852 ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle') 853 cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile] 854 cmd += archive_dirs 855 buildbot_common.Run(cmd, cwd=NACL_DIR) 856 857 858def main(args): 859 parser = optparse.OptionParser() 860 parser.add_option('--skip-tar', help='Skip generating a tarball.', 861 action='store_true') 862 parser.add_option('--archive', help='Force the archive step.', 863 action='store_true') 864 parser.add_option('--gyp', 865 help='Use gyp to build examples/libraries/Makefiles.', 866 action='store_true') 867 parser.add_option('--release', help='PPAPI release version.', 868 dest='release', default=None) 869 parser.add_option('--build-ports', 870 help='Build naclport bundle.', action='store_true') 871 parser.add_option('--experimental', 872 help='build experimental examples and libraries', action='store_true', 873 dest='build_experimental') 874 parser.add_option('--skip-toolchain', help='Skip toolchain untar', 875 action='store_true') 876 parser.add_option('--mac_sdk', 877 help='Set the mac_sdk (e.g. 10.6) to use when building with ninja.', 878 dest='mac_sdk') 879 880 global options 881 options, args = parser.parse_args(args[1:]) 882 arch = 'x86' 883 884 generate_make.use_gyp = options.gyp 885 if buildbot_common.IsSDKBuilder(): 886 options.archive = True 887 options.build_ports = True 888 889 toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host'] 890 print 'Building: ' + ' '.join(toolchains) 891 892 if options.archive and options.skip_tar: 893 parser.error('Incompatible arguments with archive.') 894 895 chrome_version = int(build_version.ChromeMajorVersion()) 896 chrome_revision = build_version.ChromeRevision() 897 nacl_revision = build_version.NaClRevision() 898 pepper_ver = str(chrome_version) 899 pepper_old = str(chrome_version - 1) 900 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 901 pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old) 902 tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2' 903 tarfile = os.path.join(OUT_DIR, tarname) 904 905 if options.release: 906 pepper_ver = options.release 907 print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision) 908 909 if 'NACL_SDK_ROOT' in os.environ: 910 # We don't want the currently configured NACL_SDK_ROOT to have any effect 911 # of the build. 912 del os.environ['NACL_SDK_ROOT'] 913 914 if not options.skip_toolchain: 915 BuildStepCleanPepperDirs(pepperdir, pepperdir_old) 916 BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools']) 917 BuildStepDownloadToolchains() 918 BuildStepUntarToolchains(pepperdir, arch, toolchains) 919 920 BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision) 921 BuildStepBuildToolchains(pepperdir, toolchains) 922 923 BuildStepUpdateHelpers(pepperdir, True) 924 BuildStepUpdateUserProjects(pepperdir, toolchains, 925 options.build_experimental, True) 926 927 # Ship with libraries prebuilt, so run that first. 928 BuildStepBuildLibraries(pepperdir, 'src') 929 GenerateNotice(pepperdir) 930 931 # Verify the SDK contains what we expect. 932 BuildStepVerifyFilelist(pepperdir) 933 934 if not options.skip_tar: 935 BuildStepTarBundle(pepper_ver, tarfile) 936 937 if options.build_ports and getos.GetPlatform() == 'linux': 938 ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2') 939 BuildStepSyncNaClPorts() 940 BuildStepBuildNaClPorts(pepper_ver, pepperdir) 941 if not options.skip_tar: 942 BuildStepTarNaClPorts(pepper_ver, ports_tarfile) 943 944 # Archive on non-trybots. 945 if options.archive: 946 BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision, 947 tarfile) 948 if options.build_ports and getos.GetPlatform() == 'linux': 949 BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision, 950 nacl_revision, ports_tarfile) 951 BuildStepArchiveSDKTools() 952 953 return 0 954 955 956if __name__ == '__main__': 957 try: 958 sys.exit(main(sys.argv)) 959 except KeyboardInterrupt: 960 buildbot_common.ErrorExit('build_sdk: interrupted') 961