1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Entry point for both build and try bots.
7
8This script is invoked from XXX, usually without arguments
9to package an SDK. It automatically determines whether
10this SDK is for mac, win, linux.
11
12The script inspects the following environment variables:
13
14BUILDBOT_BUILDERNAME to determine whether the script is run locally
15and whether it should upload an SDK to file storage (GSTORE)
16"""
17
18# pylint: disable=W0621
19
20# std python includes
21import datetime
22import glob
23import optparse
24import os
25import re
26import sys
27
28if sys.version_info < (2, 6, 0):
29  sys.stderr.write("python 2.6 or later is required run this script\n")
30  sys.exit(1)
31
32# local includes
33import buildbot_common
34import build_projects
35import build_updater
36import build_version
37import generate_notice
38import manifest_util
39import parse_dsc
40import verify_filelist
41
42from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
43from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR
44
45# Add SDK make tools scripts to the python path.
46sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
47sys.path.append(os.path.join(NACL_DIR, 'build'))
48
49import getos
50import oshelpers
51
52BUILD_DIR = os.path.join(NACL_DIR, 'build')
53NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
54NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
55
56CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
57PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
58
59NACLPORTS_URL = 'https://chromium.googlesource.com/external/naclports.git'
60NACLPORTS_REV = '99f2417'
61
62GYPBUILD_DIR = 'gypbuild'
63
64options = None
65
66# Map of: ToolchainName: (PackageName, SDKDir).
67TOOLCHAIN_PACKAGE_MAP = {
68    'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
69    'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
70    'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
71    'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
72    'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
73    }
74
75
76def GetToolchainNaClInclude(tcname, tcpath, arch):
77  if arch == 'x86':
78    if tcname == 'pnacl':
79      return os.path.join(tcpath, 'le32-nacl', 'include')
80    return os.path.join(tcpath, 'x86_64-nacl', 'include')
81  elif arch == 'arm':
82    return os.path.join(tcpath, 'arm-nacl', 'include')
83  else:
84    buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
85
86
87def GetGypGenDir(xarch):
88  if xarch == 'arm':
89    build_dir = GYPBUILD_DIR + '-arm'
90  else:
91    build_dir = GYPBUILD_DIR
92  return os.path.join(OUT_DIR, build_dir, 'Release', 'gen')
93
94
95def GetGypBuiltLib(tcname, xarch=None):
96  if tcname == 'pnacl':
97    tcname = 'pnacl_newlib'
98  if not xarch:
99    xarch = ''
100  return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch)
101
102
103def GetToolchainNaClLib(tcname, tcpath, xarch):
104  if tcname == 'pnacl':
105    return os.path.join(tcpath, 'le32-nacl', 'lib')
106  elif xarch == '32':
107    return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
108  elif xarch == '64':
109    return os.path.join(tcpath, 'x86_64-nacl', 'lib')
110  elif xarch == 'arm':
111    return os.path.join(tcpath, 'arm-nacl', 'lib')
112
113
114def GetToolchainDirName(tcname, xarch):
115  if tcname == 'pnacl':
116    return '%s_%s' % (getos.GetPlatform(), tcname)
117  elif xarch == 'arm':
118    return '%s_arm_%s' % (getos.GetPlatform(), tcname)
119  else:
120    return '%s_x86_%s' % (getos.GetPlatform(), tcname)
121
122
123def GetGypToolchainLib(tcname, xarch):
124  if xarch == 'arm':
125    toolchain = xarch
126  else:
127    toolchain = tcname
128
129  tcpath = os.path.join(GetGypGenDir(xarch), 'sdk',
130                        '%s_x86' % getos.GetPlatform(),
131                        TOOLCHAIN_PACKAGE_MAP[toolchain][0])
132  return GetToolchainNaClLib(tcname, tcpath, xarch)
133
134
135def GetOutputToolchainLib(pepperdir, tcname, xarch):
136  tcpath = os.path.join(pepperdir, 'toolchain',
137                        GetToolchainDirName(tcname, xarch))
138  return GetToolchainNaClLib(tcname, tcpath, xarch)
139
140
141def GetPNaClTranslatorLib(tcpath, arch):
142  if arch not in ['arm', 'x86-32', 'x86-64']:
143    buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
144  return os.path.join(tcpath, 'translator', arch, 'lib')
145
146
147def BuildStepDownloadToolchains(toolchains):
148  buildbot_common.BuildStep('Running package_version.py')
149  args = [sys.executable, PKGVER, '--exclude', 'arm_trusted']
150  if 'bionic' in toolchains:
151    build_platform = '%s_x86' % getos.GetPlatform()
152    args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
153  args.append('sync')
154  buildbot_common.Run(args, cwd=NACL_DIR)
155
156
157def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
158  buildbot_common.BuildStep('Clean Pepper Dirs')
159  buildbot_common.RemoveDir(pepperdir_old)
160  buildbot_common.RemoveDir(pepperdir)
161  buildbot_common.MakeDir(pepperdir)
162
163
164def BuildStepMakePepperDirs(pepperdir, subdirs):
165  for subdir in subdirs:
166    buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
167
168TEXT_FILES = [
169  'AUTHORS',
170  'COPYING',
171  'LICENSE',
172  'README.Makefiles',
173  'getting_started/README',
174]
175
176def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
177                           nacl_revision):
178  buildbot_common.BuildStep('Add Text Files')
179  InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
180
181  # Replace a few placeholders in README
182  readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
183  readme_text = readme_text.replace('${VERSION}', pepper_ver)
184  readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
185  readme_text = readme_text.replace('${CHROME_COMMIT_POSITION}',
186                                    build_version.ChromeCommitPosition())
187  readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
188
189  # Year/Month/Day Hour:Minute:Second
190  time_format = '%Y/%m/%d %H:%M:%S'
191  readme_text = readme_text.replace('${DATE}',
192      datetime.datetime.now().strftime(time_format))
193
194  open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
195
196
197def BuildStepUntarToolchains(pepperdir, toolchains):
198  buildbot_common.BuildStep('Untar Toolchains')
199  platform = getos.GetPlatform()
200  build_platform = '%s_x86' % platform
201  tmpdir = os.path.join(OUT_DIR, 'tc_temp')
202  buildbot_common.RemoveDir(tmpdir)
203  buildbot_common.MakeDir(tmpdir)
204
205  # Create a list of extract packages tuples, the first part should be
206  # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
207  # directory relative to pepperdir/toolchain.
208  extract_packages = []
209  for toolchain in toolchains:
210    toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
211    if toolchain_map:
212      package_name, tcname = toolchain_map
213      package_tuple = (os.path.join(build_platform, package_name),
214                       tcname % {'platform': platform})
215      extract_packages.append(package_tuple)
216
217  if extract_packages:
218    # Extract all of the packages into the temp directory.
219    package_names = [package_tuple[0] for package_tuple in extract_packages]
220    buildbot_common.Run([sys.executable, PKGVER,
221                           '--packages', ','.join(package_names),
222                           '--tar-dir', NACL_TOOLCHAINTARS_DIR,
223                           '--dest-dir', tmpdir,
224                           'extract'])
225
226    # Move all the packages we extracted to the correct destination.
227    for package_name, dest_dir in extract_packages:
228      full_src_dir = os.path.join(tmpdir, package_name)
229      full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
230      buildbot_common.Move(full_src_dir, full_dst_dir)
231
232  # Cleanup the temporary directory we are no longer using.
233  buildbot_common.RemoveDir(tmpdir)
234
235
236# List of toolchain headers to install.
237# Source is relative to top of Chromium tree, destination is relative
238# to the toolchain header directory.
239NACL_HEADER_MAP = {
240  'newlib': [
241      ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
242      ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
243      ('native_client/src/untrusted/irt/irt.h', ''),
244      ('native_client/src/untrusted/irt/irt_dev.h', ''),
245      ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
246      ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
247      ('native_client/src/untrusted/pthread/pthread.h', ''),
248      ('native_client/src/untrusted/pthread/semaphore.h', ''),
249      ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
250      ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
251  ],
252  'glibc': [
253      ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
254      ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
255      ('native_client/src/untrusted/irt/irt.h', ''),
256      ('native_client/src/untrusted/irt/irt_dev.h', ''),
257      ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
258      ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
259      ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
260      ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
261  ],
262  'bionic': [
263      ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
264  ],
265  'host': []
266}
267
268def InstallFiles(src_root, dest_root, file_list):
269  """Copy a set of files from src_root to dest_root according
270  to the given mapping.  This allows files to be copied from
271  to a location in the destination tree that is different to the
272  location in the source tree.
273
274  If the destination mapping ends with a '/' then the destination
275  basename is inherited from the the source file.
276
277  Wildcards can be used in the source list but it is not recommended
278  as this can end up adding things to the SDK unintentionally.
279  """
280  for file_spec in file_list:
281    # The list of files to install can be a simple list of
282    # strings or a list of pairs, where each pair corresponds
283    # to a mapping from source to destination names.
284    if type(file_spec) == str:
285      src_file = dest_file = file_spec
286    else:
287      src_file, dest_file = file_spec
288
289    src_file = os.path.join(src_root, src_file)
290
291    # Expand sources files using glob.
292    sources = glob.glob(src_file)
293    if not sources:
294      sources = [src_file]
295
296    if len(sources) > 1 and not dest_file.endswith('/'):
297      buildbot_common.ErrorExit("Target file must end in '/' when "
298                                "using globbing to install multiple files")
299
300    for source in sources:
301      if dest_file.endswith('/'):
302        dest = os.path.join(dest_file, os.path.basename(source))
303      else:
304        dest = dest_file
305      dest = os.path.join(dest_root, dest)
306      if not os.path.isdir(os.path.dirname(dest)):
307        buildbot_common.MakeDir(os.path.dirname(dest))
308      buildbot_common.CopyFile(source, dest)
309
310
311def InstallNaClHeaders(tc_dst_inc, tc_name):
312  """Copies NaCl headers to expected locations in the toolchain."""
313  if tc_name == 'arm':
314    # arm toolchain header should be the same as the x86 newlib
315    # ones
316    tc_name = 'newlib'
317
318  InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
319
320
321def MakeNinjaRelPath(path):
322  return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
323
324
325TOOLCHAIN_LIBS = {
326  'bionic' : [
327    'libminidump_generator.a',
328    'libnacl_dyncode.a',
329    'libnacl_exception.a',
330    'libnacl_list_mappings.a',
331    'libppapi.a',
332  ],
333  'newlib' : [
334    'crti.o',
335    'crtn.o',
336    'libminidump_generator.a',
337    'libnacl.a',
338    'libnacl_dyncode.a',
339    'libnacl_exception.a',
340    'libnacl_list_mappings.a',
341    'libnosys.a',
342    'libppapi.a',
343    'libppapi_stub.a',
344    'libpthread.a',
345  ],
346  'glibc': [
347    'libminidump_generator.a',
348    'libminidump_generator.so',
349    'libnacl.a',
350    'libnacl_dyncode.a',
351    'libnacl_dyncode.so',
352    'libnacl_exception.a',
353    'libnacl_exception.so',
354    'libnacl_list_mappings.a',
355    'libnacl_list_mappings.so',
356    'libppapi.a',
357    'libppapi.so',
358    'libppapi_stub.a',
359  ],
360  'pnacl': [
361    'libminidump_generator.a',
362    'libnacl.a',
363    'libnacl_dyncode.a',
364    'libnacl_exception.a',
365    'libnacl_list_mappings.a',
366    'libnosys.a',
367    'libppapi.a',
368    'libppapi_stub.a',
369    'libpthread.a',
370  ]
371}
372
373
374def GypNinjaInstall(pepperdir, toolchains):
375  build_dir = GYPBUILD_DIR
376  ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
377  tools_files = [
378    ['sel_ldr', 'sel_ldr_x86_32'],
379    ['ncval_new', 'ncval'],
380    ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
381    ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
382  ]
383
384  platform = getos.GetPlatform()
385
386  # TODO(binji): dump_syms doesn't currently build on Windows. See
387  # http://crbug.com/245456
388  if platform != 'win':
389    tools_files += [
390      ['dump_syms', 'dump_syms'],
391      ['minidump_dump', 'minidump_dump'],
392      ['minidump_stackwalk', 'minidump_stackwalk']
393    ]
394
395  tools_files.append(['sel_ldr64', 'sel_ldr_x86_64'])
396
397  if platform == 'linux':
398    tools_files.append(['nacl_helper_bootstrap',
399                        'nacl_helper_bootstrap_x86_32'])
400    tools_files.append(['nacl_helper_bootstrap64',
401                        'nacl_helper_bootstrap_x86_64'])
402
403  buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
404
405  # Add .exe extensions to all windows tools
406  for pair in tools_files:
407    if platform == 'win' and not pair[0].endswith('.nexe'):
408      pair[0] += '.exe'
409      pair[1] += '.exe'
410
411  InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
412
413  # Add ARM binaries
414  if platform == 'linux' and not options.no_arm_trusted:
415    tools_files = [
416      ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
417      ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
418      ['sel_ldr', 'sel_ldr_arm'],
419      ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm']
420    ]
421    ninja_out_dir = os.path.join(OUT_DIR, build_dir + '-arm', 'Release')
422    InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
423
424  for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
425    if tc == 'pnacl':
426      xarches = (None,)
427    else:
428      xarches = ('arm', '32', '64')
429
430    for xarch in xarches:
431      if tc == 'glibc' and xarch == 'arm':
432        continue
433
434      src_dir = GetGypBuiltLib(tc, xarch)
435      dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
436      InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
437
438      # Copy ARM newlib components to bionic
439      if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
440        bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
441        InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
442
443      if tc != 'pnacl':
444        src_dir = GetGypToolchainLib(tc, xarch)
445        InstallFiles(src_dir, dst_dir, ['crt1.o'])
446
447
448def GypNinjaBuild_NaCl(rel_out_dir):
449  gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
450  nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
451  all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
452
453  out_dir = MakeNinjaRelPath(rel_out_dir)
454  out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
455  GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir)
456  GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
457  GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir)
458
459  platform = getos.GetPlatform()
460  if platform == 'win':
461    NinjaBuild('sel_ldr64', out_dir)
462  else:
463    out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64')
464    GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64)
465
466    # We only need sel_ldr from the 64-bit out directory.
467    # sel_ldr needs to be renamed, so we'll call it sel_ldr64.
468    files_to_copy = [('sel_ldr', 'sel_ldr64')]
469    if platform == 'linux':
470      files_to_copy.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'))
471
472    for src, dst in files_to_copy:
473      buildbot_common.CopyFile(
474          os.path.join(SRC_DIR, out_dir_64, 'Release', src),
475          os.path.join(SRC_DIR, out_dir, 'Release', dst))
476
477
478def GypNinjaBuild_Breakpad(rel_out_dir):
479  # TODO(binji): dump_syms doesn't currently build on Windows. See
480  # http://crbug.com/245456
481  if getos.GetPlatform() == 'win':
482    return
483
484  gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
485  out_dir = MakeNinjaRelPath(rel_out_dir)
486  gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
487  build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
488  GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir)
489
490
491def GypNinjaBuild_PPAPI(arch, rel_out_dir):
492  gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
493  out_dir = MakeNinjaRelPath(rel_out_dir)
494  gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
495                          'native_client.gyp')
496  GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
497
498
499def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
500  # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
501  # Chromium build, and once here. When we move more of the SDK build process
502  # to gyp, we can remove this.
503  gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
504
505  out_dir = MakeNinjaRelPath(rel_out_dir)
506  gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
507                          'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
508  targets = ['aot']
509  GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False)
510
511
512def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
513                  out_dir, force_arm_gcc=True):
514  gyp_env = dict(os.environ)
515  gyp_env['GYP_GENERATORS'] = 'ninja'
516  gyp_defines = []
517  if options.mac_sdk:
518    gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
519  if arch:
520    gyp_defines.append('target_arch=%s' % arch)
521    if arch == 'arm':
522      if getos.GetPlatform() == 'linux':
523        gyp_env['CC'] = 'arm-linux-gnueabihf-gcc'
524        gyp_env['CXX'] = 'arm-linux-gnueabihf-g++'
525        gyp_env['AR'] = 'arm-linux-gnueabihf-ar'
526        gyp_env['AS'] = 'arm-linux-gnueabihf-as'
527        gyp_env['CC_host'] = 'cc'
528        gyp_env['CXX_host'] = 'c++'
529      gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1',
530          'arm_float_abi=hard']
531      if force_arm_gcc:
532        gyp_defines.append('nacl_enable_arm_gcc=1')
533      if options.no_arm_trusted:
534        gyp_defines.append('disable_cross_trusted=1')
535  if getos.GetPlatform() == 'mac':
536    gyp_defines.append('clang=1')
537
538  gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
539  for key in ['GYP_GENERATORS', 'GYP_DEFINES', 'CC']:
540    value = gyp_env.get(key)
541    if value is not None:
542      print '%s="%s"' % (key, value)
543  gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)]
544  gyp_depth = '--depth=.'
545  buildbot_common.Run(
546      [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \
547          gyp_generator_flags,
548      cwd=SRC_DIR,
549      env=gyp_env)
550  NinjaBuild(targets, out_dir)
551
552
553def NinjaBuild(targets, out_dir):
554  if type(targets) is not list:
555    targets = [targets]
556  out_config_dir = os.path.join(out_dir, 'Release')
557  buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
558
559
560def BuildStepBuildToolchains(pepperdir, toolchains):
561  buildbot_common.BuildStep('SDK Items')
562
563  GypNinjaBuild_NaCl(GYPBUILD_DIR)
564  GypNinjaBuild_Breakpad(GYPBUILD_DIR)
565
566  platform = getos.GetPlatform()
567  newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
568  glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
569  armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
570  pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
571  bionicdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_bionic')
572
573  if set(toolchains) & set(['glibc', 'newlib']):
574    GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR)
575    GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR)
576
577  if 'arm' in toolchains:
578    GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
579
580  GypNinjaInstall(pepperdir, toolchains)
581
582  if 'newlib' in toolchains:
583    InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
584                       'newlib')
585
586  if 'glibc' in toolchains:
587    InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
588                       'glibc')
589
590  if 'arm' in toolchains:
591    InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
592                       'arm')
593
594  if 'bionic' in toolchains:
595    InstallNaClHeaders(GetToolchainNaClInclude('bionic', bionicdir, 'arm'),
596                       'bionic')
597
598  if 'pnacl' in toolchains:
599    # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
600    for arch in ('ia32', 'arm'):
601      # Fill in the latest native pnacl shim library from the chrome build.
602      build_dir = GYPBUILD_DIR + '-pnacl-' + arch
603      GypNinjaBuild_Pnacl(build_dir, arch)
604      if arch == 'ia32':
605        nacl_arches = ['x86-32', 'x86-64']
606      elif arch == 'arm':
607        nacl_arches = ['arm']
608      else:
609        buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
610      for nacl_arch in nacl_arches:
611        release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
612                                         'gen', 'tc_pnacl_translate',
613                                         'lib-' + nacl_arch)
614
615        pnacl_translator_lib_dir = GetPNaClTranslatorLib(pnacldir, nacl_arch)
616        if not os.path.isdir(pnacl_translator_lib_dir):
617          buildbot_common.ErrorExit('Expected %s directory to exist.' %
618                                    pnacl_translator_lib_dir)
619
620        buildbot_common.CopyFile(
621            os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
622            pnacl_translator_lib_dir)
623
624    InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
625                       'newlib')
626
627
628def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
629  dirpath = os.path.join(pepperdir, dirname)
630  if clobber:
631    buildbot_common.RemoveDir(dirpath)
632  buildbot_common.MakeDir(dirpath)
633
634  return dirpath
635
636
637def BuildStepUpdateHelpers(pepperdir, clobber):
638  buildbot_common.BuildStep('Update project helpers')
639  build_projects.UpdateHelpers(pepperdir, clobber=clobber)
640
641
642def BuildStepUpdateUserProjects(pepperdir, toolchains,
643                                build_experimental, clobber):
644  buildbot_common.BuildStep('Update examples and libraries')
645
646  filters = {}
647  if not build_experimental:
648    filters['EXPERIMENTAL'] = False
649  if toolchains:
650    toolchains = toolchains[:]
651
652    # arm isn't a valid toolchain for build_projects
653    if 'arm' in toolchains:
654      toolchains.remove('arm')
655
656    if 'host' in toolchains:
657      toolchains.remove('host')
658      toolchains.append(getos.GetPlatform())
659
660    filters['TOOLS'] = toolchains
661
662  # Update examples and libraries
663  filters['DEST'] = [
664    'getting_started',
665    'examples/api',
666    'examples/demo',
667    'examples/tutorial',
668    'src'
669  ]
670
671  tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
672  build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
673                                toolchains=toolchains)
674
675
676def BuildStepMakeAll(pepperdir, directory, step_name,
677                     deps=True, clean=False, config='Debug', args=None):
678  buildbot_common.BuildStep(step_name)
679  build_projects.BuildProjectsBranch(pepperdir, directory, clean,
680                                     deps, config, args)
681
682
683def BuildStepBuildLibraries(pepperdir, directory):
684  BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
685      clean=True, config='Debug')
686  BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
687      clean=True, config='Release')
688
689  # Cleanup .pyc file generated while building libraries.  Without
690  # this we would end up shipping the pyc in the SDK tarball.
691  buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc'))
692
693
694def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
695  # Look for LICENSE files
696  license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
697
698  license_files = []
699  for root, _, files in os.walk(fileroot):
700    for filename in files:
701      if license_filenames_re.match(filename):
702        path = os.path.join(root, filename)
703        license_files.append(path)
704
705  if extra_files:
706    license_files += [os.path.join(fileroot, f) for f in extra_files]
707  print '\n'.join(license_files)
708
709  if not os.path.isabs(output_filename):
710    output_filename = os.path.join(fileroot, output_filename)
711  generate_notice.Generate(output_filename, fileroot, license_files)
712
713
714def BuildStepVerifyFilelist(pepperdir):
715  buildbot_common.BuildStep('Verify SDK Files')
716  file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
717  try:
718    verify_filelist.Verify(file_list_path, pepperdir)
719    print 'OK'
720  except verify_filelist.ParseException, e:
721    buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
722  except verify_filelist.VerifyException, e:
723    file_list_rel = os.path.relpath(file_list_path)
724    verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
725    verify_filelist_py = os.path.relpath(verify_filelist_py)
726    pepperdir_rel = os.path.relpath(pepperdir)
727
728    msg = """\
729SDK verification failed:
730
731%s
732Add/remove files from %s to fix.
733
734Run:
735    ./%s %s %s
736to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
737               pepperdir_rel)
738    buildbot_common.ErrorExit(msg)
739
740
741def BuildStepTarBundle(pepper_ver, tarfile):
742  buildbot_common.BuildStep('Tar Pepper Bundle')
743  buildbot_common.MakeDir(os.path.dirname(tarfile))
744  buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
745       'pepper_' + pepper_ver], cwd=NACL_DIR)
746
747
748def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
749                      archive_url):
750  with open(tarfile, 'rb') as tarfile_stream:
751    archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
752        tarfile_stream)
753
754  archive = manifest_util.Archive(manifest_util.GetHostOS())
755  archive.url = archive_url
756  archive.size = archive_size
757  archive.checksum = archive_sha1
758
759  bundle = manifest_util.Bundle('pepper_' + pepper_ver)
760  bundle.revision = int(chrome_revision)
761  bundle.repath = 'pepper_' + pepper_ver
762  bundle.version = int(pepper_ver)
763  bundle.description = (
764      'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
765            pepper_ver, chrome_revision, nacl_revision))
766  bundle.stability = 'dev'
767  bundle.recommended = 'no'
768  bundle.archives = [archive]
769  return bundle
770
771
772def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
773                           tarfile):
774  buildbot_common.BuildStep('Archive %s' % name)
775  bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
776      build_version.ChromeVersion(),)
777  tarname = os.path.basename(tarfile)
778  tarfile_dir = os.path.dirname(tarfile)
779  buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
780
781  # generate "manifest snippet" for this archive.
782  archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
783      build_version.ChromeVersion(), tarname)
784  bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
785                             tarfile, archive_url)
786
787  manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
788  with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
789    manifest_snippet_stream.write(bundle.GetDataAsString())
790
791  buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
792                          step_link=False)
793
794
795def BuildStepArchiveSDKTools():
796  # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
797  builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
798  if builder_name == 'linux-sdk-multi':
799    buildbot_common.BuildStep('Build SDK Tools')
800    build_updater.BuildUpdater(OUT_DIR)
801
802    buildbot_common.BuildStep('Archive SDK Tools')
803    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
804        build_version.ChromeVersion(),)
805    buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
806                            step_link=False)
807    buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
808                            step_link=False)
809
810
811def BuildStepSyncNaClPorts():
812  """Pull the pinned revision of naclports from SVN."""
813  buildbot_common.BuildStep('Sync naclports')
814
815  # In case a previous svn checkout exists, remove it.
816  # TODO(sbc): remove this once all the build machines
817  # have removed the old checkout
818  if (os.path.exists(NACLPORTS_DIR) and
819      not os.path.exists(os.path.join(NACLPORTS_DIR, '.git'))):
820    buildbot_common.RemoveDir(NACLPORTS_DIR)
821
822  if not os.path.exists(NACLPORTS_DIR):
823    # checkout new copy of naclports
824    cmd = ['git', 'clone', NACLPORTS_URL, 'naclports']
825    buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
826  else:
827    # checkout new copy of naclports
828    buildbot_common.Run(['git', 'fetch'], cwd=NACLPORTS_DIR)
829
830  # sync to required revision
831  cmd = ['git', 'checkout', str(NACLPORTS_REV)]
832  buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
833
834
835def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
836  """Build selected naclports in all configurations."""
837  # TODO(sbc): currently naclports doesn't know anything about
838  # Debug builds so the Debug subfolders are all empty.
839
840  env = dict(os.environ)
841  env['NACL_SDK_ROOT'] = pepperdir
842  env['PEPPER_DIR'] = os.path.basename(pepperdir)  # pepper_NN
843  env['NACLPORTS_NO_ANNOTATE'] = "1"
844  env['NACLPORTS_NO_UPLOAD'] = "1"
845  env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV)
846
847  build_script = 'build_tools/buildbot_sdk_bundle.sh'
848  buildbot_common.BuildStep('Build naclports')
849
850  bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
851  out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
852
853  # Remove the sdk_bundle directory to remove stale files from previous builds.
854  buildbot_common.RemoveDir(bundle_dir)
855
856  buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
857
858  # Some naclports do not include a standalone LICENSE/COPYING file
859  # so we explicitly list those here for inclusion.
860  extra_licenses = ('tinyxml/readme.txt',
861                    'jpeg-8d/README',
862                    'zlib-1.2.3/README')
863  src_root = os.path.join(NACLPORTS_DIR, 'out', 'build')
864  output_license = os.path.join(out_dir, 'ports', 'LICENSE')
865  GenerateNotice(src_root , output_license, extra_licenses)
866  readme = os.path.join(out_dir, 'ports', 'README')
867  oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
868
869
870def BuildStepTarNaClPorts(pepper_ver, tarfile):
871  """Create tar archive containing headers and libs from naclports build."""
872  buildbot_common.BuildStep('Tar naclports Bundle')
873  buildbot_common.MakeDir(os.path.dirname(tarfile))
874  pepper_dir = 'pepper_%s' % pepper_ver
875  archive_dirs = [os.path.join(pepper_dir, 'ports')]
876
877  ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
878  cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
879  cmd += archive_dirs
880  buildbot_common.Run(cmd, cwd=NACL_DIR)
881
882
883def BuildStepBuildAppEngine(pepperdir, chrome_revision):
884  """Build the projects found in src/gonacl_appengine/src"""
885  buildbot_common.BuildStep('Build GoNaCl AppEngine Projects')
886  cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision]
887  env = dict(os.environ)
888  env['NACL_SDK_ROOT'] = pepperdir
889  env['NACLPORTS_NO_ANNOTATE'] = "1"
890  buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR)
891
892
893def main(args):
894  parser = optparse.OptionParser(description=__doc__)
895  parser.add_option('--nacl-tree-path',
896      help='Path to native client tree for bionic build.',
897      dest='nacl_tree_path')
898  parser.add_option('--qemu', help='Add qemu for ARM.',
899      action='store_true')
900  parser.add_option('--bionic', help='Add bionic build.',
901      action='store_true')
902  parser.add_option('--tar', help='Force the tar step.',
903      action='store_true')
904  parser.add_option('--archive', help='Force the archive step.',
905      action='store_true')
906  parser.add_option('--release', help='PPAPI release version.',
907      dest='release', default=None)
908  parser.add_option('--build-ports',
909      help='Build naclport bundle.', action='store_true')
910  parser.add_option('--build-app-engine',
911      help='Build AppEngine demos.', action='store_true')
912  parser.add_option('--experimental',
913      help='build experimental examples and libraries', action='store_true',
914      dest='build_experimental')
915  parser.add_option('--skip-toolchain', help='Skip toolchain untar',
916      action='store_true')
917  parser.add_option('--mac-sdk',
918      help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
919  parser.add_option('--no-arm-trusted', action='store_true',
920      help='Disable building of ARM trusted components (sel_ldr, etc).')
921
922  # To setup bash completion for this command first install optcomplete
923  # and then add this line to your .bashrc:
924  #  complete -F _optcomplete build_sdk.py
925  try:
926    import optcomplete
927    optcomplete.autocomplete(parser)
928  except ImportError:
929    pass
930
931  global options
932  options, args = parser.parse_args(args[1:])
933  if args:
934    parser.error("Unexpected arguments: %s" % str(args))
935
936  if options.nacl_tree_path:
937    options.bionic = True
938    toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build')
939    print 'WARNING: Building bionic toolchain from NaCl checkout.'
940    print 'This option builds bionic from the sources currently in the'
941    print 'provided NativeClient checkout, and the results instead of '
942    print 'downloading a toolchain from the builder. This may result in a'
943    print 'NaCl SDK that can not run on ToT chrome.'
944    print 'NOTE:  To clobber you will need to run toolchain_build_bionic.py'
945    print 'directly from the NativeClient checkout.'
946    print ''
947    response = raw_input("Type 'y' and hit enter to continue.\n")
948    if response != 'y' and response != 'Y':
949      print 'Aborting.'
950      return 1
951
952    # Get head version of NativeClient tree
953    buildbot_common.BuildStep('Build bionic toolchain.')
954    buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'],
955                        cwd=toolchain_build)
956  else:
957    toolchain_build = None
958
959  if buildbot_common.IsSDKBuilder():
960    options.archive = True
961    options.build_ports = True
962    # TODO(binji): re-enable app_engine build when the linux builder stops
963    # breaking when trying to git clone from github.
964    # See http://crbug.com/412969.
965    options.build_app_engine = False
966    options.tar = True
967
968  # NOTE: order matters here. This will be the order that is specified in the
969  # Makefiles; the first toolchain will be the default.
970  toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'host']
971
972  # Changes for experimental bionic builder
973  if options.bionic:
974    toolchains.append('bionic')
975    options.build_ports = False
976    options.build_app_engine = False
977
978  print 'Building: ' + ' '.join(toolchains)
979
980  if options.archive and not options.tar:
981    parser.error('Incompatible arguments with archive.')
982
983  chrome_version = int(build_version.ChromeMajorVersion())
984  chrome_revision = build_version.ChromeRevision()
985  nacl_revision = build_version.NaClRevision()
986  pepper_ver = str(chrome_version)
987  pepper_old = str(chrome_version - 1)
988  pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
989  pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
990  if options.bionic:
991    tarname = 'naclsdk_bionic.tar.bz2'
992  else:
993    tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2'
994  tarfile = os.path.join(OUT_DIR, tarname)
995
996  if options.release:
997    pepper_ver = options.release
998  print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
999
1000  if 'NACL_SDK_ROOT' in os.environ:
1001    # We don't want the currently configured NACL_SDK_ROOT to have any effect
1002    # of the build.
1003    del os.environ['NACL_SDK_ROOT']
1004
1005  if not options.skip_toolchain:
1006    BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
1007    BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
1008    BuildStepDownloadToolchains(toolchains)
1009    if options.nacl_tree_path:
1010      # Instead of untarring, copy the raw bionic toolchain
1011      not_bionic = [i for i in toolchains if i != 'bionic']
1012      BuildStepUntarToolchains(pepperdir, not_bionic)
1013      tcname = GetToolchainDirName('bionic', 'arm')
1014      srcdir = os.path.join(toolchain_build, 'out', tcname)
1015      bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
1016      oshelpers.Copy(['-r', srcdir, bionicdir])
1017    else:
1018      BuildStepUntarToolchains(pepperdir, toolchains)
1019
1020    BuildStepBuildToolchains(pepperdir, toolchains)
1021
1022  BuildStepUpdateHelpers(pepperdir, True)
1023  BuildStepUpdateUserProjects(pepperdir, toolchains,
1024                              options.build_experimental, True)
1025
1026  BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
1027
1028  # Ship with libraries prebuilt, so run that first.
1029  BuildStepBuildLibraries(pepperdir, 'src')
1030  GenerateNotice(pepperdir)
1031
1032  # Verify the SDK contains what we expect.
1033  if not options.bionic:
1034    BuildStepVerifyFilelist(pepperdir)
1035
1036  if options.tar:
1037    BuildStepTarBundle(pepper_ver, tarfile)
1038
1039  if options.build_ports and getos.GetPlatform() == 'linux':
1040    ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
1041    BuildStepSyncNaClPorts()
1042    BuildStepBuildNaClPorts(pepper_ver, pepperdir)
1043    if options.tar:
1044      BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
1045
1046  if options.build_app_engine and getos.GetPlatform() == 'linux':
1047    BuildStepBuildAppEngine(pepperdir, chrome_revision)
1048
1049  if options.qemu:
1050    qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
1051    oshelpers.Copy(['-r', qemudir, pepperdir])
1052
1053  # Archive on non-trybots.
1054  if options.archive:
1055    BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
1056                           tarfile)
1057    if options.build_ports and getos.GetPlatform() == 'linux':
1058      BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
1059                             nacl_revision, ports_tarfile)
1060    BuildStepArchiveSDKTools()
1061
1062  return 0
1063
1064
1065if __name__ == '__main__':
1066  try:
1067    sys.exit(main(sys.argv))
1068  except KeyboardInterrupt:
1069    buildbot_common.ErrorExit('build_sdk: interrupted')
1070