1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Entry point for both build and try bots.
7
8This script is invoked from XXX, usually without arguments
9to package an SDK. It automatically determines whether
10this SDK is for mac, win, linux.
11
12The script inspects the following environment variables:
13
14BUILDBOT_BUILDERNAME to determine whether the script is run locally
15and whether it should upload an SDK to file storage (GSTORE)
16"""
17
18# pylint: disable=W0621
19
20# std python includes
21import copy
22import datetime
23import glob
24import optparse
25import os
26import re
27import sys
28
29if sys.version_info < (2, 6, 0):
30  sys.stderr.write("python 2.6 or later is required run this script\n")
31  sys.exit(1)
32
33# local includes
34import buildbot_common
35import build_projects
36import build_updater
37import build_version
38import generate_make
39import generate_notice
40import manifest_util
41import parse_dsc
42import verify_filelist
43
44from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
45from build_paths import NACLPORTS_DIR, GSTORE
46
47# Add SDK make tools scripts to the python path.
48sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
49sys.path.append(os.path.join(NACL_DIR, 'build'))
50
51import getos
52import oshelpers
53
54CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
55
56NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src'
57NACLPORTS_REV = 850
58
59GYPBUILD_DIR = 'gypbuild'
60
61options = None
62
63
64def GetGlibcToolchain(host_arch):
65  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
66  tcname = 'toolchain_%s_%s.tar.bz2' % (getos.GetPlatform(), host_arch)
67  return os.path.join(tcdir, tcname)
68
69
70def GetNewlibToolchain(host_arch):
71  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
72  tcname = 'naclsdk_%s_%s.tgz' % (getos.GetPlatform(), host_arch)
73  return os.path.join(tcdir, tcname)
74
75
76def GetPNaClToolchain(host_arch):
77  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
78  tcname = 'naclsdk_pnacl_%s_%s.tgz' % (getos.GetPlatform(), host_arch)
79  return os.path.join(tcdir, tcname)
80
81
82def GetToolchainNaClInclude(tcname, tcpath, arch):
83  if arch == 'x86':
84    if tcname == 'pnacl':
85      return os.path.join(tcpath, 'newlib', 'sdk', 'include')
86    return os.path.join(tcpath, 'x86_64-nacl', 'include')
87  elif arch == 'arm':
88    return os.path.join(tcpath, 'arm-nacl', 'include')
89  else:
90    buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
91
92
93def GetGypGenDir(xarch):
94  if xarch == 'arm':
95    build_dir = GYPBUILD_DIR + '-arm'
96  else:
97    build_dir = GYPBUILD_DIR
98  return os.path.join(OUT_DIR, build_dir, 'Release', 'gen')
99
100
101def GetGypBuiltLib(tcname, xarch=None):
102  if tcname == 'pnacl':
103    tcname = 'pnacl_newlib'
104  if not xarch:
105    xarch = ''
106  return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch)
107
108
109def GetToolchainNaClLib(tcname, tcpath, xarch):
110  if tcname == 'pnacl':
111    return os.path.join(tcpath, 'newlib', 'sdk', 'lib')
112  if xarch == '32':
113    return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
114  if xarch == '64':
115    return os.path.join(tcpath, 'x86_64-nacl', 'lib')
116  if xarch == 'arm':
117    return os.path.join(tcpath, 'arm-nacl', 'lib')
118
119
120def GetToolchainDirName(tcname, xarch):
121  if xarch != 'arm':
122    xarch = 'x86'
123  return '%s_%s_%s' % (getos.GetPlatform(), xarch, tcname)
124
125
126def GetGypToolchainLib(tcname, xarch):
127  tcpath = os.path.join(GetGypGenDir(xarch), 'sdk', 'toolchain',
128                        GetToolchainDirName(tcname, xarch))
129  return GetToolchainNaClLib(tcname, tcpath, xarch)
130
131
132def GetOutputToolchainLib(pepperdir, tcname, xarch):
133  tcpath = os.path.join(pepperdir, 'toolchain',
134                        GetToolchainDirName(tcname, xarch))
135  return GetToolchainNaClLib(tcname, tcpath, xarch)
136
137
138def GetPNaClNativeLib(tcpath, arch):
139  if arch not in ['arm', 'x86-32', 'x86-64']:
140    buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
141  return os.path.join(tcpath, 'lib-' + arch)
142
143
144def BuildStepDownloadToolchains():
145  buildbot_common.BuildStep('Running download_toolchains.py')
146  download_script = os.path.join('build', 'download_toolchains.py')
147  buildbot_common.Run([sys.executable, download_script,
148                      '--no-arm-trusted', '--arm-untrusted', '--keep'],
149                      cwd=NACL_DIR)
150
151
152def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
153  buildbot_common.BuildStep('Clean Pepper Dirs')
154  buildbot_common.RemoveDir(pepperdir_old)
155  buildbot_common.RemoveDir(pepperdir)
156  buildbot_common.MakeDir(pepperdir)
157
158
159def BuildStepMakePepperDirs(pepperdir, subdirs):
160  for subdir in subdirs:
161    buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
162
163TEXT_FILES = [
164  'AUTHORS',
165  'COPYING',
166  'LICENSE',
167  'README.Makefiles',
168]
169
170def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
171                           nacl_revision):
172  buildbot_common.BuildStep('Add Text Files')
173  InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
174
175  # Replace a few placeholders in README
176  readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
177  readme_text = readme_text.replace('${VERSION}', pepper_ver)
178  readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
179  readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
180
181  # Year/Month/Day Hour:Minute:Second
182  time_format = '%Y/%m/%d %H:%M:%S'
183  readme_text = readme_text.replace('${DATE}',
184      datetime.datetime.now().strftime(time_format))
185
186  open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
187
188
189def PrunePNaClToolchain(root):
190  dirs_to_prune = [
191    'newlib/lib-bc-x86-64',
192    'newlib/usr-bc-x86-64'
193    # TODO(sbc): remove this once its really not needed.
194    # Currently we seem to rely on it at least for <bits/stat.h>
195    #'newlib/sysroot',
196  ]
197  for dirname in dirs_to_prune:
198    buildbot_common.RemoveDir(os.path.join(root, dirname))
199
200
201def BuildStepUntarToolchains(pepperdir, arch, toolchains):
202  buildbot_common.BuildStep('Untar Toolchains')
203  platform = getos.GetPlatform()
204  tcname = platform + '_' + arch
205  tmpdir = os.path.join(OUT_DIR, 'tc_temp')
206  buildbot_common.RemoveDir(tmpdir)
207  buildbot_common.MakeDir(tmpdir)
208
209  if 'newlib' in toolchains:
210    # Untar the newlib toolchains
211    tarfile = GetNewlibToolchain(arch)
212    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
213                        cwd=NACL_DIR)
214
215    # Then rename/move it to the pepper toolchain directory
216    srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
217    newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
218    buildbot_common.Move(srcdir, newlibdir)
219
220  if 'arm' in toolchains:
221    # Copy the existing arm toolchain from native_client tree
222    arm_toolchain = os.path.join(NACL_DIR, 'toolchain',
223                                 platform + '_arm_newlib')
224    arm_toolchain_sdk = os.path.join(pepperdir, 'toolchain',
225                                     os.path.basename(arm_toolchain))
226    buildbot_common.CopyDir(arm_toolchain, arm_toolchain_sdk)
227
228  if 'glibc' in toolchains:
229    # Untar the glibc toolchains
230    tarfile = GetGlibcToolchain(arch)
231    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
232                        cwd=NACL_DIR)
233
234    # Then rename/move it to the pepper toolchain directory
235    srcdir = os.path.join(tmpdir, 'toolchain', tcname)
236    glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
237    buildbot_common.Move(srcdir, glibcdir)
238
239  # Untar the pnacl toolchains
240  if 'pnacl' in toolchains:
241    tmpdir = os.path.join(tmpdir, 'pnacl')
242    buildbot_common.RemoveDir(tmpdir)
243    buildbot_common.MakeDir(tmpdir)
244    tarfile = GetPNaClToolchain(arch)
245    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
246                        cwd=NACL_DIR)
247
248    # Then rename/move it to the pepper toolchain directory
249    pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl')
250    buildbot_common.Move(tmpdir, pnacldir)
251    PrunePNaClToolchain(pnacldir)
252
253  buildbot_common.RemoveDir(tmpdir)
254
255  if options.gyp and platform != 'win':
256    # If the gyp options is specified we install a toolchain
257    # wrapper so that gyp can switch toolchains via a commandline
258    # option.
259    bindir = os.path.join(pepperdir, 'toolchain', tcname, 'bin')
260    wrapper = os.path.join(SDK_SRC_DIR, 'tools', 'compiler-wrapper.py')
261    buildbot_common.MakeDir(bindir)
262    buildbot_common.CopyFile(wrapper, bindir)
263
264    # Module 'os' has no 'symlink' member (on Windows).
265    # pylint: disable=E1101
266
267    os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-g++'))
268    os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-gcc'))
269    os.symlink('compiler-wrapper.py', os.path.join(bindir, 'i686-nacl-ar'))
270
271
272# List of toolchain headers to install.
273# Source is relative to native_client tree, destination is relative
274# to the toolchain header directory.
275NACL_HEADER_MAP = {
276  'newlib': [
277      ('src/include/nacl/nacl_exception.h', 'nacl/'),
278      ('src/include/nacl/nacl_minidump.h', 'nacl/'),
279      ('src/untrusted/irt/irt.h', ''),
280      ('src/untrusted/irt/irt_ppapi.h', ''),
281      ('src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
282      ('src/untrusted/nacl/nacl_startup.h', 'nacl/'),
283      ('src/untrusted/nacl/nacl_thread.h', 'nacl/'),
284      ('src/untrusted/nacl/pnacl.h', ''),
285      ('src/untrusted/pthread/pthread.h', ''),
286      ('src/untrusted/pthread/semaphore.h', ''),
287      ('src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
288  ],
289  'glibc': [
290      ('src/include/nacl/nacl_exception.h', 'nacl/'),
291      ('src/include/nacl/nacl_minidump.h', 'nacl/'),
292      ('src/untrusted/irt/irt.h', ''),
293      ('src/untrusted/irt/irt_ppapi.h', ''),
294      ('src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
295      ('src/untrusted/nacl/nacl_startup.h', 'nacl/'),
296      ('src/untrusted/nacl/nacl_thread.h', 'nacl/'),
297      ('src/untrusted/nacl/pnacl.h', ''),
298      ('src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
299  ],
300  'host': []
301}
302
303def InstallFiles(src_root, dest_root, file_list):
304  """Copy a set of files from src_root to dest_root according
305  to the given mapping.  This allows files to be copied from
306  to a location in the destination tree that is different to the
307  location in the source tree.
308
309  If the destination mapping ends with a '/' then the destination
310  basename is inherited from the the source file.
311
312  Wildcards can be used in the source list but it is not recommended
313  as this can end up adding things to the SDK unintentionally.
314  """
315  for file_spec in file_list:
316    # The list of files to install can be a simple list of
317    # strings or a list of pairs, where each pair corresponds
318    # to a mapping from source to destination names.
319    if type(file_spec) == str:
320      src_file = dest_file = file_spec
321    else:
322      src_file, dest_file = file_spec
323
324    src_file = os.path.join(src_root, src_file)
325
326    # Expand sources files using glob.
327    sources = glob.glob(src_file)
328    if not sources:
329      sources = [src_file]
330
331    if len(sources) > 1 and not dest_file.endswith('/'):
332      buildbot_common.ErrorExit("Target file must end in '/' when "
333                                "using globbing to install multiple files")
334
335    for source in sources:
336      if dest_file.endswith('/'):
337        dest = os.path.join(dest_file, os.path.basename(source))
338      else:
339        dest = dest_file
340      dest = os.path.join(dest_root, dest)
341      if not os.path.isdir(os.path.dirname(dest)):
342        buildbot_common.MakeDir(os.path.dirname(dest))
343      buildbot_common.CopyFile(source, dest)
344
345
346def InstallNaClHeaders(tc_dst_inc, tc_name):
347  """Copies NaCl headers to expected locations in the toolchain."""
348  if tc_name == 'arm':
349    # arm toolchain header should be the same as the x86 newlib
350    # ones
351    tc_name = 'newlib'
352
353  InstallFiles(NACL_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
354
355
356def MakeNinjaRelPath(path):
357  return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
358
359
360TOOLCHAIN_LIBS = {
361  'newlib' : [
362    'crti.o',
363    'crtn.o',
364    'libminidump_generator.a',
365    'libnacl.a',
366    'libnacl_dyncode.a',
367    'libnacl_exception.a',
368    'libnacl_list_mappings.a',
369    'libnosys.a',
370    'libppapi.a',
371    'libppapi_stub.a',
372    'libpthread.a',
373  ],
374  'glibc': [
375    'libminidump_generator.a',
376    'libminidump_generator.so',
377    'libnacl.a',
378    'libnacl_dyncode.a',
379    'libnacl_dyncode.so',
380    'libnacl_exception.a',
381    'libnacl_exception.so',
382    'libnacl_list_mappings.a',
383    'libnacl_list_mappings.so',
384    'libppapi.a',
385    'libppapi.so',
386    'libppapi_stub.a',
387  ],
388  'pnacl': [
389    'libminidump_generator.a',
390    'libnacl.a',
391    'libnacl_dyncode.a',
392    'libnacl_exception.a',
393    'libnacl_list_mappings.a',
394    'libnosys.a',
395    'libppapi.a',
396    'libppapi_stub.a',
397    'libpthread.a',
398  ]
399}
400
401
402def GypNinjaInstall(pepperdir, toolchains):
403  build_dir = GYPBUILD_DIR
404  ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
405  tools_files = [
406    ['sel_ldr', 'sel_ldr_x86_32'],
407    ['ncval_new', 'ncval'],
408    ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
409    ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
410  ]
411
412  platform = getos.GetPlatform()
413
414  # TODO(binji): dump_syms doesn't currently build on Windows. See
415  # http://crbug.com/245456
416  if platform != 'win':
417    tools_files += [
418      ['dump_syms', 'dump_syms'],
419      ['minidump_dump', 'minidump_dump'],
420      ['minidump_stackwalk', 'minidump_stackwalk']
421    ]
422
423  if platform != 'mac':
424    # Mac doesn't build 64-bit binaries.
425    tools_files.append(['sel_ldr64', 'sel_ldr_x86_64'])
426
427  if platform == 'linux':
428    tools_files.append(['nacl_helper_bootstrap',
429                        'nacl_helper_bootstrap_x86_32'])
430    tools_files.append(['nacl_helper_bootstrap64',
431                        'nacl_helper_bootstrap_x86_64'])
432
433  buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
434
435  # Add .exe extensions to all windows tools
436  for pair in tools_files:
437    if platform == 'win' and not pair[0].endswith('.nexe'):
438      pair[0] += '.exe'
439      pair[1] += '.exe'
440
441  InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
442
443  for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
444    if tc == 'pnacl':
445      xarches = (None,)
446    else:
447      xarches = ('arm', '32', '64')
448
449    for xarch in xarches:
450      if tc == 'glibc' and xarch == 'arm':
451        continue
452
453      src_dir = GetGypBuiltLib(tc, xarch)
454      dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
455      InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
456
457      if tc != 'pnacl':
458        src_dir = GetGypToolchainLib(tc, xarch)
459        InstallFiles(src_dir, dst_dir, ['crt1.o'])
460
461
462def GypNinjaBuild_NaCl(rel_out_dir):
463  gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
464  nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
465  all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
466
467  out_dir = MakeNinjaRelPath(rel_out_dir)
468  out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
469  GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir)
470  GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
471  GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir)
472
473  platform = getos.GetPlatform()
474  if platform == 'win':
475    NinjaBuild('sel_ldr64', out_dir)
476  elif platform == 'linux':
477    out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64')
478    GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64)
479
480    # We only need sel_ldr from the 64-bit out directory.
481    # sel_ldr needs to be renamed, so we'll call it sel_ldr64.
482    files_to_copy = [
483      ('sel_ldr', 'sel_ldr64'),
484      ('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'),
485    ]
486
487    for src, dst in files_to_copy:
488      buildbot_common.CopyFile(
489          os.path.join(SRC_DIR, out_dir_64, 'Release', src),
490          os.path.join(SRC_DIR, out_dir, 'Release', dst))
491
492
493def GypNinjaBuild_Breakpad(rel_out_dir):
494  # TODO(binji): dump_syms doesn't currently build on Windows. See
495  # http://crbug.com/245456
496  if getos.GetPlatform() == 'win':
497    return
498
499  gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
500  out_dir = MakeNinjaRelPath(rel_out_dir)
501  gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
502  build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
503  GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir)
504
505
506def GypNinjaBuild_PPAPI(arch, rel_out_dir):
507  gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
508  out_dir = MakeNinjaRelPath(rel_out_dir)
509  gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
510                          'native_client.gyp')
511  GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
512
513
514def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
515  # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
516  # Chromium build, and once here. When we move more of the SDK build process
517  # to gyp, we can remove this.
518  gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
519
520  out_dir = MakeNinjaRelPath(rel_out_dir)
521  gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
522                          'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
523  targets = ['pnacl_irt_shim']
524  GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False)
525
526
527def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
528                  out_dir, force_arm_gcc=True):
529  gyp_env = copy.copy(os.environ)
530  gyp_env['GYP_GENERATORS'] = 'ninja'
531  gyp_defines = []
532  if options.mac_sdk:
533    gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
534  if arch:
535    gyp_defines.append('target_arch=%s' % arch)
536    if arch == 'arm':
537      gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1']
538      if force_arm_gcc:
539        gyp_defines += ['nacl_enable_arm_gcc=1']
540
541  gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
542  for key in ['GYP_GENERATORS', 'GYP_DEFINES']:
543    value = gyp_env[key]
544    print '%s="%s"' % (key, value)
545  gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)]
546  gyp_depth = '--depth=.'
547  buildbot_common.Run(
548      [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \
549          gyp_generator_flags,
550      cwd=SRC_DIR,
551      env=gyp_env)
552  NinjaBuild(targets, out_dir)
553
554
555def NinjaBuild(targets, out_dir):
556  if type(targets) is not list:
557    targets = [targets]
558  out_config_dir = os.path.join(out_dir, 'Release')
559  buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
560
561
562def BuildStepBuildToolchains(pepperdir, toolchains):
563  buildbot_common.BuildStep('SDK Items')
564
565  GypNinjaBuild_NaCl(GYPBUILD_DIR)
566  GypNinjaBuild_Breakpad(GYPBUILD_DIR)
567
568  platform = getos.GetPlatform()
569  tcname = platform + '_x86'
570  newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
571  glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
572  pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl')
573
574  if set(toolchains) & set(['glibc', 'newlib']):
575    GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR)
576
577  if 'arm' in toolchains:
578    GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
579
580  GypNinjaInstall(pepperdir, toolchains)
581
582  if 'newlib' in toolchains:
583    InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
584                       'newlib')
585
586  if 'glibc' in toolchains:
587    InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
588                       'glibc')
589
590  if 'arm' in toolchains:
591    tcname = platform + '_arm_newlib'
592    armdir = os.path.join(pepperdir, 'toolchain', tcname)
593    InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
594                       'arm')
595
596  if 'pnacl' in toolchains:
597    # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
598    for arch in ('ia32', 'arm'):
599      # Fill in the latest native pnacl shim library from the chrome build.
600      build_dir = GYPBUILD_DIR + '-pnacl-' + arch
601      GypNinjaBuild_Pnacl(build_dir, arch)
602      if arch == 'ia32':
603        nacl_arches = ['x86-32', 'x86-64']
604      elif arch == 'arm':
605        nacl_arches = ['arm']
606      else:
607        buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
608      for nacl_arch in nacl_arches:
609        release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
610                                         'gen', 'tc_pnacl_translate',
611                                         'lib-' + nacl_arch)
612
613        buildbot_common.CopyFile(
614            os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
615            GetPNaClNativeLib(pnacldir, nacl_arch))
616
617    InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
618                       'newlib')
619
620
621def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
622  dirpath = os.path.join(pepperdir, dirname)
623  if clobber:
624    buildbot_common.RemoveDir(dirpath)
625  buildbot_common.MakeDir(dirpath)
626
627  return dirpath
628
629
630def BuildStepUpdateHelpers(pepperdir, clobber):
631  buildbot_common.BuildStep('Update project helpers')
632  build_projects.UpdateHelpers(pepperdir, clobber=clobber)
633
634
635def BuildStepUpdateUserProjects(pepperdir, toolchains,
636                                build_experimental, clobber):
637  buildbot_common.BuildStep('Update examples and libraries')
638
639  filters = {}
640  if not build_experimental:
641    filters['EXPERIMENTAL'] = False
642  if toolchains:
643    toolchains = toolchains[:]
644
645    # arm isn't a valid toolchain for build_projects
646    if 'arm' in toolchains:
647      toolchains.remove('arm')
648
649    if 'host' in toolchains:
650      toolchains.remove('host')
651      toolchains.append(getos.GetPlatform())
652
653    filters['TOOLS'] = toolchains
654
655  # Update examples and libraries
656  filters['DEST'] = [
657    'examples/api',
658    'examples/demo',
659    'examples/getting_started',
660    'examples/tutorial',
661    'src'
662  ]
663
664  tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
665  build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
666                                toolchains=toolchains)
667
668
669def BuildStepMakeAll(pepperdir, directory, step_name,
670                     deps=True, clean=False, config='Debug', args=None):
671  buildbot_common.BuildStep(step_name)
672  build_projects.BuildProjectsBranch(pepperdir, directory, clean,
673                                     deps, config, args)
674
675
676def BuildStepBuildLibraries(pepperdir, directory):
677  BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
678      clean=True, config='Debug')
679  BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
680      clean=True, config='Release')
681
682
683def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
684  # Look for LICENSE files
685  license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
686
687  license_files = []
688  for root, _, files in os.walk(fileroot):
689    for filename in files:
690      if license_filenames_re.match(filename):
691        path = os.path.join(root, filename)
692        license_files.append(path)
693
694  if extra_files:
695    license_files += [os.path.join(fileroot, f) for f in extra_files]
696  print '\n'.join(license_files)
697
698  if not os.path.isabs(output_filename):
699    output_filename = os.path.join(fileroot, output_filename)
700  generate_notice.Generate(output_filename, fileroot, license_files)
701
702
703def BuildStepVerifyFilelist(pepperdir):
704  buildbot_common.BuildStep('Verify SDK Files')
705  file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
706  try:
707    verify_filelist.Verify(file_list_path, pepperdir)
708    print 'OK'
709  except verify_filelist.ParseException, e:
710    buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
711  except verify_filelist.VerifyException, e:
712    file_list_rel = os.path.relpath(file_list_path)
713    verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
714    verify_filelist_py = os.path.relpath(verify_filelist_py)
715    pepperdir_rel = os.path.relpath(pepperdir)
716
717    msg = """\
718SDK verification failed:
719
720%s
721Add/remove files from %s to fix.
722
723Run:
724    ./%s %s %s
725to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
726               pepperdir_rel)
727    buildbot_common.ErrorExit(msg)
728
729
730
731def BuildStepTarBundle(pepper_ver, tarfile):
732  buildbot_common.BuildStep('Tar Pepper Bundle')
733  buildbot_common.MakeDir(os.path.dirname(tarfile))
734  buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
735       'pepper_' + pepper_ver], cwd=NACL_DIR)
736
737
738
739def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
740                      archive_url):
741  with open(tarfile, 'rb') as tarfile_stream:
742    archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
743        tarfile_stream)
744
745  archive = manifest_util.Archive(manifest_util.GetHostOS())
746  archive.url = archive_url
747  archive.size = archive_size
748  archive.checksum = archive_sha1
749
750  bundle = manifest_util.Bundle('pepper_' + pepper_ver)
751  bundle.revision = int(chrome_revision)
752  bundle.repath = 'pepper_' + pepper_ver
753  bundle.version = int(pepper_ver)
754  bundle.description = (
755      'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
756            pepper_ver, chrome_revision, nacl_revision))
757  bundle.stability = 'dev'
758  bundle.recommended = 'no'
759  bundle.archives = [archive]
760  return bundle
761
762
763def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
764                           tarfile):
765  buildbot_common.BuildStep('Archive %s' % name)
766  bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
767      build_version.ChromeVersion(),)
768  tarname = os.path.basename(tarfile)
769  tarfile_dir = os.path.dirname(tarfile)
770  buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
771
772  # generate "manifest snippet" for this archive.
773  archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
774      build_version.ChromeVersion(), tarname)
775  bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
776                             tarfile, archive_url)
777
778  manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
779  with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
780    manifest_snippet_stream.write(bundle.GetDataAsString())
781
782  buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
783                          step_link=False)
784
785
786def BuildStepArchiveSDKTools():
787  # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
788  builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
789  if builder_name == 'linux-sdk-multi':
790    buildbot_common.BuildStep('Build SDK Tools')
791    build_updater.BuildUpdater(OUT_DIR)
792
793    buildbot_common.BuildStep('Archive SDK Tools')
794    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
795        build_version.ChromeVersion(),)
796    buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
797                            step_link=False)
798    buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
799                            step_link=False)
800
801
802def BuildStepSyncNaClPorts():
803  """Pull the pinned revision of naclports from SVN."""
804  buildbot_common.BuildStep('Sync naclports')
805  if not os.path.exists(NACLPORTS_DIR):
806    # checkout new copy of naclports
807    cmd = ['svn', 'checkout', '-q', '-r', str(NACLPORTS_REV), NACLPORTS_URL,
808           'naclports']
809    buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
810  else:
811    # sync existing copy to pinned revision.
812    cmd = ['svn', 'update', '-r', str(NACLPORTS_REV)]
813    buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
814
815
816def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
817  """Build selected naclports in all configurations."""
818  # TODO(sbc): currently naclports doesn't know anything about
819  # Debug builds so the Debug subfolders are all empty.
820  bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
821
822  env = dict(os.environ)
823  env['NACL_SDK_ROOT'] = pepperdir
824  env['NACLPORTS_NO_ANNOTATE'] = "1"
825  env['NACLPORTS_NO_UPLOAD'] = "1"
826
827  build_script = 'build_tools/bots/linux/naclports-linux-sdk-bundle.sh'
828  buildbot_common.BuildStep('Build naclports')
829  buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
830
831  out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
832
833  # Some naclports do not include a standalone LICENSE/COPYING file
834  # so we explicitly list those here for inclusion.
835  extra_licenses = ('tinyxml/readme.txt',
836                    'jpeg-8d/README',
837                    'zlib-1.2.3/README')
838  src_root = os.path.join(NACLPORTS_DIR, 'out', 'repository-i686')
839  output_license = os.path.join(out_dir, 'ports', 'LICENSE')
840  GenerateNotice(src_root , output_license, extra_licenses)
841  readme = os.path.join(out_dir, 'ports', 'README')
842  oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
843
844
845def BuildStepTarNaClPorts(pepper_ver, tarfile):
846  """Create tar archive containing headers and libs from naclports build."""
847  buildbot_common.BuildStep('Tar naclports Bundle')
848  buildbot_common.MakeDir(os.path.dirname(tarfile))
849  pepper_dir = 'pepper_%s' % pepper_ver
850  archive_dirs = [os.path.join(pepper_dir, 'ports')]
851
852  ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
853  cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
854  cmd += archive_dirs
855  buildbot_common.Run(cmd, cwd=NACL_DIR)
856
857
858def main(args):
859  parser = optparse.OptionParser()
860  parser.add_option('--skip-tar', help='Skip generating a tarball.',
861      action='store_true')
862  parser.add_option('--archive', help='Force the archive step.',
863      action='store_true')
864  parser.add_option('--gyp',
865      help='Use gyp to build examples/libraries/Makefiles.',
866      action='store_true')
867  parser.add_option('--release', help='PPAPI release version.',
868      dest='release', default=None)
869  parser.add_option('--build-ports',
870      help='Build naclport bundle.', action='store_true')
871  parser.add_option('--experimental',
872      help='build experimental examples and libraries', action='store_true',
873      dest='build_experimental')
874  parser.add_option('--skip-toolchain', help='Skip toolchain untar',
875      action='store_true')
876  parser.add_option('--mac_sdk',
877      help='Set the mac_sdk (e.g. 10.6) to use when building with ninja.',
878      dest='mac_sdk')
879
880  global options
881  options, args = parser.parse_args(args[1:])
882  arch = 'x86'
883
884  generate_make.use_gyp = options.gyp
885  if buildbot_common.IsSDKBuilder():
886    options.archive = True
887    options.build_ports = True
888
889  toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host']
890  print 'Building: ' + ' '.join(toolchains)
891
892  if options.archive and options.skip_tar:
893    parser.error('Incompatible arguments with archive.')
894
895  chrome_version = int(build_version.ChromeMajorVersion())
896  chrome_revision = build_version.ChromeRevision()
897  nacl_revision = build_version.NaClRevision()
898  pepper_ver = str(chrome_version)
899  pepper_old = str(chrome_version - 1)
900  pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
901  pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
902  tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2'
903  tarfile = os.path.join(OUT_DIR, tarname)
904
905  if options.release:
906    pepper_ver = options.release
907  print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
908
909  if 'NACL_SDK_ROOT' in os.environ:
910    # We don't want the currently configured NACL_SDK_ROOT to have any effect
911    # of the build.
912    del os.environ['NACL_SDK_ROOT']
913
914  if not options.skip_toolchain:
915    BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
916    BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
917    BuildStepDownloadToolchains()
918    BuildStepUntarToolchains(pepperdir, arch, toolchains)
919
920  BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
921  BuildStepBuildToolchains(pepperdir, toolchains)
922
923  BuildStepUpdateHelpers(pepperdir, True)
924  BuildStepUpdateUserProjects(pepperdir, toolchains,
925                              options.build_experimental, True)
926
927  # Ship with libraries prebuilt, so run that first.
928  BuildStepBuildLibraries(pepperdir, 'src')
929  GenerateNotice(pepperdir)
930
931  # Verify the SDK contains what we expect.
932  BuildStepVerifyFilelist(pepperdir)
933
934  if not options.skip_tar:
935    BuildStepTarBundle(pepper_ver, tarfile)
936
937  if options.build_ports and getos.GetPlatform() == 'linux':
938    ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
939    BuildStepSyncNaClPorts()
940    BuildStepBuildNaClPorts(pepper_ver, pepperdir)
941    if not options.skip_tar:
942      BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
943
944  # Archive on non-trybots.
945  if options.archive:
946    BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
947                           tarfile)
948    if options.build_ports and getos.GetPlatform() == 'linux':
949      BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
950                             nacl_revision, ports_tarfile)
951    BuildStepArchiveSDKTools()
952
953  return 0
954
955
956if __name__ == '__main__':
957  try:
958    sys.exit(main(sys.argv))
959  except KeyboardInterrupt:
960    buildbot_common.ErrorExit('build_sdk: interrupted')
961