1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import multiprocessing
7import optparse
8import os
9import posixpath
10import sys
11import urllib2
12
13import buildbot_common
14import build_version
15import generate_make
16import parse_dsc
17
18from build_paths import SDK_SRC_DIR, OUT_DIR, SDK_RESOURCE_DIR
19from build_paths import GSTORE
20from generate_index import LandingPage
21
22sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
23import getos
24
25
26MAKE = 'nacl_sdk/make_3.99.90-26-gf80222c/make.exe'
27LIB_DICT = {
28  'linux': [],
29  'mac': [],
30  'win': ['x86_32']
31}
32VALID_TOOLCHAINS = [
33  'bionic',
34  'newlib',
35  'glibc',
36  'pnacl',
37  'win',
38  'linux',
39  'mac',
40]
41
42# Global verbosity setting.
43# If set to True (normally via a command line arg) then build_projects will
44# add V=1 to all calls to 'make'
45verbose = False
46
47
48def Trace(msg):
49  if verbose:
50    sys.stderr.write(str(msg) + '\n')
51
52
53def CopyFilesFromTo(filelist, srcdir, dstdir):
54  for filename in filelist:
55    srcpath = os.path.join(srcdir, filename)
56    dstpath = os.path.join(dstdir, filename)
57    buildbot_common.CopyFile(srcpath, dstpath)
58
59
60def UpdateHelpers(pepperdir, clobber=False):
61  tools_dir = os.path.join(pepperdir, 'tools')
62  if not os.path.exists(tools_dir):
63    buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir)
64
65  exampledir = os.path.join(pepperdir, 'examples')
66  if clobber:
67    buildbot_common.RemoveDir(exampledir)
68  buildbot_common.MakeDir(exampledir)
69
70  # Copy files for individual build and landing page
71  files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js',
72      'button_close.png', 'button_close_hover.png']
73  CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir)
74
75  # Copy tools scripts and make includes
76  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
77      tools_dir)
78  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
79      tools_dir)
80
81  # Copy tools/lib scripts
82  tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib')
83  buildbot_common.MakeDir(tools_lib_dir)
84  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'),
85      tools_lib_dir)
86
87  # On Windows add a prebuilt make
88  if getos.GetPlatform() == 'win':
89    buildbot_common.BuildStep('Add MAKE')
90    make_url = posixpath.join(GSTORE, MAKE)
91    make_exe = os.path.join(tools_dir, 'make.exe')
92    with open(make_exe, 'wb') as f:
93      f.write(urllib2.urlopen(make_url).read())
94
95
96def ValidateToolchains(toolchains):
97  invalid_toolchains = set(toolchains) - set(VALID_TOOLCHAINS)
98  if invalid_toolchains:
99    buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
100        ', '.join(invalid_toolchains)))
101
102def GetDeps(projects):
103  out = {}
104
105  # Build list of all project names
106  localtargets = [proj['NAME'] for proj in projects]
107
108  # For each project
109  for proj in projects:
110    deplist = []
111    # generate a list of dependencies
112    for targ in proj.get('TARGETS', []):
113      deplist.extend(targ.get('DEPS', []) + targ.get('LIBS', []))
114
115    # and add dependencies to targets built in this subtree
116    localdeps = [dep for dep in deplist if dep in localtargets]
117    if localdeps:
118      out[proj['NAME']] = localdeps
119
120  return out
121
122
123def UpdateProjects(pepperdir, project_tree, toolchains,
124                   clobber=False, configs=None, first_toolchain=False):
125  if configs is None:
126    configs = ['Debug', 'Release']
127  if not os.path.exists(os.path.join(pepperdir, 'tools')):
128    buildbot_common.ErrorExit('Examples depend on missing tools.')
129  if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
130    buildbot_common.ErrorExit('Examples depend on missing toolchains.')
131
132  ValidateToolchains(toolchains)
133
134  # Create the library output directories
135  libdir = os.path.join(pepperdir, 'lib')
136  platform = getos.GetPlatform()
137  for config in configs:
138    for arch in LIB_DICT[platform]:
139      dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config)
140      if clobber:
141        buildbot_common.RemoveDir(dirpath)
142      buildbot_common.MakeDir(dirpath)
143
144  landing_page = None
145  for branch, projects in project_tree.iteritems():
146    dirpath = os.path.join(pepperdir, branch)
147    if clobber:
148      buildbot_common.RemoveDir(dirpath)
149    buildbot_common.MakeDir(dirpath)
150    targets = [desc['NAME'] for desc in projects]
151    deps = GetDeps(projects)
152
153    # Generate master make for this branch of projects
154    generate_make.GenerateMasterMakefile(pepperdir,
155                                         os.path.join(pepperdir, branch),
156                                         targets, deps)
157
158    if branch.startswith('examples') and not landing_page:
159      landing_page = LandingPage()
160
161    # Generate individual projects
162    for desc in projects:
163      srcroot = os.path.dirname(desc['FILEPATH'])
164      generate_make.ProcessProject(pepperdir, srcroot, pepperdir, desc,
165                                   toolchains, configs=configs,
166                                   first_toolchain=first_toolchain)
167
168      if branch.startswith('examples'):
169        landing_page.AddDesc(desc)
170
171  if landing_page:
172    # Generate the landing page text file.
173    index_html = os.path.join(pepperdir, 'examples', 'index.html')
174    index_template = os.path.join(SDK_RESOURCE_DIR, 'index.html.template')
175    with open(index_html, 'w') as fh:
176      out = landing_page.GeneratePage(index_template)
177      fh.write(out)
178
179  # Generate top Make for examples
180  targets = ['api', 'demo', 'getting_started', 'tutorial']
181  targets = [x for x in targets if 'examples/'+x in project_tree]
182  branch_name = 'examples'
183  generate_make.GenerateMasterMakefile(pepperdir,
184                                       os.path.join(pepperdir, branch_name),
185                                       targets, {})
186
187
188def BuildProjectsBranch(pepperdir, branch, deps, clean, config, args=None):
189  make_dir = os.path.join(pepperdir, branch)
190  print "\nMake: " + make_dir
191
192  if getos.GetPlatform() == 'win':
193    # We need to modify the environment to build host on Windows.
194    make = os.path.join(make_dir, 'make.bat')
195  else:
196    make = 'make'
197
198  env = None
199  if os.environ.get('USE_GOMA') == '1':
200    env = dict(os.environ)
201    env['NACL_COMPILER_PREFIX'] = 'gomacc'
202    # Add -m32 to the CFLAGS when building using i686-nacl-gcc
203    # otherwise goma won't recognise it as different to the x86_64
204    # build.
205    env['X86_32_CFLAGS'] = '-m32'
206    env['X86_32_CXXFLAGS'] = '-m32'
207    jobs = '50'
208  else:
209    jobs = str(multiprocessing.cpu_count())
210
211  make_cmd = [make, '-j', jobs]
212
213  make_cmd.append('CONFIG='+config)
214  # We always ENABLE_BIONIC in case we need it.  If neither --bionic nor
215  # -t bionic have been provided on the command line, then VALID_TOOLCHAINS
216  # will not contain a bionic target.
217  make_cmd.append('ENABLE_BIONIC=1')
218  if not deps:
219    make_cmd.append('IGNORE_DEPS=1')
220
221  if verbose:
222    make_cmd.append('V=1')
223
224  if args:
225    make_cmd += args
226  else:
227    make_cmd.append('TOOLCHAIN=all')
228
229  buildbot_common.Run(make_cmd, cwd=make_dir, env=env)
230  if clean:
231    # Clean to remove temporary files but keep the built
232    buildbot_common.Run(make_cmd + ['clean'], cwd=make_dir, env=env)
233
234
235def BuildProjects(pepperdir, project_tree, deps=True,
236                  clean=False, config='Debug'):
237  # Make sure we build libraries (which live in 'src') before
238  # any of the examples.
239  build_first = [p for p in project_tree if p != 'src']
240  build_second = [p for p in project_tree if p == 'src']
241
242  for branch in build_first + build_second:
243    BuildProjectsBranch(pepperdir, branch, deps, clean, config)
244
245
246def main(argv):
247  parser = optparse.OptionParser()
248  parser.add_option('-c', '--clobber',
249      help='Clobber project directories before copying new files',
250      action='store_true', default=False)
251  parser.add_option('-b', '--build',
252      help='Build the projects. Otherwise the projects are only copied.',
253      action='store_true')
254  parser.add_option('--config',
255      help='Choose configuration to build (Debug or Release).  Builds both '
256           'by default')
257  parser.add_option('--bionic',
258      help='Enable bionic projects', action='store_true')
259  parser.add_option('-x', '--experimental',
260      help='Build experimental projects', action='store_true')
261  parser.add_option('-t', '--toolchain',
262      help='Build using toolchain. Can be passed more than once.',
263      action='append', default=[])
264  parser.add_option('-d', '--dest',
265      help='Select which build destinations (project types) are valid.',
266      action='append')
267  parser.add_option('-v', '--verbose', action='store_true')
268
269  # To setup bash completion for this command first install optcomplete
270  # and then add this line to your .bashrc:
271  #  complete -F _optcomplete build_projects.py
272  try:
273    import optcomplete
274    optcomplete.autocomplete(parser)
275  except ImportError:
276    pass
277
278  options, args = parser.parse_args(argv[1:])
279
280  global verbose
281  if options.verbose:
282    verbose = True
283
284  buildbot_common.verbose = verbose
285
286  if 'NACL_SDK_ROOT' in os.environ:
287    # We don't want the currently configured NACL_SDK_ROOT to have any effect
288    # on the build.
289    del os.environ['NACL_SDK_ROOT']
290
291  pepper_ver = str(int(build_version.ChromeMajorVersion()))
292  pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
293
294  if not options.toolchain:
295    # Order matters here: the default toolchain for an example's Makefile will
296    # be the first toolchain in this list that is available in the example.
297    # e.g. If an example supports newlib and glibc, then the default will be
298    # newlib.
299    options.toolchain = ['pnacl', 'newlib', 'glibc', 'host']
300    if options.experimental or options.bionic:
301      options.toolchain.append('bionic')
302
303  if 'host' in options.toolchain:
304    options.toolchain.remove('host')
305    options.toolchain.append(getos.GetPlatform())
306    Trace('Adding platform: ' + getos.GetPlatform())
307
308  ValidateToolchains(options.toolchain)
309
310  filters = {}
311  if options.toolchain:
312    filters['TOOLS'] = options.toolchain
313    Trace('Filter by toolchain: ' + str(options.toolchain))
314  if not options.experimental:
315    filters['EXPERIMENTAL'] = False
316  if options.dest:
317    filters['DEST'] = options.dest
318    Trace('Filter by type: ' + str(options.dest))
319  if args:
320    filters['NAME'] = args
321    Trace('Filter by name: ' + str(args))
322
323  try:
324    project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
325  except parse_dsc.ValidationError as e:
326    buildbot_common.ErrorExit(str(e))
327
328  if verbose:
329    parse_dsc.PrintProjectTree(project_tree)
330
331  UpdateHelpers(pepperdir, clobber=options.clobber)
332  UpdateProjects(pepperdir, project_tree, options.toolchain,
333                 clobber=options.clobber)
334
335  if options.build:
336    if options.config:
337      configs = [options.config]
338    else:
339      configs = ['Debug', 'Release']
340    for config in configs:
341      BuildProjects(pepperdir, project_tree, config=config, deps=False)
342
343  return 0
344
345
346if __name__ == '__main__':
347  script_name = os.path.basename(sys.argv[0])
348  try:
349    sys.exit(main(sys.argv))
350  except parse_dsc.ValidationError as e:
351    buildbot_common.ErrorExit('%s: %s' % (script_name, e))
352  except KeyboardInterrupt:
353    buildbot_common.ErrorExit('%s: interrupted' % script_name)
354