1# Copyright (c) 2015, Google Inc.
2#
3# Permission to use, copy, modify, and/or distribute this software for any
4# purpose with or without fee is hereby granted, provided that the above
5# copyright notice and this permission notice appear in all copies.
6#
7# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15"""Enumerates the BoringSSL source in src/ and either generates two gypi files
16  (boringssl.gypi and boringssl_tests.gypi) for Chromium, or generates
17  source-list files for Android."""
18
19import os
20import subprocess
21import sys
22import json
23
24
25# OS_ARCH_COMBOS maps from OS and platform to the OpenSSL assembly "style" for
26# that platform and the extension used by asm files.
27OS_ARCH_COMBOS = [
28    ('linux', 'arm', 'linux32', [], 'S'),
29    ('linux', 'aarch64', 'linux64', [], 'S'),
30    ('linux', 'x86', 'elf', ['-fPIC', '-DOPENSSL_IA32_SSE2'], 'S'),
31    ('linux', 'x86_64', 'elf', [], 'S'),
32    ('mac', 'x86', 'macosx', ['-fPIC', '-DOPENSSL_IA32_SSE2'], 'S'),
33    ('mac', 'x86_64', 'macosx', [], 'S'),
34    ('win', 'x86', 'win32n', ['-DOPENSSL_IA32_SSE2'], 'asm'),
35    ('win', 'x86_64', 'nasm', [], 'asm'),
36]
37
38# NON_PERL_FILES enumerates assembly files that are not processed by the
39# perlasm system.
40NON_PERL_FILES = {
41    ('linux', 'arm'): [
42        'src/crypto/chacha/chacha_vec_arm.S',
43        'src/crypto/cpu-arm-asm.S',
44        'src/crypto/curve25519/asm/x25519-asm-arm.S',
45        'src/crypto/poly1305/poly1305_arm_asm.S',
46    ],
47}
48
49
50class Chromium(object):
51
52  def __init__(self):
53    self.header = \
54"""# Copyright (c) 2014 The Chromium Authors. All rights reserved.
55# Use of this source code is governed by a BSD-style license that can be
56# found in the LICENSE file.
57
58# This file is created by generate_build_files.py. Do not edit manually.
59
60"""
61
62  def PrintVariableSection(self, out, name, files):
63    out.write('    \'%s\': [\n' % name)
64    for f in sorted(files):
65      out.write('      \'%s\',\n' % f)
66    out.write('    ],\n')
67
68  def WriteFiles(self, files, asm_outputs):
69    with open('boringssl.gypi', 'w+') as gypi:
70      gypi.write(self.header + '{\n  \'variables\': {\n')
71
72      self.PrintVariableSection(
73          gypi, 'boringssl_ssl_sources', files['ssl'])
74      self.PrintVariableSection(
75          gypi, 'boringssl_crypto_sources', files['crypto'])
76
77      for ((osname, arch), asm_files) in asm_outputs:
78        self.PrintVariableSection(gypi, 'boringssl_%s_%s_sources' %
79                                  (osname, arch), asm_files)
80
81      gypi.write('  }\n}\n')
82
83    with open('boringssl_tests.gypi', 'w+') as test_gypi:
84      test_gypi.write(self.header + '{\n  \'targets\': [\n')
85
86      test_names = []
87      for test in sorted(files['test']):
88        test_name = 'boringssl_%s' % os.path.splitext(os.path.basename(test))[0]
89        test_gypi.write("""    {
90      'target_name': '%s',
91      'type': 'executable',
92      'dependencies': [
93        'boringssl.gyp:boringssl',
94      ],
95      'sources': [
96        '%s',
97        '<@(boringssl_test_support_sources)',
98      ],
99      # TODO(davidben): Fix size_t truncations in BoringSSL.
100      # https://crbug.com/429039
101      'msvs_disabled_warnings': [ 4267, ],
102    },\n""" % (test_name, test))
103        test_names.append(test_name)
104
105      test_names.sort()
106
107      test_gypi.write('  ],\n  \'variables\': {\n')
108
109      self.PrintVariableSection(
110          test_gypi, 'boringssl_test_support_sources', files['test_support'])
111
112      test_gypi.write('    \'boringssl_test_targets\': [\n')
113
114      for test in test_names:
115        test_gypi.write("""      '%s',\n""" % test)
116
117      test_gypi.write('    ],\n  }\n}\n')
118
119
120class Android(object):
121
122  def __init__(self):
123    self.header = \
124"""# Copyright (C) 2015 The Android Open Source Project
125#
126# Licensed under the Apache License, Version 2.0 (the "License");
127# you may not use this file except in compliance with the License.
128# You may obtain a copy of the License at
129#
130#      http://www.apache.org/licenses/LICENSE-2.0
131#
132# Unless required by applicable law or agreed to in writing, software
133# distributed under the License is distributed on an "AS IS" BASIS,
134# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
135# See the License for the specific language governing permissions and
136# limitations under the License.
137
138"""
139
140  def ExtraFiles(self):
141    return ['android_compat_hacks.c', 'android_compat_keywrap.c']
142
143  def PrintVariableSection(self, out, name, files):
144    out.write('%s := \\\n' % name)
145    for f in sorted(files):
146      out.write('  %s\\\n' % f)
147    out.write('\n')
148
149  def WriteFiles(self, files, asm_outputs):
150    with open('sources.mk', 'w+') as makefile:
151      makefile.write(self.header)
152
153      crypto_files = files['crypto'] + self.ExtraFiles()
154      self.PrintVariableSection(makefile, 'crypto_sources', crypto_files)
155      self.PrintVariableSection(makefile, 'ssl_sources', files['ssl'])
156      self.PrintVariableSection(makefile, 'tool_sources', files['tool'])
157
158      for ((osname, arch), asm_files) in asm_outputs:
159        self.PrintVariableSection(
160            makefile, '%s_%s_sources' % (osname, arch), asm_files)
161
162
163class AndroidStandalone(Android):
164  """AndroidStandalone is for Android builds outside of the Android-system, i.e.
165
166  for applications that wish wish to ship BoringSSL.
167  """
168
169  def ExtraFiles(self):
170    return []
171
172
173class Bazel(object):
174  """Bazel outputs files suitable for including in Bazel files."""
175
176  def __init__(self):
177    self.firstSection = True
178    self.header = \
179"""# This file is created by generate_build_files.py. Do not edit manually.
180
181"""
182
183  def PrintVariableSection(self, out, name, files):
184    if not self.firstSection:
185      out.write('\n')
186    self.firstSection = False
187
188    out.write('%s = [\n' % name)
189    for f in sorted(files):
190      out.write('    "%s",\n' % f)
191    out.write(']\n')
192
193  def WriteFiles(self, files, asm_outputs):
194    with open('BUILD.generated.bzl', 'w+') as out:
195      out.write(self.header)
196
197      self.PrintVariableSection(out, 'ssl_headers', files['ssl_headers'])
198      self.PrintVariableSection(
199          out, 'ssl_internal_headers', files['ssl_internal_headers'])
200      self.PrintVariableSection(out, 'ssl_sources', files['ssl'])
201      self.PrintVariableSection(out, 'crypto_headers', files['crypto_headers'])
202      self.PrintVariableSection(
203          out, 'crypto_internal_headers', files['crypto_internal_headers'])
204      self.PrintVariableSection(out, 'crypto_sources', files['crypto'])
205      self.PrintVariableSection(out, 'tool_sources', files['tool'])
206
207      for ((osname, arch), asm_files) in asm_outputs:
208        self.PrintVariableSection(
209            out, 'crypto_sources_%s_%s' % (osname, arch), asm_files)
210
211    with open('BUILD.generated_tests.bzl', 'w+') as out:
212      out.write(self.header)
213
214      out.write('test_support_sources = [\n')
215      for filename in files['test_support']:
216        if os.path.basename(filename) == 'malloc.cc':
217          continue
218        out.write('    "%s",\n' % filename)
219
220      out.write(']\n\n')
221
222      out.write('def create_tests(copts):\n')
223      out.write('  test_support_sources_complete = test_support_sources + \\\n')
224      out.write('      native.glob(["src/crypto/test/*.h"])\n')
225      name_counts = {}
226      for test in files['tests']:
227        name = os.path.basename(test[0])
228        name_counts[name] = name_counts.get(name, 0) + 1
229
230      first = True
231      for test in files['tests']:
232        name = os.path.basename(test[0])
233        if name_counts[name] > 1:
234          if '/' in test[1]:
235            name += '_' + os.path.splitext(os.path.basename(test[1]))[0]
236          else:
237            name += '_' + test[1].replace('-', '_')
238
239        if not first:
240          out.write('\n')
241        first = False
242
243        src_prefix = 'src/' + test[0]
244        for src in files['test']:
245          if src.startswith(src_prefix):
246            src = src
247            break
248        else:
249          raise ValueError("Can't find source for %s" % test[0])
250
251        out.write('  native.cc_test(\n')
252        out.write('      name = "%s",\n' % name)
253        out.write('      size = "small",\n')
254        out.write('      srcs = ["%s"] + test_support_sources_complete,\n' % src)
255
256        data_files = []
257        if len(test) > 1:
258
259          out.write('      args = [\n')
260          for arg in test[1:]:
261            if '/' in arg:
262              out.write('          "$(location src/%s)",\n' % arg)
263              data_files.append('src/%s' % arg)
264            else:
265              out.write('          "%s",\n' % arg)
266          out.write('      ],\n')
267
268        out.write('      copts = copts,\n')
269
270        if len(data_files) > 0:
271          out.write('      data = [\n')
272          for filename in data_files:
273            out.write('          "%s",\n' % filename)
274          out.write('      ],\n')
275
276        if 'ssl/' in test[0]:
277          out.write('      deps = [\n')
278          out.write('          ":crypto",\n')
279          out.write('          ":ssl",\n')
280          out.write('      ],\n')
281        else:
282          out.write('      deps = [":crypto"],\n')
283        out.write('  )\n')
284
285
286def FindCMakeFiles(directory):
287  """Returns list of all CMakeLists.txt files recursively in directory."""
288  cmakefiles = []
289
290  for (path, _, filenames) in os.walk(directory):
291    for filename in filenames:
292      if filename == 'CMakeLists.txt':
293        cmakefiles.append(os.path.join(path, filename))
294
295  return cmakefiles
296
297
298def NoTests(dent, is_dir):
299  """Filter function that can be passed to FindCFiles in order to remove test
300  sources."""
301  if is_dir:
302    return dent != 'test'
303  return 'test.' not in dent and not dent.startswith('example_')
304
305
306def OnlyTests(dent, is_dir):
307  """Filter function that can be passed to FindCFiles in order to remove
308  non-test sources."""
309  if is_dir:
310    return dent != 'test'
311  return '_test.' in dent or dent.startswith('example_')
312
313
314def AllFiles(dent, is_dir):
315  """Filter function that can be passed to FindCFiles in order to include all
316  sources."""
317  return True
318
319
320def SSLHeaderFiles(dent, is_dir):
321  return dent in ['ssl.h', 'tls1.h', 'ssl23.h', 'ssl3.h', 'dtls1.h']
322
323
324def FindCFiles(directory, filter_func):
325  """Recurses through directory and returns a list of paths to all the C source
326  files that pass filter_func."""
327  cfiles = []
328
329  for (path, dirnames, filenames) in os.walk(directory):
330    for filename in filenames:
331      if not filename.endswith('.c') and not filename.endswith('.cc'):
332        continue
333      if not filter_func(filename, False):
334        continue
335      cfiles.append(os.path.join(path, filename))
336
337    for (i, dirname) in enumerate(dirnames):
338      if not filter_func(dirname, True):
339        del dirnames[i]
340
341  return cfiles
342
343
344def FindHeaderFiles(directory, filter_func):
345  """Recurses through directory and returns a list of paths to all the header files that pass filter_func."""
346  hfiles = []
347
348  for (path, dirnames, filenames) in os.walk(directory):
349    for filename in filenames:
350      if not filename.endswith('.h'):
351        continue
352      if not filter_func(filename, False):
353        continue
354      hfiles.append(os.path.join(path, filename))
355
356  return hfiles
357
358
359def ExtractPerlAsmFromCMakeFile(cmakefile):
360  """Parses the contents of the CMakeLists.txt file passed as an argument and
361  returns a list of all the perlasm() directives found in the file."""
362  perlasms = []
363  with open(cmakefile) as f:
364    for line in f:
365      line = line.strip()
366      if not line.startswith('perlasm('):
367        continue
368      if not line.endswith(')'):
369        raise ValueError('Bad perlasm line in %s' % cmakefile)
370      # Remove "perlasm(" from start and ")" from end
371      params = line[8:-1].split()
372      if len(params) < 2:
373        raise ValueError('Bad perlasm line in %s' % cmakefile)
374      perlasms.append({
375          'extra_args': params[2:],
376          'input': os.path.join(os.path.dirname(cmakefile), params[1]),
377          'output': os.path.join(os.path.dirname(cmakefile), params[0]),
378      })
379
380  return perlasms
381
382
383def ReadPerlAsmOperations():
384  """Returns a list of all perlasm() directives found in CMake config files in
385  src/."""
386  perlasms = []
387  cmakefiles = FindCMakeFiles('src')
388
389  for cmakefile in cmakefiles:
390    perlasms.extend(ExtractPerlAsmFromCMakeFile(cmakefile))
391
392  return perlasms
393
394
395def PerlAsm(output_filename, input_filename, perlasm_style, extra_args):
396  """Runs the a perlasm script and puts the output into output_filename."""
397  base_dir = os.path.dirname(output_filename)
398  if not os.path.isdir(base_dir):
399    os.makedirs(base_dir)
400  output = subprocess.check_output(
401      ['perl', input_filename, perlasm_style] + extra_args)
402  with open(output_filename, 'w+') as out_file:
403    out_file.write(output)
404
405
406def ArchForAsmFilename(filename):
407  """Returns the architectures that a given asm file should be compiled for
408  based on substrings in the filename."""
409
410  if 'x86_64' in filename or 'avx2' in filename:
411    return ['x86_64']
412  elif ('x86' in filename and 'x86_64' not in filename) or '586' in filename:
413    return ['x86']
414  elif 'armx' in filename:
415    return ['arm', 'aarch64']
416  elif 'armv8' in filename:
417    return ['aarch64']
418  elif 'arm' in filename:
419    return ['arm']
420  else:
421    raise ValueError('Unknown arch for asm filename: ' + filename)
422
423
424def WriteAsmFiles(perlasms):
425  """Generates asm files from perlasm directives for each supported OS x
426  platform combination."""
427  asmfiles = {}
428
429  for osarch in OS_ARCH_COMBOS:
430    (osname, arch, perlasm_style, extra_args, asm_ext) = osarch
431    key = (osname, arch)
432    outDir = '%s-%s' % key
433
434    for perlasm in perlasms:
435      filename = os.path.basename(perlasm['input'])
436      output = perlasm['output']
437      if not output.startswith('src'):
438        raise ValueError('output missing src: %s' % output)
439      output = os.path.join(outDir, output[4:])
440      if output.endswith('-armx.${ASM_EXT}'):
441        output = output.replace('-armx',
442                                '-armx64' if arch == 'aarch64' else '-armx32')
443      output = output.replace('${ASM_EXT}', asm_ext)
444
445      if arch in ArchForAsmFilename(filename):
446        PerlAsm(output, perlasm['input'], perlasm_style,
447                perlasm['extra_args'] + extra_args)
448        asmfiles.setdefault(key, []).append(output)
449
450  for (key, non_perl_asm_files) in NON_PERL_FILES.iteritems():
451    asmfiles.setdefault(key, []).extend(non_perl_asm_files)
452
453  return asmfiles
454
455
456def main(platforms):
457  crypto_c_files = FindCFiles(os.path.join('src', 'crypto'), NoTests)
458  ssl_c_files = FindCFiles(os.path.join('src', 'ssl'), NoTests)
459  tool_cc_files = FindCFiles(os.path.join('src', 'tool'), NoTests)
460
461  # Generate err_data.c
462  with open('err_data.c', 'w+') as err_data:
463    subprocess.check_call(['go', 'run', 'err_data_generate.go'],
464                          cwd=os.path.join('src', 'crypto', 'err'),
465                          stdout=err_data)
466  crypto_c_files.append('err_data.c')
467
468  test_support_cc_files = FindCFiles(os.path.join('src', 'crypto', 'test'),
469                                     AllFiles)
470
471  test_c_files = FindCFiles(os.path.join('src', 'crypto'), OnlyTests)
472  test_c_files += FindCFiles(os.path.join('src', 'ssl'), OnlyTests)
473
474  ssl_h_files = (
475      FindHeaderFiles(
476          os.path.join('src', 'include', 'openssl'),
477          SSLHeaderFiles))
478
479  def NotSSLHeaderFiles(filename, is_dir):
480    return not SSLHeaderFiles(filename, is_dir)
481  crypto_h_files = (
482      FindHeaderFiles(
483          os.path.join('src', 'include', 'openssl'),
484          NotSSLHeaderFiles))
485
486  ssl_internal_h_files = FindHeaderFiles(os.path.join('src', 'ssl'), NoTests)
487  crypto_internal_h_files = FindHeaderFiles(
488      os.path.join('src', 'crypto'), NoTests)
489
490  with open('src/util/all_tests.json', 'r') as f:
491    tests = json.load(f)
492  test_binaries = set([test[0] for test in tests])
493  test_sources = set([
494      test.replace('.cc', '').replace('.c', '').replace(
495          'src/',
496          '')
497      for test in test_c_files])
498  if test_binaries != test_sources:
499    print 'Test sources and configured tests do not match'
500    a = test_binaries.difference(test_sources)
501    if len(a) > 0:
502      print 'These tests are configured without sources: ' + str(a)
503    b = test_sources.difference(test_binaries)
504    if len(b) > 0:
505      print 'These test sources are not configured: ' + str(b)
506
507  files = {
508      'crypto': crypto_c_files,
509      'crypto_headers': crypto_h_files,
510      'crypto_internal_headers': crypto_internal_h_files,
511      'ssl': ssl_c_files,
512      'ssl_headers': ssl_h_files,
513      'ssl_internal_headers': ssl_internal_h_files,
514      'tool': tool_cc_files,
515      'test': test_c_files,
516      'test_support': test_support_cc_files,
517      'tests': tests,
518  }
519
520  asm_outputs = sorted(WriteAsmFiles(ReadPerlAsmOperations()).iteritems())
521
522  for platform in platforms:
523    platform.WriteFiles(files, asm_outputs)
524
525  return 0
526
527
528def Usage():
529  print 'Usage: python %s [chromium|android|android-standalone|bazel]' % sys.argv[0]
530  sys.exit(1)
531
532
533if __name__ == '__main__':
534  if len(sys.argv) < 2:
535    Usage()
536
537  platforms = []
538  for s in sys.argv[1:]:
539    if s == 'chromium' or s == 'gyp':
540      platforms.append(Chromium())
541    elif s == 'android':
542      platforms.append(Android())
543    elif s == 'android-standalone':
544      platforms.append(AndroidStandalone())
545    elif s == 'bazel':
546      platforms.append(Bazel())
547    else:
548      Usage()
549
550  sys.exit(main(platforms))
551