1#!/usr/bin/env python
2# Copyright (c) 2012 Google Inc. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Utility functions to perform Xcode-style build steps.
7
8These functions are executed via gyp-mac-tool when using the Makefile generator.
9"""
10
11import fcntl
12import fnmatch
13import glob
14import json
15import os
16import plistlib
17import re
18import shutil
19import string
20import struct
21import subprocess
22import sys
23import tempfile
24
25
26def main(args):
27  executor = MacTool()
28  exit_code = executor.Dispatch(args)
29  if exit_code is not None:
30    sys.exit(exit_code)
31
32
33class MacTool(object):
34  """This class performs all the Mac tooling steps. The methods can either be
35  executed directly, or dispatched from an argument list."""
36
37  def Dispatch(self, args):
38    """Dispatches a string command to a method."""
39    if len(args) < 1:
40      raise Exception("Not enough arguments")
41
42    method = "Exec%s" % self._CommandifyName(args[0])
43    return getattr(self, method)(*args[1:])
44
45  def _CommandifyName(self, name_string):
46    """Transforms a tool name like copy-info-plist to CopyInfoPlist"""
47    return name_string.title().replace('-', '')
48
49  def ExecCopyBundleResource(self, source, dest, convert_to_binary):
50    """Copies a resource file to the bundle/Resources directory, performing any
51    necessary compilation on each resource."""
52    extension = os.path.splitext(source)[1].lower()
53    if os.path.isdir(source):
54      # Copy tree.
55      # TODO(thakis): This copies file attributes like mtime, while the
56      # single-file branch below doesn't. This should probably be changed to
57      # be consistent with the single-file branch.
58      if os.path.exists(dest):
59        shutil.rmtree(dest)
60      shutil.copytree(source, dest)
61    elif extension == '.xib':
62      return self._CopyXIBFile(source, dest)
63    elif extension == '.storyboard':
64      return self._CopyXIBFile(source, dest)
65    elif extension == '.strings':
66      self._CopyStringsFile(source, dest)
67    else:
68      if os.path.exists(dest):
69        os.unlink(dest)
70      shutil.copy(source, dest)
71
72    if extension in ('.plist', '.strings') and convert_to_binary == 'True':
73      self._ConvertToBinary(dest)
74
75  def _CopyXIBFile(self, source, dest):
76    """Compiles a XIB file with ibtool into a binary plist in the bundle."""
77
78    # ibtool sometimes crashes with relative paths. See crbug.com/314728.
79    base = os.path.dirname(os.path.realpath(__file__))
80    if os.path.relpath(source):
81      source = os.path.join(base, source)
82    if os.path.relpath(dest):
83      dest = os.path.join(base, dest)
84
85    args = ['xcrun', 'ibtool', '--errors', '--warnings', '--notices']
86
87    if os.environ['XCODE_VERSION_ACTUAL'] > '0700':
88      args.extend(['--auto-activate-custom-fonts'])
89      if 'IPHONEOS_DEPLOYMENT_TARGET' in os.environ:
90        args.extend([
91            '--target-device', 'iphone', '--target-device', 'ipad',
92            '--minimum-deployment-target',
93            os.environ['IPHONEOS_DEPLOYMENT_TARGET'],
94        ])
95      else:
96        args.extend([
97            '--target-device', 'mac',
98            '--minimum-deployment-target',
99            os.environ['MACOSX_DEPLOYMENT_TARGET'],
100        ])
101
102    args.extend(['--output-format', 'human-readable-text', '--compile', dest,
103        source])
104
105    ibtool_section_re = re.compile(r'/\*.*\*/')
106    ibtool_re = re.compile(r'.*note:.*is clipping its content')
107    ibtoolout = subprocess.Popen(args, stdout=subprocess.PIPE)
108    current_section_header = None
109    for line in ibtoolout.stdout:
110      if ibtool_section_re.match(line):
111        current_section_header = line
112      elif not ibtool_re.match(line):
113        if current_section_header:
114          sys.stdout.write(current_section_header)
115          current_section_header = None
116        sys.stdout.write(line)
117    return ibtoolout.returncode
118
119  def _ConvertToBinary(self, dest):
120    subprocess.check_call([
121        'xcrun', 'plutil', '-convert', 'binary1', '-o', dest, dest])
122
123  def _CopyStringsFile(self, source, dest):
124    """Copies a .strings file using iconv to reconvert the input into UTF-16."""
125    input_code = self._DetectInputEncoding(source) or "UTF-8"
126
127    # Xcode's CpyCopyStringsFile / builtin-copyStrings seems to call
128    # CFPropertyListCreateFromXMLData() behind the scenes; at least it prints
129    #     CFPropertyListCreateFromXMLData(): Old-style plist parser: missing
130    #     semicolon in dictionary.
131    # on invalid files. Do the same kind of validation.
132    import CoreFoundation
133    s = open(source, 'rb').read()
134    d = CoreFoundation.CFDataCreate(None, s, len(s))
135    _, error = CoreFoundation.CFPropertyListCreateFromXMLData(None, d, 0, None)
136    if error:
137      return
138
139    fp = open(dest, 'wb')
140    fp.write(s.decode(input_code).encode('UTF-16'))
141    fp.close()
142
143  def _DetectInputEncoding(self, file_name):
144    """Reads the first few bytes from file_name and tries to guess the text
145    encoding. Returns None as a guess if it can't detect it."""
146    fp = open(file_name, 'rb')
147    try:
148      header = fp.read(3)
149    except e:
150      fp.close()
151      return None
152    fp.close()
153    if header.startswith("\xFE\xFF"):
154      return "UTF-16"
155    elif header.startswith("\xFF\xFE"):
156      return "UTF-16"
157    elif header.startswith("\xEF\xBB\xBF"):
158      return "UTF-8"
159    else:
160      return None
161
162  def ExecCopyInfoPlist(self, source, dest, convert_to_binary, *keys):
163    """Copies the |source| Info.plist to the destination directory |dest|."""
164    # Read the source Info.plist into memory.
165    fd = open(source, 'r')
166    lines = fd.read()
167    fd.close()
168
169    # Insert synthesized key/value pairs (e.g. BuildMachineOSBuild).
170    plist = plistlib.readPlistFromString(lines)
171    if keys:
172      plist = dict(plist.items() + json.loads(keys[0]).items())
173    lines = plistlib.writePlistToString(plist)
174
175    # Go through all the environment variables and replace them as variables in
176    # the file.
177    IDENT_RE = re.compile(r'[/\s]')
178    for key in os.environ:
179      if key.startswith('_'):
180        continue
181      evar = '${%s}' % key
182      evalue = os.environ[key]
183      lines = string.replace(lines, evar, evalue)
184
185      # Xcode supports various suffices on environment variables, which are
186      # all undocumented. :rfc1034identifier is used in the standard project
187      # template these days, and :identifier was used earlier. They are used to
188      # convert non-url characters into things that look like valid urls --
189      # except that the replacement character for :identifier, '_' isn't valid
190      # in a URL either -- oops, hence :rfc1034identifier was born.
191      evar = '${%s:identifier}' % key
192      evalue = IDENT_RE.sub('_', os.environ[key])
193      lines = string.replace(lines, evar, evalue)
194
195      evar = '${%s:rfc1034identifier}' % key
196      evalue = IDENT_RE.sub('-', os.environ[key])
197      lines = string.replace(lines, evar, evalue)
198
199    # Remove any keys with values that haven't been replaced.
200    lines = lines.split('\n')
201    for i in range(len(lines)):
202      if lines[i].strip().startswith("<string>${"):
203        lines[i] = None
204        lines[i - 1] = None
205    lines = '\n'.join(filter(lambda x: x is not None, lines))
206
207    # Write out the file with variables replaced.
208    fd = open(dest, 'w')
209    fd.write(lines)
210    fd.close()
211
212    # Now write out PkgInfo file now that the Info.plist file has been
213    # "compiled".
214    self._WritePkgInfo(dest)
215
216    if convert_to_binary == 'True':
217      self._ConvertToBinary(dest)
218
219  def _WritePkgInfo(self, info_plist):
220    """This writes the PkgInfo file from the data stored in Info.plist."""
221    plist = plistlib.readPlist(info_plist)
222    if not plist:
223      return
224
225    # Only create PkgInfo for executable types.
226    package_type = plist['CFBundlePackageType']
227    if package_type != 'APPL':
228      return
229
230    # The format of PkgInfo is eight characters, representing the bundle type
231    # and bundle signature, each four characters. If that is missing, four
232    # '?' characters are used instead.
233    signature_code = plist.get('CFBundleSignature', '????')
234    if len(signature_code) != 4:  # Wrong length resets everything, too.
235      signature_code = '?' * 4
236
237    dest = os.path.join(os.path.dirname(info_plist), 'PkgInfo')
238    fp = open(dest, 'w')
239    fp.write('%s%s' % (package_type, signature_code))
240    fp.close()
241
242  def ExecFlock(self, lockfile, *cmd_list):
243    """Emulates the most basic behavior of Linux's flock(1)."""
244    # Rely on exception handling to report errors.
245    fd = os.open(lockfile, os.O_RDONLY|os.O_NOCTTY|os.O_CREAT, 0o666)
246    fcntl.flock(fd, fcntl.LOCK_EX)
247    return subprocess.call(cmd_list)
248
249  def ExecFilterLibtool(self, *cmd_list):
250    """Calls libtool and filters out '/path/to/libtool: file: foo.o has no
251    symbols'."""
252    libtool_re = re.compile(r'^.*libtool: (?:for architecture: \S* )?'
253                            r'file: .* has no symbols$')
254    libtool_re5 = re.compile(
255        r'^.*libtool: warning for library: ' +
256        r'.* the table of contents is empty ' +
257        r'\(no object file members in the library define global symbols\)$')
258    env = os.environ.copy()
259    # Ref:
260    # http://www.opensource.apple.com/source/cctools/cctools-809/misc/libtool.c
261    # The problem with this flag is that it resets the file mtime on the file to
262    # epoch=0, e.g. 1970-1-1 or 1969-12-31 depending on timezone.
263    env['ZERO_AR_DATE'] = '1'
264    libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, env=env)
265    _, err = libtoolout.communicate()
266    for line in err.splitlines():
267      if not libtool_re.match(line) and not libtool_re5.match(line):
268        print >>sys.stderr, line
269    # Unconditionally touch the output .a file on the command line if present
270    # and the command succeeded. A bit hacky.
271    if not libtoolout.returncode:
272      for i in range(len(cmd_list) - 1):
273        if cmd_list[i] == "-o" and cmd_list[i+1].endswith('.a'):
274          os.utime(cmd_list[i+1], None)
275          break
276    return libtoolout.returncode
277
278  def ExecPackageIosFramework(self, framework):
279    # Find the name of the binary based on the part before the ".framework".
280    binary = os.path.basename(framework).split('.')[0]
281    module_path = os.path.join(framework, 'Modules');
282    if not os.path.exists(module_path):
283      os.mkdir(module_path)
284    module_template = 'framework module %s {\n' \
285                      '  umbrella header "%s.h"\n' \
286                      '\n' \
287                      '  export *\n' \
288                      '  module * { export * }\n' \
289                      '}\n' % (binary, binary)
290
291    module_file = open(os.path.join(module_path, 'module.modulemap'), "w")
292    module_file.write(module_template)
293    module_file.close()
294
295  def ExecPackageFramework(self, framework, version):
296    """Takes a path to Something.framework and the Current version of that and
297    sets up all the symlinks."""
298    # Find the name of the binary based on the part before the ".framework".
299    binary = os.path.basename(framework).split('.')[0]
300
301    CURRENT = 'Current'
302    RESOURCES = 'Resources'
303    VERSIONS = 'Versions'
304
305    if not os.path.exists(os.path.join(framework, VERSIONS, version, binary)):
306      # Binary-less frameworks don't seem to contain symlinks (see e.g.
307      # chromium's out/Debug/org.chromium.Chromium.manifest/ bundle).
308      return
309
310    # Move into the framework directory to set the symlinks correctly.
311    pwd = os.getcwd()
312    os.chdir(framework)
313
314    # Set up the Current version.
315    self._Relink(version, os.path.join(VERSIONS, CURRENT))
316
317    # Set up the root symlinks.
318    self._Relink(os.path.join(VERSIONS, CURRENT, binary), binary)
319    self._Relink(os.path.join(VERSIONS, CURRENT, RESOURCES), RESOURCES)
320
321    # Back to where we were before!
322    os.chdir(pwd)
323
324  def _Relink(self, dest, link):
325    """Creates a symlink to |dest| named |link|. If |link| already exists,
326    it is overwritten."""
327    if os.path.lexists(link):
328      os.remove(link)
329    os.symlink(dest, link)
330
331  def ExecCompileIosFrameworkHeaderMap(self, out, framework, *all_headers):
332    framework_name = os.path.basename(framework).split('.')[0]
333    all_headers = map(os.path.abspath, all_headers)
334    filelist = {}
335    for header in all_headers:
336      filename = os.path.basename(header)
337      filelist[filename] = header
338      filelist[os.path.join(framework_name, filename)] = header
339    WriteHmap(out, filelist)
340
341  def ExecCopyIosFrameworkHeaders(self, framework, *copy_headers):
342    header_path = os.path.join(framework, 'Headers');
343    if not os.path.exists(header_path):
344      os.makedirs(header_path)
345    for header in copy_headers:
346      shutil.copy(header, os.path.join(header_path, os.path.basename(header)))
347
348  def ExecCompileXcassets(self, keys, *inputs):
349    """Compiles multiple .xcassets files into a single .car file.
350
351    This invokes 'actool' to compile all the inputs .xcassets files. The
352    |keys| arguments is a json-encoded dictionary of extra arguments to
353    pass to 'actool' when the asset catalogs contains an application icon
354    or a launch image.
355
356    Note that 'actool' does not create the Assets.car file if the asset
357    catalogs does not contains imageset.
358    """
359    command_line = [
360      'xcrun', 'actool', '--output-format', 'human-readable-text',
361      '--compress-pngs', '--notices', '--warnings', '--errors',
362    ]
363    is_iphone_target = 'IPHONEOS_DEPLOYMENT_TARGET' in os.environ
364    if is_iphone_target:
365      platform = os.environ['CONFIGURATION'].split('-')[-1]
366      if platform not in ('iphoneos', 'iphonesimulator'):
367        platform = 'iphonesimulator'
368      command_line.extend([
369          '--platform', platform, '--target-device', 'iphone',
370          '--target-device', 'ipad', '--minimum-deployment-target',
371          os.environ['IPHONEOS_DEPLOYMENT_TARGET'], '--compile',
372          os.path.abspath(os.environ['CONTENTS_FOLDER_PATH']),
373      ])
374    else:
375      command_line.extend([
376          '--platform', 'macosx', '--target-device', 'mac',
377          '--minimum-deployment-target', os.environ['MACOSX_DEPLOYMENT_TARGET'],
378          '--compile',
379          os.path.abspath(os.environ['UNLOCALIZED_RESOURCES_FOLDER_PATH']),
380      ])
381    if keys:
382      keys = json.loads(keys)
383      for key, value in keys.iteritems():
384        arg_name = '--' + key
385        if isinstance(value, bool):
386          if value:
387            command_line.append(arg_name)
388        elif isinstance(value, list):
389          for v in value:
390            command_line.append(arg_name)
391            command_line.append(str(v))
392        else:
393          command_line.append(arg_name)
394          command_line.append(str(value))
395    # Note: actool crashes if inputs path are relative, so use os.path.abspath
396    # to get absolute path name for inputs.
397    command_line.extend(map(os.path.abspath, inputs))
398    subprocess.check_call(command_line)
399
400  def ExecMergeInfoPlist(self, output, *inputs):
401    """Merge multiple .plist files into a single .plist file."""
402    merged_plist = {}
403    for path in inputs:
404      plist = self._LoadPlistMaybeBinary(path)
405      self._MergePlist(merged_plist, plist)
406    plistlib.writePlist(merged_plist, output)
407
408  def ExecCodeSignBundle(self, key, entitlements, provisioning, path, preserve):
409    """Code sign a bundle.
410
411    This function tries to code sign an iOS bundle, following the same
412    algorithm as Xcode:
413      1. pick the provisioning profile that best match the bundle identifier,
414         and copy it into the bundle as embedded.mobileprovision,
415      2. copy Entitlements.plist from user or SDK next to the bundle,
416      3. code sign the bundle.
417    """
418    substitutions, overrides = self._InstallProvisioningProfile(
419        provisioning, self._GetCFBundleIdentifier())
420    entitlements_path = self._InstallEntitlements(
421        entitlements, substitutions, overrides)
422
423    args = ['codesign', '--force', '--sign', key]
424    if preserve == 'True':
425      args.extend(['--deep', '--preserve-metadata=identifier,entitlements'])
426    else:
427      args.extend(['--entitlements', entitlements_path])
428    args.extend(['--timestamp=none', path])
429    subprocess.check_call(args)
430
431  def _InstallProvisioningProfile(self, profile, bundle_identifier):
432    """Installs embedded.mobileprovision into the bundle.
433
434    Args:
435      profile: string, optional, short name of the .mobileprovision file
436        to use, if empty or the file is missing, the best file installed
437        will be used
438      bundle_identifier: string, value of CFBundleIdentifier from Info.plist
439
440    Returns:
441      A tuple containing two dictionary: variables substitutions and values
442      to overrides when generating the entitlements file.
443    """
444    source_path, provisioning_data, team_id = self._FindProvisioningProfile(
445        profile, bundle_identifier)
446    target_path = os.path.join(
447        os.environ['BUILT_PRODUCTS_DIR'],
448        os.environ['CONTENTS_FOLDER_PATH'],
449        'embedded.mobileprovision')
450    shutil.copy2(source_path, target_path)
451    substitutions = self._GetSubstitutions(bundle_identifier, team_id + '.')
452    return substitutions, provisioning_data['Entitlements']
453
454  def _FindProvisioningProfile(self, profile, bundle_identifier):
455    """Finds the .mobileprovision file to use for signing the bundle.
456
457    Checks all the installed provisioning profiles (or if the user specified
458    the PROVISIONING_PROFILE variable, only consult it) and select the most
459    specific that correspond to the bundle identifier.
460
461    Args:
462      profile: string, optional, short name of the .mobileprovision file
463        to use, if empty or the file is missing, the best file installed
464        will be used
465      bundle_identifier: string, value of CFBundleIdentifier from Info.plist
466
467    Returns:
468      A tuple of the path to the selected provisioning profile, the data of
469      the embedded plist in the provisioning profile and the team identifier
470      to use for code signing.
471
472    Raises:
473      SystemExit: if no .mobileprovision can be used to sign the bundle.
474    """
475    profiles_dir = os.path.join(
476        os.environ['HOME'], 'Library', 'MobileDevice', 'Provisioning Profiles')
477    if not os.path.isdir(profiles_dir):
478      print >>sys.stderr, (
479          'cannot find mobile provisioning for %s' % bundle_identifier)
480      sys.exit(1)
481    provisioning_profiles = None
482    if profile:
483      profile_path = os.path.join(profiles_dir, profile + '.mobileprovision')
484      if os.path.exists(profile_path):
485        provisioning_profiles = [profile_path]
486    if not provisioning_profiles:
487      provisioning_profiles = glob.glob(
488          os.path.join(profiles_dir, '*.mobileprovision'))
489    valid_provisioning_profiles = {}
490    for profile_path in provisioning_profiles:
491      profile_data = self._LoadProvisioningProfile(profile_path)
492      app_id_pattern = profile_data.get(
493          'Entitlements', {}).get('application-identifier', '')
494      for team_identifier in profile_data.get('TeamIdentifier', []):
495        app_id = '%s.%s' % (team_identifier, bundle_identifier)
496        if fnmatch.fnmatch(app_id, app_id_pattern):
497          valid_provisioning_profiles[app_id_pattern] = (
498              profile_path, profile_data, team_identifier)
499    if not valid_provisioning_profiles:
500      print >>sys.stderr, (
501          'cannot find mobile provisioning for %s' % bundle_identifier)
502      sys.exit(1)
503    # If the user has multiple provisioning profiles installed that can be
504    # used for ${bundle_identifier}, pick the most specific one (ie. the
505    # provisioning profile whose pattern is the longest).
506    selected_key = max(valid_provisioning_profiles, key=lambda v: len(v))
507    return valid_provisioning_profiles[selected_key]
508
509  def _LoadProvisioningProfile(self, profile_path):
510    """Extracts the plist embedded in a provisioning profile.
511
512    Args:
513      profile_path: string, path to the .mobileprovision file
514
515    Returns:
516      Content of the plist embedded in the provisioning profile as a dictionary.
517    """
518    with tempfile.NamedTemporaryFile() as temp:
519      subprocess.check_call([
520          'security', 'cms', '-D', '-i', profile_path, '-o', temp.name])
521      return self._LoadPlistMaybeBinary(temp.name)
522
523  def _MergePlist(self, merged_plist, plist):
524    """Merge |plist| into |merged_plist|."""
525    for key, value in plist.iteritems():
526      if isinstance(value, dict):
527        merged_value = merged_plist.get(key, {})
528        if isinstance(merged_value, dict):
529          self._MergePlist(merged_value, value)
530          merged_plist[key] = merged_value
531        else:
532          merged_plist[key] = value
533      else:
534        merged_plist[key] = value
535
536  def _LoadPlistMaybeBinary(self, plist_path):
537    """Loads into a memory a plist possibly encoded in binary format.
538
539    This is a wrapper around plistlib.readPlist that tries to convert the
540    plist to the XML format if it can't be parsed (assuming that it is in
541    the binary format).
542
543    Args:
544      plist_path: string, path to a plist file, in XML or binary format
545
546    Returns:
547      Content of the plist as a dictionary.
548    """
549    try:
550      # First, try to read the file using plistlib that only supports XML,
551      # and if an exception is raised, convert a temporary copy to XML and
552      # load that copy.
553      return plistlib.readPlist(plist_path)
554    except:
555      pass
556    with tempfile.NamedTemporaryFile() as temp:
557      shutil.copy2(plist_path, temp.name)
558      subprocess.check_call(['plutil', '-convert', 'xml1', temp.name])
559      return plistlib.readPlist(temp.name)
560
561  def _GetSubstitutions(self, bundle_identifier, app_identifier_prefix):
562    """Constructs a dictionary of variable substitutions for Entitlements.plist.
563
564    Args:
565      bundle_identifier: string, value of CFBundleIdentifier from Info.plist
566      app_identifier_prefix: string, value for AppIdentifierPrefix
567
568    Returns:
569      Dictionary of substitutions to apply when generating Entitlements.plist.
570    """
571    return {
572      'CFBundleIdentifier': bundle_identifier,
573      'AppIdentifierPrefix': app_identifier_prefix,
574    }
575
576  def _GetCFBundleIdentifier(self):
577    """Extracts CFBundleIdentifier value from Info.plist in the bundle.
578
579    Returns:
580      Value of CFBundleIdentifier in the Info.plist located in the bundle.
581    """
582    info_plist_path = os.path.join(
583        os.environ['TARGET_BUILD_DIR'],
584        os.environ['INFOPLIST_PATH'])
585    info_plist_data = self._LoadPlistMaybeBinary(info_plist_path)
586    return info_plist_data['CFBundleIdentifier']
587
588  def _InstallEntitlements(self, entitlements, substitutions, overrides):
589    """Generates and install the ${BundleName}.xcent entitlements file.
590
591    Expands variables "$(variable)" pattern in the source entitlements file,
592    add extra entitlements defined in the .mobileprovision file and the copy
593    the generated plist to "${BundlePath}.xcent".
594
595    Args:
596      entitlements: string, optional, path to the Entitlements.plist template
597        to use, defaults to "${SDKROOT}/Entitlements.plist"
598      substitutions: dictionary, variable substitutions
599      overrides: dictionary, values to add to the entitlements
600
601    Returns:
602      Path to the generated entitlements file.
603    """
604    source_path = entitlements
605    target_path = os.path.join(
606        os.environ['BUILT_PRODUCTS_DIR'],
607        os.environ['PRODUCT_NAME'] + '.xcent')
608    if not source_path:
609      source_path = os.path.join(
610          os.environ['SDKROOT'],
611          'Entitlements.plist')
612    shutil.copy2(source_path, target_path)
613    data = self._LoadPlistMaybeBinary(target_path)
614    data = self._ExpandVariables(data, substitutions)
615    if overrides:
616      for key in overrides:
617        if key not in data:
618          data[key] = overrides[key]
619    plistlib.writePlist(data, target_path)
620    return target_path
621
622  def _ExpandVariables(self, data, substitutions):
623    """Expands variables "$(variable)" in data.
624
625    Args:
626      data: object, can be either string, list or dictionary
627      substitutions: dictionary, variable substitutions to perform
628
629    Returns:
630      Copy of data where each references to "$(variable)" has been replaced
631      by the corresponding value found in substitutions, or left intact if
632      the key was not found.
633    """
634    if isinstance(data, str):
635      for key, value in substitutions.iteritems():
636        data = data.replace('$(%s)' % key, value)
637      return data
638    if isinstance(data, list):
639      return [self._ExpandVariables(v, substitutions) for v in data]
640    if isinstance(data, dict):
641      return {k: self._ExpandVariables(data[k], substitutions) for k in data}
642    return data
643
644def NextGreaterPowerOf2(x):
645  return 2**(x).bit_length()
646
647def WriteHmap(output_name, filelist):
648  """Generates a header map based on |filelist|.
649
650  Per Mark Mentovai:
651    A header map is structured essentially as a hash table, keyed by names used
652    in #includes, and providing pathnames to the actual files.
653
654  The implementation below and the comment above comes from inspecting:
655    http://www.opensource.apple.com/source/distcc/distcc-2503/distcc_dist/include_server/headermap.py?txt
656  while also looking at the implementation in clang in:
657    https://llvm.org/svn/llvm-project/cfe/trunk/lib/Lex/HeaderMap.cpp
658  """
659  magic = 1751998832
660  version = 1
661  _reserved = 0
662  count = len(filelist)
663  capacity = NextGreaterPowerOf2(count)
664  strings_offset = 24 + (12 * capacity)
665  max_value_length = len(max(filelist.items(), key=lambda (k,v):len(v))[1])
666
667  out = open(output_name, "wb")
668  out.write(struct.pack('<LHHLLLL', magic, version, _reserved, strings_offset,
669                        count, capacity, max_value_length))
670
671  # Create empty hashmap buckets.
672  buckets = [None] * capacity
673  for file, path in filelist.items():
674    key = 0
675    for c in file:
676      key += ord(c.lower()) * 13
677
678    # Fill next empty bucket.
679    while buckets[key & capacity - 1] is not None:
680      key = key + 1
681    buckets[key & capacity - 1] = (file, path)
682
683  next_offset = 1
684  for bucket in buckets:
685    if bucket is None:
686      out.write(struct.pack('<LLL', 0, 0, 0))
687    else:
688      (file, path) = bucket
689      key_offset = next_offset
690      prefix_offset = key_offset + len(file) + 1
691      suffix_offset = prefix_offset + len(os.path.dirname(path) + os.sep) + 1
692      next_offset = suffix_offset + len(os.path.basename(path)) + 1
693      out.write(struct.pack('<LLL', key_offset, prefix_offset, suffix_offset))
694
695  # Pad byte since next offset starts at 1.
696  out.write(struct.pack('<x'))
697
698  for bucket in buckets:
699    if bucket is not None:
700      (file, path) = bucket
701      out.write(struct.pack('<%ds' % len(file), file))
702      out.write(struct.pack('<s', '\0'))
703      base = os.path.dirname(path) + os.sep
704      out.write(struct.pack('<%ds' % len(base), base))
705      out.write(struct.pack('<s', '\0'))
706      path = os.path.basename(path)
707      out.write(struct.pack('<%ds' % len(path), path))
708      out.write(struct.pack('<s', '\0'))
709
710if __name__ == '__main__':
711  sys.exit(main(sys.argv[1:]))
712