jsbundler.py revision 46d4c2bc3267f3f028f39e7e311b0f89aba2e4fd 
1#!/usr/bin/env python
2
3# Copyright 2014 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7'''Produces various output formats from a set of JavaScript files with
8closure style require/provide calls.
9
10Scans one or more directory trees for JavaScript files.  Then, from a
11given list of top-level files, sorts all required input files topologically.
12The top-level files are appended to the sorted list in the order specified
13on the command line.  If no root directories are specified, the source
14files are assumed to be ordered already and no dependency analysis is
15performed.  The resulting file list can then be used in one of the following
16ways:
17
18- list: a plain list of files, one per line is output.
19
20- html: a series of html <script> tags with src attributes containing paths
21  is output.
22
23- bundle: a concatenation of all the files, separated by newlines is output.
24
25- compressed_bundle: A bundle where non-significant whitespace, including
26  comments, has been stripped is output.
27
28- copy: the files are copied, or hard linked if possible, to the destination
29  directory.  In this case, no output is generated.
30'''
31
32
33import optparse
34import os
35import shutil
36import sys
37
38_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
39_CHROME_SOURCE = os.path.realpath(
40    os.path.join(_SCRIPT_DIR, *[os.path.pardir] * 6))
41sys.path.insert(0, os.path.join(
42    _CHROME_SOURCE, 'third_party/WebKit/Source/build/scripts'))
43sys.path.insert(0, os.path.join(
44    _CHROME_SOURCE, ('chrome/third_party/chromevox/third_party/' +
45                     'closure-library/closure/bin/build')))
46import depstree
47import rjsmin
48import source
49import treescan
50
51
52def Die(message):
53  '''Prints an error message and exit the program.'''
54  print >>sys.stderr, message
55  sys.exit(1)
56
57
58class SourceWithPaths(source.Source):
59  '''A source.Source object with its relative input and output paths'''
60
61  def __init__(self, content, in_path, out_path):
62    super(SourceWithPaths, self).__init__(content)
63    self._in_path = in_path
64    self._out_path = out_path
65
66  def GetInPath(self):
67    return self._in_path
68
69  def GetOutPath(self):
70    return self._out_path
71
72
73class Bundle():
74  '''An ordered list of sources without duplicates.'''
75
76  def __init__(self):
77    self._added_paths = set()
78    self._added_sources = []
79
80  def Add(self, sources):
81    '''Appends one or more source objects the list if it doesn't already
82    exist.
83
84    Args:
85      sources: A SourceWithPath or an iterable of such objects.
86    '''
87    if isinstance(sources, SourceWithPaths):
88      sources = [sources]
89    for source in sources:
90      path = source.GetInPath()
91      if path not in self._added_paths:
92        self._added_paths.add(path)
93        self._added_sources.append(source)
94
95  def GetOutPaths(self):
96    return (source.GetOutPath() for source in self._added_sources)
97
98  def GetSources(self):
99    return self._added_sources
100
101  def GetUncompressedSource(self):
102    return '\n'.join((s.GetSource() for s in self._added_sources))
103
104  def GetCompressedSource(self):
105    return rjsmin.jsmin(self.GetUncompressedSource())
106
107
108class PathRewriter():
109  '''A list of simple path rewrite rules to map relative input paths to
110  relative output paths.
111  '''
112
113  def __init__(self, specs):
114    '''Args:
115      specs: A list of mappings, each consisting of the input prefix and
116        the corresponding output prefix separated by colons.
117    '''
118    self._prefix_map = []
119    for spec in specs:
120      parts = spec.split(':')
121      if len(parts) != 2:
122        Die('Invalid prefix rewrite spec %s' % spec)
123      if not parts[0].endswith('/') and parts[0] != '':
124        parts[0] += '/'
125      self._prefix_map.append(parts)
126    self._prefix_map.sort(reverse=True)
127
128  def RewritePath(self, in_path):
129    '''Rewrites an input path according to the list of rules.
130
131    Args:
132      in_path, str: The input path to rewrite.
133    Returns:
134      str: The corresponding output path.
135    '''
136    for in_prefix, out_prefix in self._prefix_map:
137      if in_path.startswith(in_prefix):
138        return os.path.join(out_prefix, in_path[len(in_prefix):])
139    return in_path
140
141
142def ReadSources(options, args):
143  '''Reads all source specified on the command line, including sources
144  included by --root options.
145  '''
146
147  def EnsureSourceLoaded(in_path, sources, path_rewriter):
148    if in_path not in sources:
149      out_path = path_rewriter.RewritePath(in_path)
150      sources[in_path] = SourceWithPaths(source.GetFileContents(in_path),
151                                         in_path, out_path)
152
153  # Only read the actual source file if we will do a dependency analysis or
154  # if we'll need it for the output.
155  need_source_text = (len(options.roots) > 0 or
156                      options.mode in ('bundle', 'compressed_bundle'))
157  path_rewriter = PathRewriter(options.prefix_map)
158  sources = {}
159  for root in options.roots:
160    for name in treescan.ScanTreeForJsFiles(root):
161      EnsureSourceLoaded(name, sources, path_rewriter)
162  for path in args:
163    if need_source_text:
164      EnsureSourceLoaded(path, sources, path_rewriter)
165    else:
166      # Just add an empty representation of the source.
167      sources[path] = SourceWithPaths(
168          '', path, path_rewriter.RewritePath(path))
169  return sources
170
171
172def CalcDeps(bundle, sources, top_level):
173  '''Calculates dependencies for a set of top-level files.
174
175  Args:
176    bundle: Bundle to add the sources to.
177    sources, dict: Mapping from input path to SourceWithPaths objects.
178    top_level, list: List of top-level input paths to calculate dependencies
179      for.
180  '''
181  def GetBase(sources):
182    for source in sources.itervalues():
183      if (os.path.basename(source.GetInPath()) == 'base.js' and
184          'goog' in source.provides):
185        return source
186    Die('goog.base not provided by any file')
187
188  providers = [s for s in sources.itervalues() if len(s.provides) > 0]
189  deps = depstree.DepsTree(providers)
190  namespaces = []
191  for path in top_level:
192    namespaces.extend(sources[path].requires)
193  # base.js is an implicit dependency that always goes first.
194  bundle.Add(GetBase(sources))
195  bundle.Add(deps.GetDependencies(namespaces))
196
197
198def LinkOrCopyFiles(sources, dest_dir):
199  '''Copies a list of sources to a destination directory.'''
200
201  def LinkOrCopyOneFile(src, dst):
202    if not os.path.exists(os.path.dirname(dst)):
203      os.makedirs(os.path.dirname(dst))
204    if os.path.exists(dst):
205      # Avoid clobbering the inode if source and destination refer to the
206      # same file already.
207      if os.path.samefile(src, dst):
208        return
209      os.unlink(dst)
210    try:
211      os.link(src, dst)
212    except:
213      shutil.copy(src, dst)
214
215  for source in sources:
216    LinkOrCopyOneFile(source.GetInPath(),
217                      os.path.join(dest_dir, source.GetOutPath()))
218
219
220def WriteOutput(bundle, format, out_file, dest_dir):
221  '''Writes output in the specified format.
222
223  Args:
224    bundle: The ordered bundle iwth all sources already added.
225    format: Output format, one of list, html, bundle, compressed_bundle.
226    out_file: File object to receive the output.
227    dest_dir: Prepended to each path mentioned in the output, if applicable.
228  '''
229  if format == 'list':
230    paths = bundle.GetOutPaths()
231    if dest_dir:
232      paths = (os.path.join(dest_dir, p) for p in paths)
233    paths = (os.path.normpath(p) for p in paths)
234    out_file.write('\n'.join(paths))
235  elif format == 'html':
236    HTML_TEMPLATE = '<script src=\'%s\'>'
237    script_lines = (HTML_TEMPLATE % p for p in bundle.GetOutPaths())
238    out_file.write('\n'.join(script_lines))
239  elif format == 'bundle':
240    out_file.write(bundle.GetUncompressedSource())
241  elif format == 'compressed_bundle':
242    out_file.write(bundle.GetCompressedSource())
243  out_file.write('\n')
244
245
246def CreateOptionParser():
247  parser = optparse.OptionParser(description=__doc__)
248  parser.usage = '%prog [options] <top_level_file>...'
249  parser.add_option('-d', '--dest_dir', action='store', metavar='DIR',
250                    help=('Destination directory.  Used when translating ' +
251                          'input paths to output paths and when copying '
252                          'files.'))
253  parser.add_option('-o', '--output_file', action='store', metavar='FILE',
254                    help=('File to output result to for modes that output '
255                          'a single file.'))
256  parser.add_option('-r', '--root', dest='roots', action='append', default=[],
257                    metavar='ROOT',
258                    help='Roots of directory trees to scan for sources.')
259  parser.add_option('-w', '--rewrite_prefix', action='append', default=[],
260                    dest='prefix_map', metavar='SPEC',
261                    help=('Two path prefixes, separated by colons ' +
262                          'specifying that a file whose (relative) path ' +
263                          'name starts with the first prefix should have ' +
264                          'that prefix replaced by the second prefix to ' +
265                          'form a path relative to the output directory.'))
266  parser.add_option('-m', '--mode', type='choice', action='store',
267                    choices=['list', 'html', 'bundle',
268                             'compressed_bundle', 'copy'],
269                    default='list', metavar='MODE',
270                    help=("Otput mode. One of 'list', 'html', 'bundle', " +
271                          "'compressed_bundle' or 'copy'."))
272  return parser
273
274
275def main():
276  options, args = CreateOptionParser().parse_args()
277  if len(args) < 1:
278    Die('At least one top-level source file must be specified.')
279  sources = ReadSources(options, args)
280  bundle = Bundle()
281  if len(options.roots) > 0:
282    CalcDeps(bundle, sources, args)
283  bundle.Add((sources[name] for name in args))
284  if options.mode == 'copy':
285    if options.dest_dir is None:
286      Die('Must specify --dest_dir when copying.')
287    LinkOrCopyFiles(bundle.GetSources(), options.dest_dir)
288  else:
289    if options.output_file:
290      out_file = open(options.output_file, 'w')
291    else:
292      out_file = sys.stdout
293    try:
294      WriteOutput(bundle, options.mode, out_file, options.dest_dir)
295    finally:
296      if options.output_file:
297        out_file.close()
298
299
300if __name__ == '__main__':
301  main()
302