jsbundler.py revision cedac228d2dd51db4b79ea1e72c7f249408ee061 
1#!/usr/bin/env python
2
3# Copyright 2014 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7'''Produces various output formats from a set of JavaScript files with
8closure style require/provide calls.
9
10Scans one or more directory trees for JavaScript files.  Then, from a
11given list of top-level files, sorts all required input files topologically.
12The top-level files are appended to the sorted list in the order specified
13on the command line.  If no root directories are specified, the source
14files are assumed to be ordered already and no dependency analysis is
15performed.  The resulting file list can then be used in one of the following
16ways:
17
18- list: a plain list of files, one per line is output.
19
20- html: a series of html <script> tags with src attributes containing paths
21  is output.
22
23- bundle: a concatenation of all the files, separated by newlines is output.
24
25- compressed_bundle: A bundle where non-significant whitespace, including
26  comments, has been stripped is output.
27
28- copy: the files are copied, or hard linked if possible, to the destination
29  directory.  In this case, no output is generated.
30'''
31
32
33import optparse
34import os
35import shutil
36import sys
37
38_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
39_CHROME_SOURCE = os.path.realpath(
40    os.path.join(_SCRIPT_DIR, *[os.path.pardir] * 6))
41sys.path.insert(0, os.path.join(
42    _CHROME_SOURCE, 'third_party/WebKit/Source/build/scripts'))
43sys.path.insert(0, os.path.join(
44    _CHROME_SOURCE, ('chrome/third_party/chromevox/third_party/' +
45                     'closure-library/closure/bin/build')))
46import depstree
47import rjsmin
48import source
49import treescan
50
51
52def Die(message):
53  '''Prints an error message and exit the program.'''
54  print >>sys.stderr, message
55  sys.exit(1)
56
57
58class SourceWithPaths(source.Source):
59  '''A source.Source object with its relative input and output paths'''
60
61  def __init__(self, content, in_path, out_path):
62    super(SourceWithPaths, self).__init__(content)
63    self._in_path = in_path
64    self._out_path = out_path
65
66  def GetInPath(self):
67    return self._in_path
68
69  def GetOutPath(self):
70    return self._out_path
71
72
73class Bundle():
74  '''An ordered list of sources without duplicates.'''
75
76  def __init__(self):
77    self._added_paths = set()
78    self._added_sources = []
79
80  def Add(self, sources):
81    '''Appends one or more source objects the list if it doesn't already
82    exist.
83
84    Args:
85      sources: A SourceWithPath or an iterable of such objects.
86    '''
87    if isinstance(sources, SourceWithPaths):
88      sources = [sources]
89    for source in sources:
90      path = source.GetInPath()
91      if path not in self._added_paths:
92        self._added_paths.add(path)
93        self._added_sources.append(source)
94
95  def GetOutPaths(self):
96    return (source.GetOutPath() for source in self._added_sources)
97
98  def GetSources(self):
99    return self._added_sources
100
101  def GetUncompressedSource(self):
102    return '\n'.join((s.GetSource() for s in self._added_sources))
103
104  def GetCompressedSource(self):
105    return rjsmin.jsmin(self.GetUncompressedSource())
106
107
108class PathRewriter():
109  '''A list of simple path rewrite rules to map relative input paths to
110  relative output paths.
111  '''
112
113  def __init__(self, specs):
114    '''Args:
115      specs: A list of mappings, each consisting of the input prefix and
116        the corresponding output prefix separated by colons.
117    '''
118    self._prefix_map = []
119    for spec in specs:
120      parts = spec.split(':')
121      if len(parts) != 2:
122        Die('Invalid prefix rewrite spec %s' % spec)
123      if not parts[0].endswith('/'):
124        parts[0] += '/'
125      self._prefix_map.append(parts)
126
127  def RewritePath(self, in_path):
128    '''Rewrites an input path according to the list of rules.
129
130    Args:
131      in_path, str: The input path to rewrite.
132    Returns:
133      str: The corresponding output path.
134    '''
135    for in_prefix, out_prefix in self._prefix_map:
136      if in_path.startswith(in_prefix):
137        return os.path.join(out_prefix, in_path[len(in_prefix):])
138    return in_path
139
140
141def ReadSources(options, args):
142  '''Reads all source specified on the command line, including sources
143  included by --root options.
144  '''
145
146  def EnsureSourceLoaded(in_path, sources, path_rewriter):
147    if in_path not in sources:
148      out_path = path_rewriter.RewritePath(in_path)
149      sources[in_path] = SourceWithPaths(source.GetFileContents(in_path),
150                                         in_path, out_path)
151
152  # Only read the actual source file if we will do a dependency analysis or
153  # if we'll need it for the output.
154  need_source_text = (len(options.roots) > 0 or
155                      options.mode in ('bundle', 'compressed_bundle'))
156  path_rewriter = PathRewriter(options.prefix_map)
157  sources = {}
158  for root in options.roots:
159    for name in treescan.ScanTreeForJsFiles(root):
160      EnsureSourceLoaded(name, sources, path_rewriter)
161  for path in args:
162    if need_source_text:
163      EnsureSourceLoaded(path, sources, path_rewriter)
164    else:
165      # Just add an empty representation of the source.
166      sources[path] = SourceWithPaths(
167          '', path, path_rewriter.RewritePath(path))
168  return sources
169
170
171def CalcDeps(bundle, sources, top_level):
172  '''Calculates dependencies for a set of top-level files.
173
174  Args:
175    bundle: Bundle to add the sources to.
176    sources, dict: Mapping from input path to SourceWithPaths objects.
177    top_level, list: List of top-level input paths to calculate dependencies
178      for.
179  '''
180  def GetBase(sources):
181    for source in sources.itervalues():
182      if (os.path.basename(source.GetInPath()) == 'base.js' and
183          'goog' in source.provides):
184        return source
185    Die('goog.base not provided by any file')
186
187  providers = [s for s in sources.itervalues() if len(s.provides) > 0]
188  deps = depstree.DepsTree(providers)
189  namespaces = []
190  for path in top_level:
191    namespaces.extend(sources[path].requires)
192  # base.js is an implicit dependency that always goes first.
193  bundle.Add(GetBase(sources))
194  bundle.Add(deps.GetDependencies(namespaces))
195
196
197def LinkOrCopyFiles(sources, dest_dir):
198  '''Copies a list of sources to a destination directory.'''
199
200  def LinkOrCopyOneFile(src, dst):
201    if not os.path.exists(os.path.dirname(dst)):
202      os.makedirs(os.path.dirname(dst))
203    if os.path.exists(dst):
204      # Avoid clobbering the inode if source and destination refer to the
205      # same file already.
206      if os.path.samefile(src, dst):
207        return
208      os.unlink(dst)
209    try:
210      os.link(src, dst)
211    except:
212      shutil.copy(src, dst)
213
214  for source in sources:
215    LinkOrCopyOneFile(source.GetInPath(),
216                      os.path.join(dest_dir, source.GetOutPath()))
217
218
219def WriteOutput(bundle, format, out_file, dest_dir):
220  '''Writes output in the specified format.
221
222  Args:
223    bundle: The ordered bundle iwth all sources already added.
224    format: Output format, one of list, html, bundle, compressed_bundle.
225    out_file: File object to receive the output.
226    dest_dir: Prepended to each path mentioned in the output, if applicable.
227  '''
228  if format == 'list':
229    paths = bundle.GetOutPaths()
230    if dest_dir:
231      paths = (os.path.join(dest_dir, p) for p in paths)
232    paths = (os.path.normpath(p) for p in paths)
233    out_file.write('\n'.join(paths))
234  elif format == 'html':
235    HTML_TEMPLATE = '<script src=\'%s\'>'
236    script_lines = (HTML_TEMPLATE % p for p in bundle.GetOutPaths())
237    out_file.write('\n'.join(script_lines))
238  elif format == 'bundle':
239    out_file.write(bundle.GetUncompressedSource())
240  elif format == 'compressed_bundle':
241    out_file.write(bundle.GetCompressedSource())
242  out_file.write('\n')
243
244
245def CreateOptionParser():
246  parser = optparse.OptionParser(description=__doc__)
247  parser.usage = '%prog [options] <top_level_file>...'
248  parser.add_option('-d', '--dest_dir', action='store', metavar='DIR',
249                    help=('Destination directory.  Used when translating ' +
250                          'input paths to output paths and when copying '
251                          'files.'))
252  parser.add_option('-o', '--output_file', action='store', metavar='FILE',
253                    help=('File to output result to for modes that output '
254                          'a single file.'))
255  parser.add_option('-r', '--root', dest='roots', action='append', default=[],
256                    metavar='ROOT',
257                    help='Roots of directory trees to scan for sources.')
258  parser.add_option('-w', '--rewrite_prefix', action='append', default=[],
259                    dest='prefix_map', metavar='SPEC',
260                    help=('Two path prefixes, separated by colons ' +
261                          'specifying that a file whose (relative) path ' +
262                          'name starts with the first prefix should have ' +
263                          'that prefix replaced by the second prefix to ' +
264                          'form a path relative to the output directory.'))
265  parser.add_option('-m', '--mode', type='choice', action='store',
266                    choices=['list', 'html', 'bundle',
267                             'compressed_bundle', 'copy'],
268                    default='list', metavar='MODE',
269                    help=("Otput mode. One of 'list', 'html', 'bundle', " +
270                          "'compressed_bundle' or 'copy'."))
271  return parser
272
273
274def main():
275  options, args = CreateOptionParser().parse_args()
276  if len(args) < 1:
277    Die('At least one top-level source file must be specified.')
278  sources = ReadSources(options, args)
279  bundle = Bundle()
280  if len(options.roots) > 0:
281    CalcDeps(bundle, sources, args)
282  bundle.Add((sources[name] for name in args))
283  if options.mode == 'copy':
284    if options.dest_dir is None:
285      Die('Must specify --dest_dir when copying.')
286    LinkOrCopyFiles(bundle.GetSources(), options.dest_dir)
287  else:
288    if options.output_file:
289      out_file = open(options.output_file, 'w')
290    else:
291      out_file = sys.stdout
292    try:
293      WriteOutput(bundle, options.mode, out_file, options.dest_dir)
294    finally:
295      if options.output_file:
296        out_file.close()
297
298
299if __name__ == '__main__':
300  main()
301