1#!/usr/bin/env python
2
3# Copyright 2014 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7'''Produces various output formats from a set of JavaScript files with
8closure style require/provide calls.
9
10Scans one or more directory trees for JavaScript files.  Then, from a
11given list of top-level files, sorts all required input files topologically.
12The top-level files are appended to the sorted list in the order specified
13on the command line.  If no root directories are specified, the source
14files are assumed to be ordered already and no dependency analysis is
15performed.  The resulting file list can then be used in one of the following
16ways:
17
18- list: a plain list of files, one per line is output.
19
20- html: a series of html <script> tags with src attributes containing paths
21  is output.
22
23- bundle: a concatenation of all the files, separated by newlines is output.
24
25- compressed_bundle: A bundle where non-significant whitespace, including
26  comments, has been stripped is output.
27
28- copy: the files are copied, or hard linked if possible, to the destination
29  directory.  In this case, no output is generated.
30'''
31
32
33import optparse
34import os
35import re
36import shutil
37import sys
38
39_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
40_CHROME_SOURCE = os.path.realpath(
41    os.path.join(_SCRIPT_DIR, *[os.path.pardir] * 6))
42sys.path.insert(0, os.path.join(
43    _CHROME_SOURCE, 'third_party/WebKit/Source/build/scripts'))
44sys.path.insert(0, os.path.join(
45    _CHROME_SOURCE, ('chrome/third_party/chromevox/third_party/' +
46                     'closure-library/closure/bin/build')))
47import depstree
48import rjsmin
49import source
50import treescan
51
52
53def Die(message):
54  '''Prints an error message and exit the program.'''
55  print >>sys.stderr, message
56  sys.exit(1)
57
58
59class SourceWithPaths(source.Source):
60  '''A source.Source object with its relative input and output paths'''
61
62  def __init__(self, content, in_path, out_path):
63    super(SourceWithPaths, self).__init__(content)
64    self._in_path = in_path
65    self._out_path = out_path
66
67  def GetInPath(self):
68    return self._in_path
69
70  def GetOutPath(self):
71    return self._out_path
72
73
74class Bundle():
75  '''An ordered list of sources without duplicates.'''
76
77  def __init__(self):
78    self._added_paths = set()
79    self._added_sources = []
80
81  def Add(self, sources):
82    '''Appends one or more source objects the list if it doesn't already
83    exist.
84
85    Args:
86      sources: A SourceWithPath or an iterable of such objects.
87    '''
88    if isinstance(sources, SourceWithPaths):
89      sources = [sources]
90    for source in sources:
91      path = source.GetInPath()
92      if path not in self._added_paths:
93        self._added_paths.add(path)
94        self._added_sources.append(source)
95
96  def GetInPaths(self):
97    return (source.GetInPath() for source in self._added_sources)
98
99  def GetOutPaths(self):
100    return (source.GetOutPath() for source in self._added_sources)
101
102  def GetSources(self):
103    return self._added_sources
104
105  def GetUncompressedSource(self):
106    return '\n'.join((s.GetSource() for s in self._added_sources))
107
108  def GetCompressedSource(self):
109    return rjsmin.jsmin(self.GetUncompressedSource())
110
111
112class PathRewriter():
113  '''A list of simple path rewrite rules to map relative input paths to
114  relative output paths.
115  '''
116
117  def __init__(self, specs=[]):
118    '''Args:
119      specs: A list of mappings, each consisting of the input prefix and
120        the corresponding output prefix separated by colons.
121    '''
122    self._prefix_map = []
123    for spec in specs:
124      parts = spec.split(':')
125      if len(parts) != 2:
126        Die('Invalid prefix rewrite spec %s' % spec)
127      if not parts[0].endswith('/') and parts[0] != '':
128        parts[0] += '/'
129      self._prefix_map.append(parts)
130    self._prefix_map.sort(reverse=True)
131
132  def RewritePath(self, in_path):
133    '''Rewrites an input path according to the list of rules.
134
135    Args:
136      in_path, str: The input path to rewrite.
137    Returns:
138      str: The corresponding output path.
139    '''
140    for in_prefix, out_prefix in self._prefix_map:
141      if in_path.startswith(in_prefix):
142        return os.path.join(out_prefix, in_path[len(in_prefix):])
143    return in_path
144
145
146def ReadSources(roots=[], source_files=[], need_source_text=False,
147                path_rewriter=PathRewriter(), exclude=[]):
148  '''Reads all source specified on the command line, including sources
149  included by --root options.
150  '''
151
152  def EnsureSourceLoaded(in_path, sources):
153    if in_path not in sources:
154      out_path = path_rewriter.RewritePath(in_path)
155      sources[in_path] = SourceWithPaths(source.GetFileContents(in_path),
156                                         in_path, out_path)
157
158  # Only read the actual source file if we will do a dependency analysis or
159  # the caller asks for it.
160  need_source_text = need_source_text or len(roots) > 0
161  sources = {}
162  for root in roots:
163    for name in treescan.ScanTreeForJsFiles(root):
164      if any((r.search(name) for r in exclude)):
165        continue
166      EnsureSourceLoaded(name, sources)
167  for path in source_files:
168    if need_source_text:
169      EnsureSourceLoaded(path, sources)
170    else:
171      # Just add an empty representation of the source.
172      sources[path] = SourceWithPaths(
173          '', path, path_rewriter.RewritePath(path))
174  return sources
175
176
177def _GetBase(sources):
178  '''Gets the closure base.js file if present among the sources.
179
180  Args:
181    sources: Dictionary with input path names as keys and SourceWithPaths
182      as values.
183  Returns:
184    SourceWithPath: The source file providing the goog namespace.
185  '''
186  for source in sources.itervalues():
187    if (os.path.basename(source.GetInPath()) == 'base.js' and
188        'goog' in source.provides):
189      return source
190  Die('goog.base not provided by any file.')
191
192
193def CalcDeps(bundle, sources, top_level):
194  '''Calculates dependencies for a set of top-level files.
195
196  Args:
197    bundle: Bundle to add the sources to.
198    sources, dict: Mapping from input path to SourceWithPaths objects.
199    top_level, list: List of top-level input paths to calculate dependencies
200      for.
201  '''
202  providers = [s for s in sources.itervalues() if len(s.provides) > 0]
203  deps = depstree.DepsTree(providers)
204  namespaces = []
205  for path in top_level:
206    namespaces.extend(sources[path].requires)
207  # base.js is an implicit dependency that always goes first.
208  bundle.Add(_GetBase(sources))
209  bundle.Add(deps.GetDependencies(namespaces))
210
211
212def _MarkAsCompiled(sources):
213  '''Sets COMPILED to true in the Closure base.js source.
214
215  Args:
216    sources: Dictionary with input paths names as keys and SourcWithPaths
217      objects as values.
218  '''
219  base = _GetBase(sources)
220  new_content, count = re.subn('^var COMPILED = false;$',
221                               'var COMPILED = true;',
222                               base.GetSource(),
223                               count=1,
224                               flags=re.MULTILINE)
225  if count != 1:
226    Die('COMPILED var assignment not found in %s' % base.GetInPath())
227  sources[base.GetInPath()] = SourceWithPaths(
228      new_content,
229      base.GetInPath(),
230      base.GetOutPath())
231
232def LinkOrCopyFiles(sources, dest_dir):
233  '''Copies a list of sources to a destination directory.'''
234
235  def LinkOrCopyOneFile(src, dst):
236    if not os.path.exists(os.path.dirname(dst)):
237      os.makedirs(os.path.dirname(dst))
238    if os.path.exists(dst):
239      # Avoid clobbering the inode if source and destination refer to the
240      # same file already.
241      if os.path.samefile(src, dst):
242        return
243      os.unlink(dst)
244    try:
245      os.link(src, dst)
246    except:
247      shutil.copy(src, dst)
248
249  for source in sources:
250    LinkOrCopyOneFile(source.GetInPath(),
251                      os.path.join(dest_dir, source.GetOutPath()))
252
253
254def WriteOutput(bundle, format, out_file, dest_dir):
255  '''Writes output in the specified format.
256
257  Args:
258    bundle: The ordered bundle iwth all sources already added.
259    format: Output format, one of list, html, bundle, compressed_bundle.
260    out_file: File object to receive the output.
261    dest_dir: Prepended to each path mentioned in the output, if applicable.
262  '''
263  if format == 'list':
264    paths = bundle.GetOutPaths()
265    if dest_dir:
266      paths = (os.path.join(dest_dir, p) for p in paths)
267    paths = (os.path.normpath(p) for p in paths)
268    out_file.write('\n'.join(paths))
269  elif format == 'html':
270    HTML_TEMPLATE = '<script src=\'%s\'>'
271    script_lines = (HTML_TEMPLATE % p for p in bundle.GetOutPaths())
272    out_file.write('\n'.join(script_lines))
273  elif format == 'bundle':
274    out_file.write(bundle.GetUncompressedSource())
275  elif format == 'compressed_bundle':
276    out_file.write(bundle.GetCompressedSource())
277  out_file.write('\n')
278
279
280def CreateOptionParser():
281  parser = optparse.OptionParser(description=__doc__)
282  parser.usage = '%prog [options] <top_level_file>...'
283  parser.add_option('-d', '--dest_dir', action='store', metavar='DIR',
284                    help=('Destination directory.  Used when translating ' +
285                          'input paths to output paths and when copying '
286                          'files.'))
287  parser.add_option('-o', '--output_file', action='store', metavar='FILE',
288                    help=('File to output result to for modes that output '
289                          'a single file.'))
290  parser.add_option('-r', '--root', dest='roots', action='append', default=[],
291                    metavar='ROOT',
292                    help='Roots of directory trees to scan for sources.')
293  parser.add_option('-w', '--rewrite_prefix', action='append', default=[],
294                    dest='prefix_map', metavar='SPEC',
295                    help=('Two path prefixes, separated by colons ' +
296                          'specifying that a file whose (relative) path ' +
297                          'name starts with the first prefix should have ' +
298                          'that prefix replaced by the second prefix to ' +
299                          'form a path relative to the output directory.'))
300  parser.add_option('-m', '--mode', type='choice', action='store',
301                    choices=['list', 'html', 'bundle',
302                             'compressed_bundle', 'copy'],
303                    default='list', metavar='MODE',
304                    help=("Otput mode. One of 'list', 'html', 'bundle', " +
305                          "'compressed_bundle' or 'copy'."))
306  parser.add_option('-x', '--exclude', action='append', default=[],
307                    help=('Exclude files whose full path contains a match for '
308                          'the given regular expression.  Does not apply to '
309                          'filenames given as arguments.'))
310  return parser
311
312
313def main():
314  options, args = CreateOptionParser().parse_args()
315  if len(args) < 1:
316    Die('At least one top-level source file must be specified.')
317  will_output_source_text = options.mode in ('bundle', 'compressed_bundle')
318  path_rewriter = PathRewriter(options.prefix_map)
319  exclude = [re.compile(r) for r in options.exclude]
320  sources = ReadSources(options.roots, args, will_output_source_text,
321                        path_rewriter, exclude)
322  if will_output_source_text:
323    _MarkAsCompiled(sources)
324  bundle = Bundle()
325  if len(options.roots) > 0:
326    CalcDeps(bundle, sources, args)
327  bundle.Add((sources[name] for name in args))
328  if options.mode == 'copy':
329    if options.dest_dir is None:
330      Die('Must specify --dest_dir when copying.')
331    LinkOrCopyFiles(bundle.GetSources(), options.dest_dir)
332  else:
333    if options.output_file:
334      out_file = open(options.output_file, 'w')
335    else:
336      out_file = sys.stdout
337    try:
338      WriteOutput(bundle, options.mode, out_file, options.dest_dir)
339    finally:
340      if options.output_file:
341        out_file.close()
342
343
344if __name__ == '__main__':
345  main()
346