1#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import optparse
8import os
9import shutil
10import re
11import sys
12import textwrap
13
14from util import build_utils
15from util import md5_check
16
17import jar
18
19sys.path.append(build_utils.COLORAMA_ROOT)
20import colorama
21
22
23def ColorJavacOutput(output):
24  fileline_prefix = r'(?P<fileline>(?P<file>[-.\w/\\]+.java):(?P<line>[0-9]+):)'
25  warning_re = re.compile(
26      fileline_prefix + r'(?P<full_message> warning: (?P<message>.*))$')
27  error_re = re.compile(
28      fileline_prefix + r'(?P<full_message> (?P<message>.*))$')
29  marker_re = re.compile(r'\s*(?P<marker>\^)\s*$')
30
31  warning_color = ['full_message', colorama.Fore.YELLOW + colorama.Style.DIM]
32  error_color = ['full_message', colorama.Fore.MAGENTA + colorama.Style.BRIGHT]
33  marker_color = ['marker',  colorama.Fore.BLUE + colorama.Style.BRIGHT]
34
35  def Colorize(line, regex, color):
36    match = regex.match(line)
37    start = match.start(color[0])
38    end = match.end(color[0])
39    return (line[:start]
40            + color[1] + line[start:end]
41            + colorama.Fore.RESET + colorama.Style.RESET_ALL
42            + line[end:])
43
44  def ApplyColor(line):
45    if warning_re.match(line):
46      line = Colorize(line, warning_re, warning_color)
47    elif error_re.match(line):
48      line = Colorize(line, error_re, error_color)
49    elif marker_re.match(line):
50      line = Colorize(line, marker_re, marker_color)
51    return line
52
53  return '\n'.join(map(ApplyColor, output.split('\n')))
54
55
56ERRORPRONE_OPTIONS = [
57  # These crash on lots of targets.
58  '-Xep:ParameterPackage:OFF',
59  '-Xep:OverridesGuiceInjectableMethod:OFF',
60  '-Xep:OverridesJavaxInjectableMethod:OFF',
61]
62
63
64def _FilterJavaFiles(paths, filters):
65  return [f for f in paths
66          if not filters or build_utils.MatchesGlob(f, filters)]
67
68
69_MAX_MANIFEST_LINE_LEN = 72
70
71
72def _ExtractClassFiles(jar_path, dest_dir, java_files):
73  """Extracts all .class files not corresponding to |java_files|."""
74  # Two challenges exist here:
75  # 1. |java_files| have prefixes that are not represented in the the jar paths.
76  # 2. A single .java file results in multiple .class files when it contains
77  #    nested classes.
78  # Here's an example:
79  #   source path: ../../base/android/java/src/org/chromium/Foo.java
80  #   jar paths: org/chromium/Foo.class, org/chromium/Foo$Inner.class
81  # To extract only .class files not related to the given .java files, we strip
82  # off ".class" and "$*.class" and use a substring match against java_files.
83  def extract_predicate(path):
84    if not path.endswith('.class'):
85      return False
86    path_without_suffix = re.sub(r'(?:\$|\.)[^/]+class$', '', path)
87    partial_java_path = path_without_suffix + '.java'
88    return not any(p.endswith(partial_java_path) for p in java_files)
89
90  build_utils.ExtractAll(jar_path, path=dest_dir, predicate=extract_predicate)
91  for path in build_utils.FindInDirectory(dest_dir, '*.class'):
92    shutil.copystat(jar_path, path)
93
94
95def _ConvertToJMakeArgs(javac_cmd, pdb_path):
96  new_args = ['bin/jmake', '-pdb', pdb_path]
97  if javac_cmd[0] != 'javac':
98    new_args.extend(('-jcexec', new_args[0]))
99  if md5_check.PRINT_EXPLANATIONS:
100    new_args.append('-Xtiming')
101
102  do_not_prefix = ('-classpath', '-bootclasspath')
103  skip_next = False
104  for arg in javac_cmd[1:]:
105    if not skip_next and arg not in do_not_prefix:
106      arg = '-C' + arg
107    new_args.append(arg)
108    skip_next = arg in do_not_prefix
109
110  return new_args
111
112
113def _FixTempPathsInIncrementalMetadata(pdb_path, temp_dir):
114  # The .pdb records absolute paths. Fix up paths within /tmp (srcjars).
115  if os.path.exists(pdb_path):
116    # Although its a binary file, search/replace still seems to work fine.
117    with open(pdb_path) as fileobj:
118      pdb_data = fileobj.read()
119    with open(pdb_path, 'w') as fileobj:
120      fileobj.write(re.sub(r'/tmp/[^/]*', temp_dir, pdb_data))
121
122
123def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs):
124  with build_utils.TempDir() as temp_dir:
125    srcjars = options.java_srcjars
126    # The .excluded.jar contains .class files excluded from the main jar.
127    # It is used for incremental compiles.
128    excluded_jar_path = options.jar_path.replace('.jar', '.excluded.jar')
129
130    classes_dir = os.path.join(temp_dir, 'classes')
131    os.makedirs(classes_dir)
132
133    changed_paths = None
134    # jmake can handle deleted files, but it's a rare case and it would
135    # complicate this script's logic.
136    if options.incremental and changes.AddedOrModifiedOnly():
137      changed_paths = set(changes.IterChangedPaths())
138      # Do a full compile if classpath has changed.
139      # jmake doesn't seem to do this on its own... Might be that ijars mess up
140      # its change-detection logic.
141      if any(p in changed_paths for p in classpath_inputs):
142        changed_paths = None
143
144    if options.incremental:
145      # jmake is a compiler wrapper that figures out the minimal set of .java
146      # files that need to be rebuilt given a set of .java files that have
147      # changed.
148      # jmake determines what files are stale based on timestamps between .java
149      # and .class files. Since we use .jars, .srcjars, and md5 checks,
150      # timestamp info isn't accurate for this purpose. Rather than use jmake's
151      # programatic interface (like we eventually should), we ensure that all
152      # .class files are newer than their .java files, and convey to jmake which
153      # sources are stale by having their .class files be missing entirely
154      # (by not extracting them).
155      pdb_path = options.jar_path + '.pdb'
156      javac_cmd = _ConvertToJMakeArgs(javac_cmd, pdb_path)
157      if srcjars:
158        _FixTempPathsInIncrementalMetadata(pdb_path, temp_dir)
159
160    if srcjars:
161      java_dir = os.path.join(temp_dir, 'java')
162      os.makedirs(java_dir)
163      for srcjar in options.java_srcjars:
164        if changed_paths:
165          changed_paths.update(os.path.join(java_dir, f)
166                               for f in changes.IterChangedSubpaths(srcjar))
167        build_utils.ExtractAll(srcjar, path=java_dir, pattern='*.java')
168      jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
169      jar_srcs = _FilterJavaFiles(jar_srcs, options.javac_includes)
170      java_files.extend(jar_srcs)
171      if changed_paths:
172        # Set the mtime of all sources to 0 since we use the absense of .class
173        # files to tell jmake which files are stale.
174        for path in jar_srcs:
175          os.utime(path, (0, 0))
176
177    if java_files:
178      if changed_paths:
179        changed_java_files = [p for p in java_files if p in changed_paths]
180        if os.path.exists(options.jar_path):
181          _ExtractClassFiles(options.jar_path, classes_dir, changed_java_files)
182        if os.path.exists(excluded_jar_path):
183          _ExtractClassFiles(excluded_jar_path, classes_dir, changed_java_files)
184        # Add the extracted files to the classpath. This is required because
185        # when compiling only a subset of files, classes that haven't changed
186        # need to be findable.
187        classpath_idx = javac_cmd.index('-classpath')
188        javac_cmd[classpath_idx + 1] += ':' + classes_dir
189
190      # Can happen when a target goes from having no sources, to having sources.
191      # It's created by the call to build_utils.Touch() below.
192      if options.incremental:
193        if os.path.exists(pdb_path) and not os.path.getsize(pdb_path):
194          os.unlink(pdb_path)
195
196      # Don't include the output directory in the initial set of args since it
197      # being in a temp dir makes it unstable (breaks md5 stamping).
198      cmd = javac_cmd + ['-d', classes_dir] + java_files
199
200      # JMake prints out some diagnostic logs that we want to ignore.
201      # This assumes that all compiler output goes through stderr.
202      stdout_filter = lambda s: ''
203      if md5_check.PRINT_EXPLANATIONS:
204        stdout_filter = None
205
206      attempt_build = lambda: build_utils.CheckOutput(
207          cmd,
208          print_stdout=options.chromium_code,
209          stdout_filter=stdout_filter,
210          stderr_filter=ColorJavacOutput)
211      try:
212        attempt_build()
213      except build_utils.CalledProcessError as e:
214        # Work-around for a bug in jmake (http://crbug.com/551449).
215        if 'project database corrupted' not in e.output:
216          raise
217        print ('Applying work-around for jmake project database corrupted '
218               '(http://crbug.com/551449).')
219        os.unlink(pdb_path)
220        attempt_build()
221    elif options.incremental:
222      # Make sure output exists.
223      build_utils.Touch(pdb_path)
224
225    glob = options.jar_excluded_classes
226    inclusion_predicate = lambda f: not build_utils.MatchesGlob(f, glob)
227    exclusion_predicate = lambda f: not inclusion_predicate(f)
228
229    jar.JarDirectory(classes_dir,
230                     options.jar_path,
231                     predicate=inclusion_predicate)
232    jar.JarDirectory(classes_dir,
233                     excluded_jar_path,
234                     predicate=exclusion_predicate)
235
236
237def _ParseOptions(argv):
238  parser = optparse.OptionParser()
239  build_utils.AddDepfileOption(parser)
240
241  parser.add_option(
242      '--src-gendirs',
243      help='Directories containing generated java files.')
244  parser.add_option(
245      '--java-srcjars',
246      action='append',
247      default=[],
248      help='List of srcjars to include in compilation.')
249  parser.add_option(
250      '--bootclasspath',
251      action='append',
252      default=[],
253      help='Boot classpath for javac. If this is specified multiple times, '
254      'they will all be appended to construct the classpath.')
255  parser.add_option(
256      '--classpath',
257      action='append',
258      help='Classpath for javac. If this is specified multiple times, they '
259      'will all be appended to construct the classpath.')
260  parser.add_option(
261      '--incremental',
262      action='store_true',
263      help='Whether to re-use .class files rather than recompiling them '
264           '(when possible).')
265  parser.add_option(
266      '--javac-includes',
267      default='',
268      help='A list of file patterns. If provided, only java files that match'
269      'one of the patterns will be compiled.')
270  parser.add_option(
271      '--jar-excluded-classes',
272      default='',
273      help='List of .class file patterns to exclude from the jar.')
274  parser.add_option(
275      '--chromium-code',
276      type='int',
277      help='Whether code being compiled should be built with stricter '
278      'warnings for chromium code.')
279  parser.add_option(
280      '--use-errorprone-path',
281      help='Use the Errorprone compiler at this path.')
282  parser.add_option('--jar-path', help='Jar output path.')
283  parser.add_option('--stamp', help='Path to touch on success.')
284
285  options, args = parser.parse_args(argv)
286  build_utils.CheckOptions(options, parser, required=('jar_path',))
287
288  bootclasspath = []
289  for arg in options.bootclasspath:
290    bootclasspath += build_utils.ParseGypList(arg)
291  options.bootclasspath = bootclasspath
292
293  classpath = []
294  for arg in options.classpath:
295    classpath += build_utils.ParseGypList(arg)
296  options.classpath = classpath
297
298  java_srcjars = []
299  for arg in options.java_srcjars:
300    java_srcjars += build_utils.ParseGypList(arg)
301  options.java_srcjars = java_srcjars
302
303  if options.src_gendirs:
304    options.src_gendirs = build_utils.ParseGypList(options.src_gendirs)
305
306  options.javac_includes = build_utils.ParseGypList(options.javac_includes)
307  options.jar_excluded_classes = (
308      build_utils.ParseGypList(options.jar_excluded_classes))
309  return options, args
310
311
312def main(argv):
313  colorama.init()
314
315  argv = build_utils.ExpandFileArgs(argv)
316  options, java_files = _ParseOptions(argv)
317
318  if options.src_gendirs:
319    java_files += build_utils.FindInDirectories(options.src_gendirs, '*.java')
320
321  java_files = _FilterJavaFiles(java_files, options.javac_includes)
322
323  javac_cmd = ['javac']
324  if options.use_errorprone_path:
325    javac_cmd = [options.use_errorprone_path] + ERRORPRONE_OPTIONS
326
327  javac_cmd.extend((
328      '-g',
329      # Chromium only allows UTF8 source files.  Being explicit avoids
330      # javac pulling a default encoding from the user's environment.
331      '-encoding', 'UTF-8',
332      '-classpath', ':'.join(options.classpath),
333      # Prevent compiler from compiling .java files not listed as inputs.
334      # See: http://blog.ltgt.net/most-build-tools-misuse-javac/
335      '-sourcepath', ''
336  ))
337
338  if options.bootclasspath:
339    javac_cmd.extend([
340        '-bootclasspath', ':'.join(options.bootclasspath),
341        '-source', '1.7',
342        '-target', '1.7',
343        ])
344
345  if options.chromium_code:
346    javac_cmd.extend(['-Xlint:unchecked', '-Xlint:deprecation'])
347  else:
348    # XDignore.symbol.file makes javac compile against rt.jar instead of
349    # ct.sym. This means that using a java internal package/class will not
350    # trigger a compile warning or error.
351    javac_cmd.extend(['-XDignore.symbol.file'])
352
353  classpath_inputs = options.bootclasspath
354  if options.classpath:
355    if options.classpath[0].endswith('.interface.jar'):
356      classpath_inputs.extend(options.classpath)
357    else:
358      # TODO(agrieve): Remove this .TOC heuristic once GYP is no more.
359      for path in options.classpath:
360        if os.path.exists(path + '.TOC'):
361          classpath_inputs.append(path + '.TOC')
362        else:
363          classpath_inputs.append(path)
364
365  # Compute the list of paths that when changed, we need to rebuild.
366  input_paths = classpath_inputs + options.java_srcjars + java_files
367
368  output_paths = [
369      options.jar_path,
370      options.jar_path.replace('.jar', '.excluded.jar'),
371  ]
372  if options.incremental:
373    output_paths.append(options.jar_path + '.pdb')
374
375  # An escape hatch to be able to check if incremental compiles are causing
376  # problems.
377  force = int(os.environ.get('DISABLE_INCREMENTAL_JAVAC', 0))
378
379  # List python deps in input_strings rather than input_paths since the contents
380  # of them does not change what gets written to the depsfile.
381  build_utils.CallAndWriteDepfileIfStale(
382      lambda changes: _OnStaleMd5(changes, options, javac_cmd, java_files,
383                                  classpath_inputs),
384      options,
385      input_paths=input_paths,
386      input_strings=javac_cmd,
387      output_paths=output_paths,
388      force=force,
389      pass_changes=True)
390
391
392if __name__ == '__main__':
393  sys.exit(main(sys.argv[1:]))
394