1#!/usr/bin/python
2
3#
4# Copyright 2015, The Android Open Source Project
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17#
18
19"""Script that is used by developers to run style checks on Java files."""
20
21import argparse
22import errno
23import os
24import shutil
25import subprocess
26import sys
27import tempfile
28import xml.dom.minidom
29import gitlint.git as git
30
31
32def _FindFoldersContaining(root, wanted):
33  """Recursively finds directories that have a file with the given name.
34
35  Args:
36    root: Root folder to start the search from.
37    wanted: The filename that we are looking for.
38
39  Returns:
40    List of folders that has a file with the given name
41  """
42
43  if not root:
44    return []
45  if os.path.islink(root):
46    return []
47  result = []
48  for file_name in os.listdir(root):
49    file_path = os.path.join(root, file_name)
50    if os.path.isdir(file_path):
51      sub_result = _FindFoldersContaining(file_path, wanted)
52      result.extend(sub_result)
53    else:
54      if file_name == wanted:
55        result.append(root)
56  return result
57
58MAIN_DIRECTORY = os.path.normpath(os.path.dirname(__file__))
59CHECKSTYLE_JAR = os.path.join(MAIN_DIRECTORY, 'checkstyle.jar')
60CHECKSTYLE_STYLE = os.path.join(MAIN_DIRECTORY, 'android-style.xml')
61FORCED_RULES = ['com.puppycrawl.tools.checkstyle.checks.imports.ImportOrderCheck',
62                'com.puppycrawl.tools.checkstyle.checks.imports.UnusedImportsCheck']
63SKIPPED_RULES_FOR_TEST_FILES = ['com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocTypeCheck',
64                                'com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocMethodCheck']
65SUBPATH_FOR_TEST_FILES = ['/tests/', '/test/', '/androidTest/']
66SUBPATH_FOR_TEST_DATA_FILES = _FindFoldersContaining(git.repository_root(),
67                                                     'IGNORE_CHECKSTYLE')
68ERROR_UNCOMMITTED = 'You need to commit all modified files before running Checkstyle\n'
69ERROR_UNTRACKED = 'You have untracked java files that are not being checked:\n'
70
71
72def RunCheckstyleOnFiles(java_files, classpath=CHECKSTYLE_JAR, config_xml=CHECKSTYLE_STYLE):
73  """Runs Checkstyle checks on a given set of java_files.
74
75  Args:
76    java_files: A list of files to check.
77    classpath: The colon-delimited list of JARs in the classpath.
78    config_xml: Path of the checkstyle XML configuration file.
79
80  Returns:
81    A tuple of errors and warnings.
82  """
83  print 'Running Checkstyle on inputted files'
84  java_files = map(os.path.abspath, java_files)
85  stdout = _ExecuteCheckstyle(java_files, classpath, config_xml)
86  (errors, warnings) = _ParseAndFilterOutput(stdout)
87  _PrintErrorsAndWarnings(errors, warnings)
88  return errors, warnings
89
90
91def RunCheckstyleOnACommit(commit,
92                           classpath=CHECKSTYLE_JAR,
93                           config_xml=CHECKSTYLE_STYLE,
94                           file_whitelist=None):
95  """Runs Checkstyle checks on a given commit.
96
97  It will run Checkstyle on the changed Java files in a specified commit SHA-1
98  and if that is None it will fallback to check the latest commit of the
99  currently checked out branch.
100
101  Args:
102    commit: A full 40 character SHA-1 of a commit to check.
103    classpath: The colon-delimited list of JARs in the classpath.
104    config_xml: Path of the checkstyle XML configuration file.
105    file_whitelist: A list of whitelisted file paths that should be checked.
106
107  Returns:
108    A tuple of errors and warnings.
109  """
110  if not git.repository_root():
111    print 'FAILURE: not inside a git repository'
112    sys.exit(1)
113  explicit_commit = commit is not None
114  if not explicit_commit:
115    _WarnIfUntrackedFiles()
116    commit = git.last_commit()
117  print 'Running Checkstyle on %s commit' % commit
118  commit_modified_files = _GetModifiedFiles(commit, explicit_commit)
119  commit_modified_files = _FilterFiles(commit_modified_files, file_whitelist)
120  if not commit_modified_files.keys():
121    print 'No Java files to check'
122    return [], []
123
124  (tmp_dir, tmp_file_map) = _GetTempFilesForCommit(
125      commit_modified_files.keys(), commit)
126
127  java_files = tmp_file_map.keys()
128  stdout = _ExecuteCheckstyle(java_files, classpath, config_xml)
129
130  # Remove all the temporary files.
131  shutil.rmtree(tmp_dir)
132
133  (errors, warnings) = _ParseAndFilterOutput(stdout,
134                                             commit,
135                                             commit_modified_files,
136                                             tmp_file_map)
137  _PrintErrorsAndWarnings(errors, warnings)
138  return errors, warnings
139
140
141def _WarnIfUntrackedFiles(out=sys.stdout):
142  """Prints a warning and a list of untracked files if needed."""
143  root = git.repository_root()
144  untracked_files = git.modified_files(root, False)
145  untracked_files = {f for f in untracked_files if f.endswith('.java')}
146  if untracked_files:
147    out.write(ERROR_UNTRACKED)
148    for untracked_file in untracked_files:
149      out.write(untracked_file + '\n')
150    out.write('\n')
151
152
153def _PrintErrorsAndWarnings(errors, warnings):
154  """Prints given errors and warnings."""
155  if errors:
156    print 'ERRORS:'
157    print '\n'.join(errors)
158  if warnings:
159    print 'WARNINGS:'
160    print '\n'.join(warnings)
161
162
163def _ExecuteCheckstyle(java_files, classpath, config_xml):
164  """Runs Checkstyle to check give Java files for style errors.
165
166  Args:
167    java_files: A list of Java files that needs to be checked.
168    classpath: The colon-delimited list of JARs in the classpath.
169    config_xml: Path of the checkstyle XML configuration file.
170
171  Returns:
172    Checkstyle output in XML format.
173  """
174  # Run checkstyle
175  checkstyle_env = os.environ.copy()
176  checkstyle_env['JAVA_CMD'] = 'java'
177  try:
178    check = subprocess.Popen(['java', '-cp', classpath,
179                              'com.puppycrawl.tools.checkstyle.Main', '-c',
180                              config_xml, '-f', 'xml'] + java_files,
181                             stdout=subprocess.PIPE, env=checkstyle_env)
182    stdout, _ = check.communicate()
183  except OSError as e:
184    if e.errno == errno.ENOENT:
185      print 'Error running Checkstyle!'
186      sys.exit(1)
187
188  # A work-around for Checkstyle printing error count to stdio.
189  if 'Checkstyle ends with' in stdout.splitlines()[-1]:
190    stdout = '\n'.join(stdout.splitlines()[:-1])
191  return stdout
192
193
194def _ParseAndFilterOutput(stdout,
195                          sha=None,
196                          commit_modified_files=None,
197                          tmp_file_map=None):
198  result_errors = []
199  result_warnings = []
200  root = xml.dom.minidom.parseString(stdout)
201  for file_element in root.getElementsByTagName('file'):
202    file_name = file_element.attributes['name'].value
203    if tmp_file_map:
204      file_name = tmp_file_map[file_name]
205    modified_lines = None
206    if commit_modified_files:
207      modified_lines = git.modified_lines(file_name,
208                                          commit_modified_files[file_name],
209                                          sha)
210    test_class = any(substring in file_name for substring
211                     in SUBPATH_FOR_TEST_FILES)
212    test_data_class = any(substring in file_name for substring
213                          in SUBPATH_FOR_TEST_DATA_FILES)
214    file_name = os.path.relpath(file_name)
215    errors = file_element.getElementsByTagName('error')
216    for error in errors:
217      line = int(error.attributes['line'].value)
218      rule = error.attributes['source'].value
219      if _ShouldSkip(commit_modified_files, modified_lines, line, rule,
220                     test_class, test_data_class):
221        continue
222
223      column = ''
224      if error.hasAttribute('column'):
225        column = '%s:' % error.attributes['column'].value
226      message = error.attributes['message'].value
227      project = ''
228      if os.environ.get('REPO_PROJECT'):
229        project = '[' + os.environ.get('REPO_PROJECT') + '] '
230
231      result = '  %s%s:%s:%s %s' % (project, file_name, line, column, message)
232
233      severity = error.attributes['severity'].value
234      if severity == 'error':
235        result_errors.append(result)
236      elif severity == 'warning':
237        result_warnings.append(result)
238  return result_errors, result_warnings
239
240
241def _ShouldSkip(commit_check, modified_lines, line, rule, test_class=False,
242                test_data_class=False):
243  """Returns whether an error on a given line should be skipped.
244
245  Args:
246    commit_check: Whether Checkstyle is being run on a specific commit.
247    modified_lines: A list of lines that has been modified.
248    line: The line that has a rule violation.
249    rule: The type of rule that a given line is violating.
250    test_class: Whether the file being checked is a test class.
251    test_data_class: Whether the file being check is a class used as test data.
252
253  Returns:
254    A boolean whether a given line should be skipped in the reporting.
255  """
256  # None modified_lines means checked file is new and nothing should be skipped.
257  if test_data_class:
258    return True
259  if test_class and rule in SKIPPED_RULES_FOR_TEST_FILES:
260    return True
261  if not commit_check:
262    return False
263  if modified_lines is None:
264    return False
265  return line not in modified_lines and rule not in FORCED_RULES
266
267
268def _GetModifiedFiles(commit, explicit_commit=False, out=sys.stdout):
269  root = git.repository_root()
270  pending_files = git.modified_files(root, True)
271  if pending_files and not explicit_commit:
272    out.write(ERROR_UNCOMMITTED)
273    sys.exit(1)
274
275  modified_files = git.modified_files(root, True, commit)
276  modified_files = {f: modified_files[f] for f
277                    in modified_files if f.endswith('.java')}
278  return modified_files
279
280
281def _FilterFiles(files, file_whitelist):
282  if not file_whitelist:
283    return files
284  return {f: files[f] for f in files
285          for whitelist in file_whitelist if whitelist in f}
286
287
288def _GetTempFilesForCommit(file_names, commit):
289  """Creates a temporary snapshot of the files in at a commit.
290
291  Retrieves the state of every file in file_names at a given commit and writes
292  them all out to a temporary directory.
293
294  Args:
295    file_names: A list of files that need to be retrieved.
296    commit: A full 40 character SHA-1 of a commit.
297
298  Returns:
299    A tuple of temprorary directory name and a directionary of
300    temp_file_name: filename. For example:
301
302    ('/tmp/random/', {'/tmp/random/blarg.java': 'real/path/to/file.java' }
303  """
304  tmp_dir_name = tempfile.mkdtemp()
305  tmp_file_names = {}
306  for file_name in file_names:
307    rel_path = os.path.relpath(file_name)
308    content = subprocess.check_output(
309        ['git', 'show', commit + ':' + rel_path])
310
311    tmp_file_name = os.path.join(tmp_dir_name, rel_path)
312    # create directory for the file if it doesn't exist
313    if not os.path.exists(os.path.dirname(tmp_file_name)):
314      os.makedirs(os.path.dirname(tmp_file_name))
315
316    tmp_file = open(tmp_file_name, 'w')
317    tmp_file.write(content)
318    tmp_file.close()
319    tmp_file_names[tmp_file_name] = file_name
320  return tmp_dir_name, tmp_file_names
321
322
323def main(args=None):
324  """Runs Checkstyle checks on a given set of java files or a commit.
325
326  It will run Checkstyle on the list of java files first, if unspecified,
327  then the check will be run on a specified commit SHA-1 and if that
328  is None it will fallback to check the latest commit of the currently checked
329  out branch.
330  """
331  parser = argparse.ArgumentParser()
332  parser.add_argument('--file', '-f', nargs='+')
333  parser.add_argument('--sha', '-s')
334  parser.add_argument('--config_xml', '-c')
335  parser.add_argument('--file_whitelist', '-fw', nargs='+')
336  parser.add_argument('--add_classpath', '-p')
337  args = parser.parse_args()
338
339  config_xml = args.config_xml or CHECKSTYLE_STYLE
340
341  if not os.path.exists(config_xml):
342    print 'Java checkstyle configuration file is missing'
343    sys.exit(1)
344
345  classpath = CHECKSTYLE_JAR
346
347  if args.add_classpath:
348    classpath = args.add_classpath + ':' + classpath
349
350  if args.file:
351    # Files to check were specified via command line.
352    (errors, warnings) = RunCheckstyleOnFiles(args.file, classpath, config_xml)
353  else:
354    (errors, warnings) = RunCheckstyleOnACommit(args.sha, classpath, config_xml,
355                                                args.file_whitelist)
356
357  if errors or warnings:
358    sys.exit(1)
359
360  print 'SUCCESS! NO ISSUES FOUND'
361  sys.exit(0)
362
363
364if __name__ == '__main__':
365  main()
366