1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30try:
31  import hashlib
32  md5er = hashlib.md5
33except ImportError, e:
34  import md5
35  md5er = md5.new
36
37
38import json
39import optparse
40import os
41from os.path import abspath, join, dirname, basename, exists
42import pickle
43import re
44import sys
45import subprocess
46import multiprocessing
47from subprocess import PIPE
48
49from testrunner.local import statusfile
50from testrunner.local import testsuite
51from testrunner.local import utils
52
53# Special LINT rules diverging from default and reason.
54# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
55# build/include_what_you_use: Started giving false positives for variables
56#   named "string" and "map" assuming that you needed to include STL headers.
57# TODO(bmeurer): Fix and re-enable readability/check
58
59LINT_RULES = """
60-build/header_guard
61-build/include_what_you_use
62-build/namespaces
63-readability/check
64+readability/streams
65-runtime/references
66""".split()
67
68LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
69FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
70
71def CppLintWorker(command):
72  try:
73    process = subprocess.Popen(command, stderr=subprocess.PIPE)
74    process.wait()
75    out_lines = ""
76    error_count = -1
77    while True:
78      out_line = process.stderr.readline()
79      if out_line == '' and process.poll() != None:
80        if error_count == -1:
81          print "Failed to process %s" % command.pop()
82          return 1
83        break
84      m = LINT_OUTPUT_PATTERN.match(out_line)
85      if m:
86        out_lines += out_line
87        error_count += 1
88    sys.stdout.write(out_lines)
89    return error_count
90  except KeyboardInterrupt:
91    process.kill()
92  except:
93    print('Error running cpplint.py. Please make sure you have depot_tools' +
94          ' in your $PATH. Lint check skipped.')
95    process.kill()
96
97
98class FileContentsCache(object):
99
100  def __init__(self, sums_file_name):
101    self.sums = {}
102    self.sums_file_name = sums_file_name
103
104  def Load(self):
105    try:
106      sums_file = None
107      try:
108        sums_file = open(self.sums_file_name, 'r')
109        self.sums = pickle.load(sums_file)
110      except:
111        # Cannot parse pickle for any reason. Not much we can do about it.
112        pass
113    finally:
114      if sums_file:
115        sums_file.close()
116
117  def Save(self):
118    try:
119      sums_file = open(self.sums_file_name, 'w')
120      pickle.dump(self.sums, sums_file)
121    except:
122      # Failed to write pickle. Try to clean-up behind us.
123      if sums_file:
124        sums_file.close()
125      try:
126        os.unlink(self.sums_file_name)
127      except:
128        pass
129    finally:
130      sums_file.close()
131
132  def FilterUnchangedFiles(self, files):
133    changed_or_new = []
134    for file in files:
135      try:
136        handle = open(file, "r")
137        file_sum = md5er(handle.read()).digest()
138        if not file in self.sums or self.sums[file] != file_sum:
139          changed_or_new.append(file)
140          self.sums[file] = file_sum
141      finally:
142        handle.close()
143    return changed_or_new
144
145  def RemoveFile(self, file):
146    if file in self.sums:
147      self.sums.pop(file)
148
149
150class SourceFileProcessor(object):
151  """
152  Utility class that can run through a directory structure, find all relevant
153  files and invoke a custom check on the files.
154  """
155
156  def Run(self, path):
157    all_files = []
158    for file in self.GetPathsToSearch():
159      all_files += self.FindFilesIn(join(path, file))
160    if not self.ProcessFiles(all_files, path):
161      return False
162    return True
163
164  def IgnoreDir(self, name):
165    return (name.startswith('.') or
166            name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
167                     'octane', 'sunspider'))
168
169  def IgnoreFile(self, name):
170    return name.startswith('.')
171
172  def FindFilesIn(self, path):
173    result = []
174    for (root, dirs, files) in os.walk(path):
175      for ignored in [x for x in dirs if self.IgnoreDir(x)]:
176        dirs.remove(ignored)
177      for file in files:
178        if not self.IgnoreFile(file) and self.IsRelevant(file):
179          result.append(join(root, file))
180    return result
181
182
183class CppLintProcessor(SourceFileProcessor):
184  """
185  Lint files to check that they follow the google code style.
186  """
187
188  def IsRelevant(self, name):
189    return name.endswith('.cc') or name.endswith('.h')
190
191  def IgnoreDir(self, name):
192    return (super(CppLintProcessor, self).IgnoreDir(name)
193              or (name == 'third_party'))
194
195  IGNORE_LINT = ['flag-definitions.h']
196
197  def IgnoreFile(self, name):
198    return (super(CppLintProcessor, self).IgnoreFile(name)
199              or (name in CppLintProcessor.IGNORE_LINT))
200
201  def GetPathsToSearch(self):
202    return ['src', 'include', 'samples', join('test', 'cctest'),
203            join('test', 'unittests')]
204
205  def GetCpplintScript(self, prio_path):
206    for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
207      path = path.strip('"')
208      cpplint = os.path.join(path, "cpplint.py")
209      if os.path.isfile(cpplint):
210        return cpplint
211
212    return None
213
214  def ProcessFiles(self, files, path):
215    good_files_cache = FileContentsCache('.cpplint-cache')
216    good_files_cache.Load()
217    files = good_files_cache.FilterUnchangedFiles(files)
218    if len(files) == 0:
219      print 'No changes in files detected. Skipping cpplint check.'
220      return True
221
222    filters = ",".join([n for n in LINT_RULES])
223    command = [sys.executable, 'cpplint.py', '--filter', filters]
224    cpplint = self.GetCpplintScript(join(path, "tools"))
225    if cpplint is None:
226      print('Could not find cpplint.py. Make sure '
227            'depot_tools is installed and in the path.')
228      sys.exit(1)
229
230    command = [sys.executable, cpplint, '--filter', filters]
231
232    commands = join([command + [file] for file in files])
233    count = multiprocessing.cpu_count()
234    pool = multiprocessing.Pool(count)
235    try:
236      results = pool.map_async(CppLintWorker, commands).get(999999)
237    except KeyboardInterrupt:
238      print "\nCaught KeyboardInterrupt, terminating workers."
239      sys.exit(1)
240
241    for i in range(len(files)):
242      if results[i] > 0:
243        good_files_cache.RemoveFile(files[i])
244
245    total_errors = sum(results)
246    print "Total errors found: %d" % total_errors
247    good_files_cache.Save()
248    return total_errors == 0
249
250
251COPYRIGHT_HEADER_PATTERN = re.compile(
252    r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
253
254class SourceProcessor(SourceFileProcessor):
255  """
256  Check that all files include a copyright notice and no trailing whitespaces.
257  """
258
259  RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c',
260                         '.status', '.gyp', '.gypi']
261
262  # Overwriting the one in the parent class.
263  def FindFilesIn(self, path):
264    if os.path.exists(path+'/.git'):
265      output = subprocess.Popen('git ls-files --full-name',
266                                stdout=PIPE, cwd=path, shell=True)
267      result = []
268      for file in output.stdout.read().split():
269        for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
270          if self.IgnoreDir(dir_part):
271            break
272        else:
273          if (self.IsRelevant(file) and os.path.exists(file)
274              and not self.IgnoreFile(file)):
275            result.append(join(path, file))
276      if output.wait() == 0:
277        return result
278    return super(SourceProcessor, self).FindFilesIn(path)
279
280  def IsRelevant(self, name):
281    for ext in SourceProcessor.RELEVANT_EXTENSIONS:
282      if name.endswith(ext):
283        return True
284    return False
285
286  def GetPathsToSearch(self):
287    return ['.']
288
289  def IgnoreDir(self, name):
290    return (super(SourceProcessor, self).IgnoreDir(name) or
291            name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources'))
292
293  IGNORE_COPYRIGHTS = ['box2d.js',
294                       'cpplint.py',
295                       'copy.js',
296                       'corrections.js',
297                       'crypto.js',
298                       'daemon.py',
299                       'earley-boyer.js',
300                       'fannkuch.js',
301                       'fasta.js',
302                       'jsmin.py',
303                       'libraries.cc',
304                       'libraries-empty.cc',
305                       'lua_binarytrees.js',
306                       'memops.js',
307                       'poppler.js',
308                       'primes.js',
309                       'raytrace.js',
310                       'regexp-pcre.js',
311                       'sqlite.js',
312                       'sqlite-change-heap.js',
313                       'sqlite-pointer-masking.js',
314                       'sqlite-safe-heap.js',
315                       'gnuplot-4.6.3-emscripten.js',
316                       'zlib.js']
317  IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
318
319  def EndOfDeclaration(self, line):
320    return line == "}" or line == "};"
321
322  def StartOfDeclaration(self, line):
323    return line.find("//") == 0 or \
324           line.find("/*") == 0 or \
325           line.find(") {") != -1
326
327  def ProcessContents(self, name, contents):
328    result = True
329    base = basename(name)
330    if not base in SourceProcessor.IGNORE_TABS:
331      if '\t' in contents:
332        print "%s contains tabs" % name
333        result = False
334    if not base in SourceProcessor.IGNORE_COPYRIGHTS:
335      if not COPYRIGHT_HEADER_PATTERN.search(contents):
336        print "%s is missing a correct copyright header." % name
337        result = False
338    if ' \n' in contents or contents.endswith(' '):
339      line = 0
340      lines = []
341      parts = contents.split(' \n')
342      if not contents.endswith(' '):
343        parts.pop()
344      for part in parts:
345        line += part.count('\n') + 1
346        lines.append(str(line))
347      linenumbers = ', '.join(lines)
348      if len(lines) > 1:
349        print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
350      else:
351        print "%s has trailing whitespaces in line %s." % (name, linenumbers)
352      result = False
353    if not contents.endswith('\n') or contents.endswith('\n\n'):
354      print "%s does not end with a single new line." % name
355      result = False
356    # Sanitize flags for fuzzer.
357    if "mjsunit" in name:
358      match = FLAGS_LINE.search(contents)
359      if match:
360        print "%s Flags should use '-' (not '_')" % name
361        result = False
362    return result
363
364  def ProcessFiles(self, files, path):
365    success = True
366    violations = 0
367    for file in files:
368      try:
369        handle = open(file)
370        contents = handle.read()
371        if not self.ProcessContents(file, contents):
372          success = False
373          violations += 1
374      finally:
375        handle.close()
376    print "Total violating files: %s" % violations
377    return success
378
379
380def CheckExternalReferenceRegistration(workspace):
381  code = subprocess.call(
382      [sys.executable, join(workspace, "tools", "external-reference-check.py")])
383  return code == 0
384
385
386def _CheckStatusFileForDuplicateKeys(filepath):
387  comma_space_bracket = re.compile(", *]")
388  lines = []
389  with open(filepath) as f:
390    for line in f.readlines():
391      # Skip all-comment lines.
392      if line.lstrip().startswith("#"): continue
393      # Strip away comments at the end of the line.
394      comment_start = line.find("#")
395      if comment_start != -1:
396        line = line[:comment_start]
397      line = line.strip()
398      # Strip away trailing commas within the line.
399      line = comma_space_bracket.sub("]", line)
400      if len(line) > 0:
401        lines.append(line)
402
403  # Strip away trailing commas at line ends. Ugh.
404  for i in range(len(lines) - 1):
405    if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
406        lines[i + 1][0] in ("}", "]")):
407      lines[i] = lines[i][:-1]
408
409  contents = "\n".join(lines)
410  # JSON wants double-quotes.
411  contents = contents.replace("'", '"')
412  # Fill in keywords (like PASS, SKIP).
413  for key in statusfile.KEYWORDS:
414    contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)
415
416  status = {"success": True}
417  def check_pairs(pairs):
418    keys = {}
419    for key, value in pairs:
420      if key in keys:
421        print("%s: Error: duplicate key %s" % (filepath, key))
422        status["success"] = False
423      keys[key] = True
424
425  json.loads(contents, object_pairs_hook=check_pairs)
426  return status["success"]
427
428def CheckStatusFiles(workspace):
429  success = True
430  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
431  for root in suite_paths:
432    suite_path = join(workspace, "test", root)
433    status_file_path = join(suite_path, root + ".status")
434    suite = testsuite.TestSuite.LoadTestSuite(suite_path)
435    if suite and exists(status_file_path):
436      success &= statusfile.PresubmitCheck(status_file_path)
437      success &= _CheckStatusFileForDuplicateKeys(status_file_path)
438  return success
439
440def CheckAuthorizedAuthor(input_api, output_api):
441  """For non-googler/chromites committers, verify the author's email address is
442  in AUTHORS.
443  """
444  # TODO(maruel): Add it to input_api?
445  import fnmatch
446
447  author = input_api.change.author_email
448  if not author:
449    input_api.logging.info('No author, skipping AUTHOR check')
450    return []
451  authors_path = input_api.os_path.join(
452      input_api.PresubmitLocalPath(), 'AUTHORS')
453  valid_authors = (
454      input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line)
455      for line in open(authors_path))
456  valid_authors = [item.group(1).lower() for item in valid_authors if item]
457  if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors):
458    input_api.logging.info('Valid authors are %s', ', '.join(valid_authors))
459    return [output_api.PresubmitPromptWarning(
460        ('%s is not in AUTHORS file. If you are a new contributor, please visit'
461        '\n'
462        'http://www.chromium.org/developers/contributing-code and read the '
463        '"Legal" section\n'
464        'If you are a chromite, verify the contributor signed the CLA.') %
465        author)]
466  return []
467
468def GetOptions():
469  result = optparse.OptionParser()
470  result.add_option('--no-lint', help="Do not run cpplint", default=False,
471                    action="store_true")
472  return result
473
474
475def Main():
476  workspace = abspath(join(dirname(sys.argv[0]), '..'))
477  parser = GetOptions()
478  (options, args) = parser.parse_args()
479  success = True
480  print "Running C++ lint check..."
481  if not options.no_lint:
482    success &= CppLintProcessor().Run(workspace)
483  print "Running copyright header, trailing whitespaces and " \
484        "two empty lines between declarations check..."
485  success &= SourceProcessor().Run(workspace)
486  success &= CheckExternalReferenceRegistration(workspace)
487  success &= CheckStatusFiles(workspace)
488  if success:
489    return 0
490  else:
491    return 1
492
493
494if __name__ == '__main__':
495  sys.exit(Main())
496