1#!/usr/bin/env python 2# 3# Copyright 2012 the V8 project authors. All rights reserved. 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following 12# disclaimer in the documentation and/or other materials provided 13# with the distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived 16# from this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30try: 31 import hashlib 32 md5er = hashlib.md5 33except ImportError, e: 34 import md5 35 md5er = md5.new 36 37 38import json 39import optparse 40import os 41from os.path import abspath, join, dirname, basename, exists 42import pickle 43import re 44import sys 45import subprocess 46import multiprocessing 47from subprocess import PIPE 48 49from testrunner.local import statusfile 50from testrunner.local import testsuite 51from testrunner.local import utils 52 53# Special LINT rules diverging from default and reason. 54# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_". 55# build/include_what_you_use: Started giving false positives for variables 56# named "string" and "map" assuming that you needed to include STL headers. 57# TODO(bmeurer): Fix and re-enable readability/check 58 59LINT_RULES = """ 60-build/header_guard 61-build/include_what_you_use 62-build/namespaces 63-readability/check 64+readability/streams 65-runtime/references 66""".split() 67 68LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing') 69FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n") 70 71def CppLintWorker(command): 72 try: 73 process = subprocess.Popen(command, stderr=subprocess.PIPE) 74 process.wait() 75 out_lines = "" 76 error_count = -1 77 while True: 78 out_line = process.stderr.readline() 79 if out_line == '' and process.poll() != None: 80 if error_count == -1: 81 print "Failed to process %s" % command.pop() 82 return 1 83 break 84 m = LINT_OUTPUT_PATTERN.match(out_line) 85 if m: 86 out_lines += out_line 87 error_count += 1 88 sys.stdout.write(out_lines) 89 return error_count 90 except KeyboardInterrupt: 91 process.kill() 92 except: 93 print('Error running cpplint.py. Please make sure you have depot_tools' + 94 ' in your $PATH. Lint check skipped.') 95 process.kill() 96 97 98class FileContentsCache(object): 99 100 def __init__(self, sums_file_name): 101 self.sums = {} 102 self.sums_file_name = sums_file_name 103 104 def Load(self): 105 try: 106 sums_file = None 107 try: 108 sums_file = open(self.sums_file_name, 'r') 109 self.sums = pickle.load(sums_file) 110 except: 111 # Cannot parse pickle for any reason. Not much we can do about it. 112 pass 113 finally: 114 if sums_file: 115 sums_file.close() 116 117 def Save(self): 118 try: 119 sums_file = open(self.sums_file_name, 'w') 120 pickle.dump(self.sums, sums_file) 121 except: 122 # Failed to write pickle. Try to clean-up behind us. 123 if sums_file: 124 sums_file.close() 125 try: 126 os.unlink(self.sums_file_name) 127 except: 128 pass 129 finally: 130 sums_file.close() 131 132 def FilterUnchangedFiles(self, files): 133 changed_or_new = [] 134 for file in files: 135 try: 136 handle = open(file, "r") 137 file_sum = md5er(handle.read()).digest() 138 if not file in self.sums or self.sums[file] != file_sum: 139 changed_or_new.append(file) 140 self.sums[file] = file_sum 141 finally: 142 handle.close() 143 return changed_or_new 144 145 def RemoveFile(self, file): 146 if file in self.sums: 147 self.sums.pop(file) 148 149 150class SourceFileProcessor(object): 151 """ 152 Utility class that can run through a directory structure, find all relevant 153 files and invoke a custom check on the files. 154 """ 155 156 def Run(self, path): 157 all_files = [] 158 for file in self.GetPathsToSearch(): 159 all_files += self.FindFilesIn(join(path, file)) 160 if not self.ProcessFiles(all_files, path): 161 return False 162 return True 163 164 def IgnoreDir(self, name): 165 return (name.startswith('.') or 166 name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken', 167 'octane', 'sunspider')) 168 169 def IgnoreFile(self, name): 170 return name.startswith('.') 171 172 def FindFilesIn(self, path): 173 result = [] 174 for (root, dirs, files) in os.walk(path): 175 for ignored in [x for x in dirs if self.IgnoreDir(x)]: 176 dirs.remove(ignored) 177 for file in files: 178 if not self.IgnoreFile(file) and self.IsRelevant(file): 179 result.append(join(root, file)) 180 return result 181 182 183class CppLintProcessor(SourceFileProcessor): 184 """ 185 Lint files to check that they follow the google code style. 186 """ 187 188 def IsRelevant(self, name): 189 return name.endswith('.cc') or name.endswith('.h') 190 191 def IgnoreDir(self, name): 192 return (super(CppLintProcessor, self).IgnoreDir(name) 193 or (name == 'third_party')) 194 195 IGNORE_LINT = ['flag-definitions.h'] 196 197 def IgnoreFile(self, name): 198 return (super(CppLintProcessor, self).IgnoreFile(name) 199 or (name in CppLintProcessor.IGNORE_LINT)) 200 201 def GetPathsToSearch(self): 202 return ['src', 'include', 'samples', join('test', 'cctest'), 203 join('test', 'unittests')] 204 205 def GetCpplintScript(self, prio_path): 206 for path in [prio_path] + os.environ["PATH"].split(os.pathsep): 207 path = path.strip('"') 208 cpplint = os.path.join(path, "cpplint.py") 209 if os.path.isfile(cpplint): 210 return cpplint 211 212 return None 213 214 def ProcessFiles(self, files, path): 215 good_files_cache = FileContentsCache('.cpplint-cache') 216 good_files_cache.Load() 217 files = good_files_cache.FilterUnchangedFiles(files) 218 if len(files) == 0: 219 print 'No changes in files detected. Skipping cpplint check.' 220 return True 221 222 filters = ",".join([n for n in LINT_RULES]) 223 command = [sys.executable, 'cpplint.py', '--filter', filters] 224 cpplint = self.GetCpplintScript(join(path, "tools")) 225 if cpplint is None: 226 print('Could not find cpplint.py. Make sure ' 227 'depot_tools is installed and in the path.') 228 sys.exit(1) 229 230 command = [sys.executable, cpplint, '--filter', filters] 231 232 commands = join([command + [file] for file in files]) 233 count = multiprocessing.cpu_count() 234 pool = multiprocessing.Pool(count) 235 try: 236 results = pool.map_async(CppLintWorker, commands).get(999999) 237 except KeyboardInterrupt: 238 print "\nCaught KeyboardInterrupt, terminating workers." 239 sys.exit(1) 240 241 for i in range(len(files)): 242 if results[i] > 0: 243 good_files_cache.RemoveFile(files[i]) 244 245 total_errors = sum(results) 246 print "Total errors found: %d" % total_errors 247 good_files_cache.Save() 248 return total_errors == 0 249 250 251COPYRIGHT_HEADER_PATTERN = re.compile( 252 r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.') 253 254class SourceProcessor(SourceFileProcessor): 255 """ 256 Check that all files include a copyright notice and no trailing whitespaces. 257 """ 258 259 RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', 260 '.status', '.gyp', '.gypi'] 261 262 # Overwriting the one in the parent class. 263 def FindFilesIn(self, path): 264 if os.path.exists(path+'/.git'): 265 output = subprocess.Popen('git ls-files --full-name', 266 stdout=PIPE, cwd=path, shell=True) 267 result = [] 268 for file in output.stdout.read().split(): 269 for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'): 270 if self.IgnoreDir(dir_part): 271 break 272 else: 273 if (self.IsRelevant(file) and os.path.exists(file) 274 and not self.IgnoreFile(file)): 275 result.append(join(path, file)) 276 if output.wait() == 0: 277 return result 278 return super(SourceProcessor, self).FindFilesIn(path) 279 280 def IsRelevant(self, name): 281 for ext in SourceProcessor.RELEVANT_EXTENSIONS: 282 if name.endswith(ext): 283 return True 284 return False 285 286 def GetPathsToSearch(self): 287 return ['.'] 288 289 def IgnoreDir(self, name): 290 return (super(SourceProcessor, self).IgnoreDir(name) or 291 name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources')) 292 293 IGNORE_COPYRIGHTS = ['box2d.js', 294 'cpplint.py', 295 'copy.js', 296 'corrections.js', 297 'crypto.js', 298 'daemon.py', 299 'earley-boyer.js', 300 'fannkuch.js', 301 'fasta.js', 302 'jsmin.py', 303 'libraries.cc', 304 'libraries-empty.cc', 305 'lua_binarytrees.js', 306 'memops.js', 307 'poppler.js', 308 'primes.js', 309 'raytrace.js', 310 'regexp-pcre.js', 311 'sqlite.js', 312 'sqlite-change-heap.js', 313 'sqlite-pointer-masking.js', 314 'sqlite-safe-heap.js', 315 'gnuplot-4.6.3-emscripten.js', 316 'zlib.js'] 317 IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js'] 318 319 def EndOfDeclaration(self, line): 320 return line == "}" or line == "};" 321 322 def StartOfDeclaration(self, line): 323 return line.find("//") == 0 or \ 324 line.find("/*") == 0 or \ 325 line.find(") {") != -1 326 327 def ProcessContents(self, name, contents): 328 result = True 329 base = basename(name) 330 if not base in SourceProcessor.IGNORE_TABS: 331 if '\t' in contents: 332 print "%s contains tabs" % name 333 result = False 334 if not base in SourceProcessor.IGNORE_COPYRIGHTS: 335 if not COPYRIGHT_HEADER_PATTERN.search(contents): 336 print "%s is missing a correct copyright header." % name 337 result = False 338 if ' \n' in contents or contents.endswith(' '): 339 line = 0 340 lines = [] 341 parts = contents.split(' \n') 342 if not contents.endswith(' '): 343 parts.pop() 344 for part in parts: 345 line += part.count('\n') + 1 346 lines.append(str(line)) 347 linenumbers = ', '.join(lines) 348 if len(lines) > 1: 349 print "%s has trailing whitespaces in lines %s." % (name, linenumbers) 350 else: 351 print "%s has trailing whitespaces in line %s." % (name, linenumbers) 352 result = False 353 if not contents.endswith('\n') or contents.endswith('\n\n'): 354 print "%s does not end with a single new line." % name 355 result = False 356 # Sanitize flags for fuzzer. 357 if "mjsunit" in name: 358 match = FLAGS_LINE.search(contents) 359 if match: 360 print "%s Flags should use '-' (not '_')" % name 361 result = False 362 return result 363 364 def ProcessFiles(self, files, path): 365 success = True 366 violations = 0 367 for file in files: 368 try: 369 handle = open(file) 370 contents = handle.read() 371 if not self.ProcessContents(file, contents): 372 success = False 373 violations += 1 374 finally: 375 handle.close() 376 print "Total violating files: %s" % violations 377 return success 378 379 380def CheckExternalReferenceRegistration(workspace): 381 code = subprocess.call( 382 [sys.executable, join(workspace, "tools", "external-reference-check.py")]) 383 return code == 0 384 385 386def _CheckStatusFileForDuplicateKeys(filepath): 387 comma_space_bracket = re.compile(", *]") 388 lines = [] 389 with open(filepath) as f: 390 for line in f.readlines(): 391 # Skip all-comment lines. 392 if line.lstrip().startswith("#"): continue 393 # Strip away comments at the end of the line. 394 comment_start = line.find("#") 395 if comment_start != -1: 396 line = line[:comment_start] 397 line = line.strip() 398 # Strip away trailing commas within the line. 399 line = comma_space_bracket.sub("]", line) 400 if len(line) > 0: 401 lines.append(line) 402 403 # Strip away trailing commas at line ends. Ugh. 404 for i in range(len(lines) - 1): 405 if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and 406 lines[i + 1][0] in ("}", "]")): 407 lines[i] = lines[i][:-1] 408 409 contents = "\n".join(lines) 410 # JSON wants double-quotes. 411 contents = contents.replace("'", '"') 412 # Fill in keywords (like PASS, SKIP). 413 for key in statusfile.KEYWORDS: 414 contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents) 415 416 status = {"success": True} 417 def check_pairs(pairs): 418 keys = {} 419 for key, value in pairs: 420 if key in keys: 421 print("%s: Error: duplicate key %s" % (filepath, key)) 422 status["success"] = False 423 keys[key] = True 424 425 json.loads(contents, object_pairs_hook=check_pairs) 426 return status["success"] 427 428def CheckStatusFiles(workspace): 429 success = True 430 suite_paths = utils.GetSuitePaths(join(workspace, "test")) 431 for root in suite_paths: 432 suite_path = join(workspace, "test", root) 433 status_file_path = join(suite_path, root + ".status") 434 suite = testsuite.TestSuite.LoadTestSuite(suite_path) 435 if suite and exists(status_file_path): 436 success &= statusfile.PresubmitCheck(status_file_path) 437 success &= _CheckStatusFileForDuplicateKeys(status_file_path) 438 return success 439 440def CheckAuthorizedAuthor(input_api, output_api): 441 """For non-googler/chromites committers, verify the author's email address is 442 in AUTHORS. 443 """ 444 # TODO(maruel): Add it to input_api? 445 import fnmatch 446 447 author = input_api.change.author_email 448 if not author: 449 input_api.logging.info('No author, skipping AUTHOR check') 450 return [] 451 authors_path = input_api.os_path.join( 452 input_api.PresubmitLocalPath(), 'AUTHORS') 453 valid_authors = ( 454 input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line) 455 for line in open(authors_path)) 456 valid_authors = [item.group(1).lower() for item in valid_authors if item] 457 if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors): 458 input_api.logging.info('Valid authors are %s', ', '.join(valid_authors)) 459 return [output_api.PresubmitPromptWarning( 460 ('%s is not in AUTHORS file. If you are a new contributor, please visit' 461 '\n' 462 'http://www.chromium.org/developers/contributing-code and read the ' 463 '"Legal" section\n' 464 'If you are a chromite, verify the contributor signed the CLA.') % 465 author)] 466 return [] 467 468def GetOptions(): 469 result = optparse.OptionParser() 470 result.add_option('--no-lint', help="Do not run cpplint", default=False, 471 action="store_true") 472 return result 473 474 475def Main(): 476 workspace = abspath(join(dirname(sys.argv[0]), '..')) 477 parser = GetOptions() 478 (options, args) = parser.parse_args() 479 success = True 480 print "Running C++ lint check..." 481 if not options.no_lint: 482 success &= CppLintProcessor().Run(workspace) 483 print "Running copyright header, trailing whitespaces and " \ 484 "two empty lines between declarations check..." 485 success &= SourceProcessor().Run(workspace) 486 success &= CheckExternalReferenceRegistration(workspace) 487 success &= CheckStatusFiles(workspace) 488 if success: 489 return 0 490 else: 491 return 1 492 493 494if __name__ == '__main__': 495 sys.exit(Main()) 496