199d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh#!/usr/bin/python -u 248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps""" 348fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsWrapper to patch pylint library functions to suit autotest. 448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 548fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsThis script is invoked as part of the presubmit checks for autotest python 648fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsfiles. It runs pylint on a list of files that it obtains either through 748fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsthe command line or from an environment variable set in pre-upload.py. 848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsExample: 1048fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsrun_pylint.py filename.py 1148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps""" 1299d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 13baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport fnmatch 14baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport logging 15baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport os 16baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport re 17baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport sys 1899d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 1998365d89774921bd065fe4bf2d273be7e100af43beepsimport common 2098365d89774921bd065fe4bf2d273be7e100af43beepsfrom autotest_lib.client.common_lib import autotemp, revision_control 2148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 2248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps# Do a basic check to see if pylint is even installed. 2365e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mblightry: 2465e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mbligh import pylint 25861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li from pylint.__pkginfo__ import version as pylint_version 2665e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mblighexcept ImportError: 2748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps print ("Unable to import pylint, it may need to be installed." 2848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps " Run 'sudo aptitude install pylint' if you haven't already.") 2965e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mbligh sys.exit(1) 3065e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mbligh 31861b2d54aec24228cdb3895dbc40062cb40cb2adEric Limajor, minor, release = pylint_version.split('.') 32861b2d54aec24228cdb3895dbc40062cb40cb2adEric Lipylint_version = float("%s.%s" % (major, minor)) 3399d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 3448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps# some files make pylint blow up, so make sure we ignore them 355dab08c529d9a914665cea9c542b51765e264bf5Prathmesh PrabhuBLACKLIST = ['/site-packages/*', '/contrib/*', '/frontend/afe/management.py'] 3694a6493803ec2e113c96677dda995603dac4863ajadmanski 3794a6493803ec2e113c96677dda995603dac4863ajadmanski# patch up the logilab module lookup tools to understand autotest_lib.* trash 3894a6493803ec2e113c96677dda995603dac4863ajadmanskiimport logilab.common.modutils 3994a6493803ec2e113c96677dda995603dac4863ajadmanski_ffm = logilab.common.modutils.file_from_modpath 4094a6493803ec2e113c96677dda995603dac4863ajadmanskidef file_from_modpath(modpath, path=None, context_file=None): 4148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 4248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Wrapper to eliminate autotest_lib from modpath. 4348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 4448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param modpath: name of module splitted on '.' 4548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param path: optional list of paths where module should be searched for. 4648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param context_file: path to file doing the importing. 4748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @return The path to the module as returned by the parent method invocation. 4848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @raises: ImportError if these is no such module. 4948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 5094a6493803ec2e113c96677dda995603dac4863ajadmanski if modpath[0] == "autotest_lib": 5194a6493803ec2e113c96677dda995603dac4863ajadmanski return _ffm(modpath[1:], path, context_file) 5294a6493803ec2e113c96677dda995603dac4863ajadmanski else: 5394a6493803ec2e113c96677dda995603dac4863ajadmanski return _ffm(modpath, path, context_file) 5494a6493803ec2e113c96677dda995603dac4863ajadmanskilogilab.common.modutils.file_from_modpath = file_from_modpath 5594a6493803ec2e113c96677dda995603dac4863ajadmanski 5694a6493803ec2e113c96677dda995603dac4863ajadmanski 5799d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighimport pylint.lint 582c6696426c57941c1b37331b28fb594e2a0beab7beepsfrom pylint.checkers import base, imports, variables 5999d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 6099d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh# need to put autotest root dir on sys.path so pylint will be happy 6199d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighautotest_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 6299d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighsys.path.insert(0, autotest_root) 6399d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 6499d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh# patch up pylint import checker to handle our importing magic 6548fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsROOT_MODULE = 'autotest_lib.' 66e19d303a6718cbd9995e27485554212b63a0a701beeps 67e19d303a6718cbd9995e27485554212b63a0a701beeps# A list of modules for pylint to ignore, specifically, these modules 68e19d303a6718cbd9995e27485554212b63a0a701beeps# are imported for their side-effects and are not meant to be used. 692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B_IGNORE_MODULES=['common', 'frontend_test_utils', 702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 'setup_django_environment', 71f9a36511e83c4ebeaa6192ca7219895597284681Keyar Hood 'setup_django_lite_environment', 722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 'setup_django_readonly_environment', 'setup_test_environment',] 7348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 7448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 7598365d89774921bd065fe4bf2d273be7e100af43beepsclass pylint_error(Exception): 7698365d89774921bd065fe4bf2d273be7e100af43beeps """ 7798365d89774921bd065fe4bf2d273be7e100af43beeps Error raised when pylint complains about a file. 7898365d89774921bd065fe4bf2d273be7e100af43beeps """ 7998365d89774921bd065fe4bf2d273be7e100af43beeps 8098365d89774921bd065fe4bf2d273be7e100af43beeps 8198365d89774921bd065fe4bf2d273be7e100af43beepsclass run_pylint_error(pylint_error): 8298365d89774921bd065fe4bf2d273be7e100af43beeps """ 8398365d89774921bd065fe4bf2d273be7e100af43beeps Error raised when an assumption made in this file is violated. 8498365d89774921bd065fe4bf2d273be7e100af43beeps """ 8598365d89774921bd065fe4bf2d273be7e100af43beeps 8698365d89774921bd065fe4bf2d273be7e100af43beeps 8748fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef patch_modname(modname): 8848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 8948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Patches modname so we can make sense of autotest_lib modules. 9048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 9148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param modname: name of a module, contains '.' 9248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @return modified modname string. 9348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 9448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps if modname.startswith(ROOT_MODULE) or modname.startswith(ROOT_MODULE[:-1]): 9548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps modname = modname[len(ROOT_MODULE):] 9648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps return modname 9748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 9848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 9948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef patch_consumed_list(to_consume=None, consumed=None): 10048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 101e19d303a6718cbd9995e27485554212b63a0a701beeps Patches the consumed modules list to ignore modules with side effects. 10248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 103e19d303a6718cbd9995e27485554212b63a0a701beeps Autotest relies on importing certain modules solely for their side 104e19d303a6718cbd9995e27485554212b63a0a701beeps effects. Pylint doesn't understand this and flags them as unused, since 105e19d303a6718cbd9995e27485554212b63a0a701beeps they're not referenced anywhere in the code. To overcome this we need 106e19d303a6718cbd9995e27485554212b63a0a701beeps to transplant said modules into the dictionary of modules pylint has 107e19d303a6718cbd9995e27485554212b63a0a701beeps already seen, before pylint checks it. 10848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 10948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param to_consume: a dictionary of names pylint needs to see referenced. 11048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param consumed: a dictionary of names that pylint has seen referenced. 11148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 112e19d303a6718cbd9995e27485554212b63a0a701beeps ignore_modules = [] 113e19d303a6718cbd9995e27485554212b63a0a701beeps if (to_consume is not None and consumed is not None): 114e19d303a6718cbd9995e27485554212b63a0a701beeps ignore_modules = [module_name for module_name in _IGNORE_MODULES 115e19d303a6718cbd9995e27485554212b63a0a701beeps if module_name in to_consume] 116e19d303a6718cbd9995e27485554212b63a0a701beeps 117e19d303a6718cbd9995e27485554212b63a0a701beeps for module_name in ignore_modules: 118e19d303a6718cbd9995e27485554212b63a0a701beeps consumed[module_name] = to_consume[module_name] 119e19d303a6718cbd9995e27485554212b63a0a701beeps del to_consume[module_name] 12048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 12199d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 12299d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighclass CustomImportsChecker(imports.ImportsChecker): 12348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """Modifies stock imports checker to suit autotest.""" 12448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps def visit_from(self, node): 12548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps node.modname = patch_modname(node.modname) 12648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps return super(CustomImportsChecker, self).visit_from(node) 12748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 12848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 12948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsclass CustomVariablesChecker(variables.VariablesChecker): 13048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """Modifies stock variables checker to suit autotest.""" 13148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 13248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps def visit_module(self, node): 13348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 13448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Unflag 'import common'. 13548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 13648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps _to_consume eg: [({to reference}, {referenced}, 'scope type')] 13748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Enteries are appended to this list as we drill deeper in scope. 138e19d303a6718cbd9995e27485554212b63a0a701beeps If we ever come across a module to ignore, we immediately move it 13948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps to the consumed list. 14048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 14148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param node: node of the ast we're currently checking. 14248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 14348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps super(CustomVariablesChecker, self).visit_module(node) 14448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps scoped_names = self._to_consume.pop() 14548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps patch_consumed_list(scoped_names[0],scoped_names[1]) 14648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps self._to_consume.append(scoped_names) 14748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 14899d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh def visit_from(self, node): 14948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """Patches modnames so pylints understands autotest_lib.""" 15048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps node.modname = patch_modname(node.modname) 15148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps return super(CustomVariablesChecker, self).visit_from(node) 15299d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 1532c6696426c57941c1b37331b28fb594e2a0beab7beeps 1542c6696426c57941c1b37331b28fb594e2a0beab7beepsclass CustomDocStringChecker(base.DocStringChecker): 1552c6696426c57941c1b37331b28fb594e2a0beab7beeps """Modifies stock docstring checker to suit Autotest doxygen style.""" 1562c6696426c57941c1b37331b28fb594e2a0beab7beeps 1571b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps def visit_module(self, node): 1581b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps """ 1591b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps Don't visit imported modules when checking for docstrings. 1601b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps 1611b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps @param node: the node we're visiting. 1621b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps """ 1631b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps pass 1641b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps 1651b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps 16698365d89774921bd065fe4bf2d273be7e100af43beeps def visit_function(self, node): 16798365d89774921bd065fe4bf2d273be7e100af43beeps """ 16898365d89774921bd065fe4bf2d273be7e100af43beeps Don't request docstrings for commonly overridden autotest functions. 16998365d89774921bd065fe4bf2d273be7e100af43beeps 17098365d89774921bd065fe4bf2d273be7e100af43beeps @param node: node of the ast we're currently checking. 17198365d89774921bd065fe4bf2d273be7e100af43beeps """ 172c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps 173c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps # Even plain functions will have a parent, which is the 174c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps # module they're in, and a frame, which is the context 175c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps # of said module; They need not however, always have 176c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps # ancestors. 17798365d89774921bd065fe4bf2d273be7e100af43beeps if (node.name in ('run_once', 'initialize', 'cleanup') and 178c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps hasattr(node.parent.frame(), 'ancestors') and 17998365d89774921bd065fe4bf2d273be7e100af43beeps any(ancestor.name == 'base_test' for ancestor in 18098365d89774921bd065fe4bf2d273be7e100af43beeps node.parent.frame().ancestors())): 18198365d89774921bd065fe4bf2d273be7e100af43beeps return 18298365d89774921bd065fe4bf2d273be7e100af43beeps 18398365d89774921bd065fe4bf2d273be7e100af43beeps super(CustomDocStringChecker, self).visit_function(node) 18498365d89774921bd065fe4bf2d273be7e100af43beeps 18598365d89774921bd065fe4bf2d273be7e100af43beeps 186ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa @staticmethod 187ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa def _should_skip_arg(arg): 18898365d89774921bd065fe4bf2d273be7e100af43beeps """ 18998365d89774921bd065fe4bf2d273be7e100af43beeps @return: True if the argument given by arg is whitelisted, and does 19098365d89774921bd065fe4bf2d273be7e100af43beeps not require a "@param" docstring. 19198365d89774921bd065fe4bf2d273be7e100af43beeps """ 192ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa return arg in ('self', 'cls', 'args', 'kwargs', 'dargs') 193ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa 194ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa 1952c6696426c57941c1b37331b28fb594e2a0beab7beeps def _check_docstring(self, node_type, node): 1962c6696426c57941c1b37331b28fb594e2a0beab7beeps """ 1972c6696426c57941c1b37331b28fb594e2a0beab7beeps Teaches pylint to look for @param with each argument in the 1982c6696426c57941c1b37331b28fb594e2a0beab7beeps function/method signature. 1992c6696426c57941c1b37331b28fb594e2a0beab7beeps 2002c6696426c57941c1b37331b28fb594e2a0beab7beeps @param node_type: type of the node we're currently checking. 2012c6696426c57941c1b37331b28fb594e2a0beab7beeps @param node: node of the ast we're currently checking. 2022c6696426c57941c1b37331b28fb594e2a0beab7beeps """ 2032c6696426c57941c1b37331b28fb594e2a0beab7beeps super(CustomDocStringChecker, self)._check_docstring(node_type, node) 2042c6696426c57941c1b37331b28fb594e2a0beab7beeps docstring = node.doc 205a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi if pylint_version >= 1.1: 206a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi key = 'missing-docstring' 207a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi else: 208a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi key = 'C0111' 209a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi 2102c6696426c57941c1b37331b28fb594e2a0beab7beeps if (docstring is not None and 2112c6696426c57941c1b37331b28fb594e2a0beab7beeps (node_type is 'method' or 2122c6696426c57941c1b37331b28fb594e2a0beab7beeps node_type is 'function')): 2132c6696426c57941c1b37331b28fb594e2a0beab7beeps args = node.argnames() 214a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi old_msg = self.linter._messages[key].msg 2152c6696426c57941c1b37331b28fb594e2a0beab7beeps for arg in args: 2162c6696426c57941c1b37331b28fb594e2a0beab7beeps arg_docstring_rgx = '.*@param '+arg+'.*' 2172c6696426c57941c1b37331b28fb594e2a0beab7beeps line = re.search(arg_docstring_rgx, node.doc) 218ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa if not line and not self._should_skip_arg(arg): 219a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi self.linter._messages[key].msg = ('Docstring needs ' 220a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi '"@param '+arg+':"') 221a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi self.add_message(key, node=node) 222a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi self.linter._messages[key].msg = old_msg 2232c6696426c57941c1b37331b28fb594e2a0beab7beeps 2242c6696426c57941c1b37331b28fb594e2a0beab7beepsbase.DocStringChecker = CustomDocStringChecker 22599d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighimports.ImportsChecker = CustomImportsChecker 22648fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsvariables.VariablesChecker = CustomVariablesChecker 22799d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 22848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 22998365d89774921bd065fe4bf2d273be7e100af43beepsdef batch_check_files(file_paths, base_opts): 23098365d89774921bd065fe4bf2d273be7e100af43beeps """ 23198365d89774921bd065fe4bf2d273be7e100af43beeps Run pylint on a list of files so we get consolidated errors. 23298365d89774921bd065fe4bf2d273be7e100af43beeps 23398365d89774921bd065fe4bf2d273be7e100af43beeps @param file_paths: a list of file paths. 23498365d89774921bd065fe4bf2d273be7e100af43beeps @param base_opts: a list of pylint config options. 23598365d89774921bd065fe4bf2d273be7e100af43beeps 23698365d89774921bd065fe4bf2d273be7e100af43beeps @raises: pylint_error if pylint finds problems with a file 23798365d89774921bd065fe4bf2d273be7e100af43beeps in this commit. 23898365d89774921bd065fe4bf2d273be7e100af43beeps """ 23912a3c882b514db90fcadfd1e11d1af214d1e664abeeps if not file_paths: 24012a3c882b514db90fcadfd1e11d1af214d1e664abeeps return 24112a3c882b514db90fcadfd1e11d1af214d1e664abeeps 24298365d89774921bd065fe4bf2d273be7e100af43beeps pylint_runner = pylint.lint.Run(list(base_opts) + list(file_paths), 24398365d89774921bd065fe4bf2d273be7e100af43beeps exit=False) 24498365d89774921bd065fe4bf2d273be7e100af43beeps if pylint_runner.linter.msg_status: 24598365d89774921bd065fe4bf2d273be7e100af43beeps raise pylint_error(pylint_runner.linter.msg_status) 24698365d89774921bd065fe4bf2d273be7e100af43beeps 24798365d89774921bd065fe4bf2d273be7e100af43beeps 24898365d89774921bd065fe4bf2d273be7e100af43beepsdef should_check_file(file_path): 24998365d89774921bd065fe4bf2d273be7e100af43beeps """ 25098365d89774921bd065fe4bf2d273be7e100af43beeps Don't check blacklisted or non .py files. 25198365d89774921bd065fe4bf2d273be7e100af43beeps 25298365d89774921bd065fe4bf2d273be7e100af43beeps @param file_path: abs path of file to check. 25398365d89774921bd065fe4bf2d273be7e100af43beeps @return: True if this file is a non-blacklisted python file. 25498365d89774921bd065fe4bf2d273be7e100af43beeps """ 25598365d89774921bd065fe4bf2d273be7e100af43beeps file_path = os.path.abspath(file_path) 25698365d89774921bd065fe4bf2d273be7e100af43beeps if file_path.endswith('.py'): 25798365d89774921bd065fe4bf2d273be7e100af43beeps return all(not fnmatch.fnmatch(file_path, '*' + pattern) 25898365d89774921bd065fe4bf2d273be7e100af43beeps for pattern in BLACKLIST) 25998365d89774921bd065fe4bf2d273be7e100af43beeps return False 26098365d89774921bd065fe4bf2d273be7e100af43beeps 26198365d89774921bd065fe4bf2d273be7e100af43beeps 26248fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef check_file(file_path, base_opts): 26348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 26448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Invokes pylint on files after confirming that they're not black listed. 26548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 2662c6696426c57941c1b37331b28fb594e2a0beab7beeps @param base_opts: pylint base options. 26748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param file_path: path to the file we need to run pylint on. 26848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 26998365d89774921bd065fe4bf2d273be7e100af43beeps if not isinstance(file_path, basestring): 27098365d89774921bd065fe4bf2d273be7e100af43beeps raise TypeError('expected a string as filepath, got %s'% 27198365d89774921bd065fe4bf2d273be7e100af43beeps type(file_path)) 27298365d89774921bd065fe4bf2d273be7e100af43beeps 27398365d89774921bd065fe4bf2d273be7e100af43beeps if should_check_file(file_path): 27498365d89774921bd065fe4bf2d273be7e100af43beeps pylint_runner = pylint.lint.Run(base_opts + [file_path], exit=False) 27598365d89774921bd065fe4bf2d273be7e100af43beeps if pylint_runner.linter.msg_status: 27698365d89774921bd065fe4bf2d273be7e100af43beeps pylint_error(pylint_runner.linter.msg_status) 27799d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 27899d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 27999d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighdef visit(arg, dirname, filenames): 28048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 28148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Visit function invoked in check_dir. 28248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 28348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param arg: arg from os.walk.path 28448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param dirname: dir from os.walk.path 28548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param filenames: files in dir from os.walk.path 28648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 28778ece92fc08f20e9d4f66c22b0bf034fe85ce9c2beeps for filename in filenames: 28848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps check_file(os.path.join(dirname, filename), arg) 28948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 29048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 29148fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef check_dir(dir_path, base_opts): 29248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 29348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Calls visit on files in dir_path. 29448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 2952c6696426c57941c1b37331b28fb594e2a0beab7beeps @param base_opts: pylint base options. 29648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param dir_path: path to directory. 29748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 29848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps os.path.walk(dir_path, visit, base_opts) 29948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 30048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 30148fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef extend_baseopts(base_opts, new_opt): 30248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 30348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Replaces an argument in base_opts with a cmd line argument. 30448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 30548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param base_opts: original pylint_base_opts. 30648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param new_opt: new cmd line option. 30748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 30848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps for args in base_opts: 30948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps if new_opt in args: 31048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps base_opts.remove(args) 31148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps base_opts.append(new_opt) 31299d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 31399d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 31448fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef get_cmdline_options(args_list, pylint_base_opts, rcfile): 31548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 31648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Parses args_list and extends pylint_base_opts. 31799d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh 31848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Command line arguments might include options mixed with files. 31948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps Go through this list and filter out the options, if the options are 32048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps specified in the pylintrc file we cannot replace them and the file 32148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps needs to be edited. If the options are already a part of 32248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps pylint_base_opts we replace them, and if not we append to 32348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps pylint_base_opts. 32440f4744243db135888488d22d14277b739f4acf8Alex Miller 32548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param args_list: list of files/pylint args passed in through argv. 32648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param pylint_base_opts: default pylint options. 32748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps @param rcfile: text from pylint_rc. 32848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """ 32948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps for args in args_list: 33048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps if args.startswith('--'): 33148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps opt_name = args[2:].split('=')[0] 33248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps if opt_name in rcfile and pylint_version >= 0.21: 33398365d89774921bd065fe4bf2d273be7e100af43beeps raise run_pylint_error('The rcfile already contains the %s ' 33498365d89774921bd065fe4bf2d273be7e100af43beeps 'option. Please edit pylintrc instead.' 33598365d89774921bd065fe4bf2d273be7e100af43beeps % opt_name) 33648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps else: 33748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps extend_baseopts(pylint_base_opts, args) 33848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps args_list.remove(args) 33948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 34098365d89774921bd065fe4bf2d273be7e100af43beeps 34198365d89774921bd065fe4bf2d273be7e100af43beepsdef git_show_to_temp_file(commit, original_file, new_temp_file): 34298365d89774921bd065fe4bf2d273be7e100af43beeps """ 34398365d89774921bd065fe4bf2d273be7e100af43beeps 'Git shows' the file in original_file to a tmp file with 34498365d89774921bd065fe4bf2d273be7e100af43beeps the name new_temp_file. We need to preserve the filename 34598365d89774921bd065fe4bf2d273be7e100af43beeps as it gets reflected in pylints error report. 34698365d89774921bd065fe4bf2d273be7e100af43beeps 34798365d89774921bd065fe4bf2d273be7e100af43beeps @param commit: commit hash of the commit we're running repo upload on. 34898365d89774921bd065fe4bf2d273be7e100af43beeps @param original_file: the path to the original file we'd like to run 34998365d89774921bd065fe4bf2d273be7e100af43beeps 'git show' on. 35098365d89774921bd065fe4bf2d273be7e100af43beeps @param new_temp_file: new_temp_file is the path to a temp file we write the 35198365d89774921bd065fe4bf2d273be7e100af43beeps output of 'git show' into. 35298365d89774921bd065fe4bf2d273be7e100af43beeps """ 35398365d89774921bd065fe4bf2d273be7e100af43beeps git_repo = revision_control.GitRepo(common.autotest_dir, None, None, 35498365d89774921bd065fe4bf2d273be7e100af43beeps common.autotest_dir) 35598365d89774921bd065fe4bf2d273be7e100af43beeps 35698365d89774921bd065fe4bf2d273be7e100af43beeps with open(new_temp_file, 'w') as f: 35798365d89774921bd065fe4bf2d273be7e100af43beeps output = git_repo.gitcmd('show --no-ext-diff %s:%s' 35898365d89774921bd065fe4bf2d273be7e100af43beeps % (commit, original_file), 35998365d89774921bd065fe4bf2d273be7e100af43beeps ignore_status=False).stdout 36098365d89774921bd065fe4bf2d273be7e100af43beeps f.write(output) 36198365d89774921bd065fe4bf2d273be7e100af43beeps 36298365d89774921bd065fe4bf2d273be7e100af43beeps 36398365d89774921bd065fe4bf2d273be7e100af43beepsdef check_committed_files(work_tree_files, commit, pylint_base_opts): 36498365d89774921bd065fe4bf2d273be7e100af43beeps """ 36598365d89774921bd065fe4bf2d273be7e100af43beeps Get a list of files corresponding to the commit hash. 36698365d89774921bd065fe4bf2d273be7e100af43beeps 36798365d89774921bd065fe4bf2d273be7e100af43beeps The contents of a file in the git work tree can differ from the contents 36898365d89774921bd065fe4bf2d273be7e100af43beeps of a file in the commit we mean to upload. To work around this we run 36998365d89774921bd065fe4bf2d273be7e100af43beeps pylint on a temp file into which we've 'git show'n the committed version 37098365d89774921bd065fe4bf2d273be7e100af43beeps of each file. 37198365d89774921bd065fe4bf2d273be7e100af43beeps 37298365d89774921bd065fe4bf2d273be7e100af43beeps @param work_tree_files: list of files in this commit specified by their 37398365d89774921bd065fe4bf2d273be7e100af43beeps absolute path. 37498365d89774921bd065fe4bf2d273be7e100af43beeps @param commit: hash of the commit this upload applies to. 37598365d89774921bd065fe4bf2d273be7e100af43beeps @param pylint_base_opts: a list of pylint config options. 37698365d89774921bd065fe4bf2d273be7e100af43beeps """ 37798365d89774921bd065fe4bf2d273be7e100af43beeps files_to_check = filter(should_check_file, work_tree_files) 37898365d89774921bd065fe4bf2d273be7e100af43beeps 37998365d89774921bd065fe4bf2d273be7e100af43beeps # Map the absolute path of each file so it's relative to the autotest repo. 38098365d89774921bd065fe4bf2d273be7e100af43beeps # All files that are a part of this commit should have an abs path within 38198365d89774921bd065fe4bf2d273be7e100af43beeps # the autotest repo, so this regex should never fail. 38298365d89774921bd065fe4bf2d273be7e100af43beeps work_tree_files = [re.search(r'%s/(.*)' % common.autotest_dir, f).group(1) 38398365d89774921bd065fe4bf2d273be7e100af43beeps for f in files_to_check] 38498365d89774921bd065fe4bf2d273be7e100af43beeps 38598365d89774921bd065fe4bf2d273be7e100af43beeps tempdir = None 38698365d89774921bd065fe4bf2d273be7e100af43beeps try: 38798365d89774921bd065fe4bf2d273be7e100af43beeps tempdir = autotemp.tempdir() 38898365d89774921bd065fe4bf2d273be7e100af43beeps temp_files = [os.path.join(tempdir.name, file_path.split('/')[-1:][0]) 38998365d89774921bd065fe4bf2d273be7e100af43beeps for file_path in work_tree_files] 39098365d89774921bd065fe4bf2d273be7e100af43beeps 39198365d89774921bd065fe4bf2d273be7e100af43beeps for file_tuple in zip(work_tree_files, temp_files): 39298365d89774921bd065fe4bf2d273be7e100af43beeps git_show_to_temp_file(commit, *file_tuple) 39398365d89774921bd065fe4bf2d273be7e100af43beeps # Only check if we successfully git showed all files in the commit. 39498365d89774921bd065fe4bf2d273be7e100af43beeps batch_check_files(temp_files, pylint_base_opts) 39598365d89774921bd065fe4bf2d273be7e100af43beeps finally: 39698365d89774921bd065fe4bf2d273be7e100af43beeps if tempdir: 39798365d89774921bd065fe4bf2d273be7e100af43beeps tempdir.clean() 39898365d89774921bd065fe4bf2d273be7e100af43beeps 39998365d89774921bd065fe4bf2d273be7e100af43beeps 40048fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef main(): 40148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps """Main function checks each file in a commit for pylint violations.""" 40248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 4031b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # For now all error/warning/refactor/convention exceptions except those in 4041b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # the enable string are disabled. 4051b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # W0611: All imported modules (except common) need to be used. 4061b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # W1201: Logging methods should take the form 4071b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # logging.<loggingmethod>(format_string, format_args...); and not 4081b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # logging.<loggingmethod>(format_string % (format_args...)) 4091b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # C0111: Docstring needed. Also checks @param for each arg. 4101b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps # C0112: Non-empty Docstring needed. 41148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # Ideally we would like to enable as much as we can, but if we did so at 41248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # this stage anyone who makes a tiny change to a file will be tasked with 41348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # cleaning all the lint in it. See chromium-os:37364. 41448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 415e19d303a6718cbd9995e27485554212b63a0a701beeps # Note: 416e19d303a6718cbd9995e27485554212b63a0a701beeps # 1. There are three major sources of E1101/E1103/E1120 false positives: 417e19d303a6718cbd9995e27485554212b63a0a701beeps # * common_lib.enum.Enum objects 418e19d303a6718cbd9995e27485554212b63a0a701beeps # * DB model objects (scheduler models are the worst, but Django models 419e19d303a6718cbd9995e27485554212b63a0a701beeps # also generate some errors) 420e19d303a6718cbd9995e27485554212b63a0a701beeps # 2. Docstrings are optional on private methods, and any methods that begin 421e19d303a6718cbd9995e27485554212b63a0a701beeps # with either 'set_' or 'get_'. 42248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps pylint_rc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 42348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 'pylintrc') 424e19d303a6718cbd9995e27485554212b63a0a701beeps 425e19d303a6718cbd9995e27485554212b63a0a701beeps no_docstring_rgx = r'((_.*)|(set_.*)|(get_.*))' 42648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps if pylint_version >= 0.21: 42748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps pylint_base_opts = ['--rcfile=%s' % pylint_rc, 42848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps '--reports=no', 4295553256ffaaa15b1bcaf92d58c14c06243c609fcbeeps '--disable=W,R,E,C,F', 4305e2bb4aa28611aaacaa8798fd07943ede1df46c6beeps '--enable=W0611,W1201,C0111,C0112,E0602,W0601', 431e19d303a6718cbd9995e27485554212b63a0a701beeps '--no-docstring-rgx=%s' % no_docstring_rgx,] 43248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps else: 43348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps all_failures = 'error,warning,refactor,convention' 43448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps pylint_base_opts = ['--disable-msg-cat=%s' % all_failures, 43548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps '--reports=no', 43648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps '--include-ids=y', 4371b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps '--ignore-docstrings=n', 438e19d303a6718cbd9995e27485554212b63a0a701beeps '--no-docstring-rgx=%s' % no_docstring_rgx,] 43948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 44048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # run_pylint can be invoked directly with command line arguments, 44148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # or through a presubmit hook which uses the arguments in pylintrc. In the 44248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # latter case no command line arguments are passed. If it is invoked 44348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps # directly without any arguments, it should check all files in the cwd. 44448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps args_list = sys.argv[1:] 44548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps if args_list: 44648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps get_cmdline_options(args_list, 44748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps pylint_base_opts, 44848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps open(pylint_rc).read()) 44998365d89774921bd065fe4bf2d273be7e100af43beeps batch_check_files(args_list, pylint_base_opts) 45098365d89774921bd065fe4bf2d273be7e100af43beeps elif os.environ.get('PRESUBMIT_FILES') is not None: 45198365d89774921bd065fe4bf2d273be7e100af43beeps check_committed_files( 45298365d89774921bd065fe4bf2d273be7e100af43beeps os.environ.get('PRESUBMIT_FILES').split('\n'), 45398365d89774921bd065fe4bf2d273be7e100af43beeps os.environ.get('PRESUBMIT_COMMIT'), 45498365d89774921bd065fe4bf2d273be7e100af43beeps pylint_base_opts) 45548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps else: 45698365d89774921bd065fe4bf2d273be7e100af43beeps check_dir('.', pylint_base_opts) 45748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 45848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps 45948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsif __name__ == '__main__': 460baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian try: 461baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian main() 4627e10708628c1bd21d6a08adab3913c2275204df9Prathmesh Prabhu except pylint_error as e: 463baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian logging.error(e) 464baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian sys.exit(1) 465