199d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh#!/usr/bin/python -u
248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps"""
348fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsWrapper to patch pylint library functions to suit autotest.
448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
548fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsThis script is invoked as part of the presubmit checks for autotest python
648fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsfiles. It runs pylint on a list of files that it obtains either through
748fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsthe command line or from an environment variable set in pre-upload.py.
848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsExample:
1048fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsrun_pylint.py filename.py
1148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps"""
1299d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
13baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport fnmatch
14baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport logging
15baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport os
16baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport re
17baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanianimport sys
1899d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
1998365d89774921bd065fe4bf2d273be7e100af43beepsimport common
2098365d89774921bd065fe4bf2d273be7e100af43beepsfrom autotest_lib.client.common_lib import autotemp, revision_control
2148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
2248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps# Do a basic check to see if pylint is even installed.
2365e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mblightry:
2465e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mbligh    import pylint
25861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li    from pylint.__pkginfo__ import version as pylint_version
2665e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mblighexcept ImportError:
2748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    print ("Unable to import pylint, it may need to be installed."
2848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps           " Run 'sudo aptitude install pylint' if you haven't already.")
2965e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mbligh    sys.exit(1)
3065e06b1f47e07a73ee5bf3f9ace0cc3a9d859dd3mbligh
31861b2d54aec24228cdb3895dbc40062cb40cb2adEric Limajor, minor, release = pylint_version.split('.')
32861b2d54aec24228cdb3895dbc40062cb40cb2adEric Lipylint_version = float("%s.%s" % (major, minor))
3399d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
3448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps# some files make pylint blow up, so make sure we ignore them
355dab08c529d9a914665cea9c542b51765e264bf5Prathmesh PrabhuBLACKLIST = ['/site-packages/*', '/contrib/*', '/frontend/afe/management.py']
3694a6493803ec2e113c96677dda995603dac4863ajadmanski
3794a6493803ec2e113c96677dda995603dac4863ajadmanski# patch up the logilab module lookup tools to understand autotest_lib.* trash
3894a6493803ec2e113c96677dda995603dac4863ajadmanskiimport logilab.common.modutils
3994a6493803ec2e113c96677dda995603dac4863ajadmanski_ffm = logilab.common.modutils.file_from_modpath
4094a6493803ec2e113c96677dda995603dac4863ajadmanskidef file_from_modpath(modpath, path=None, context_file=None):
4148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
4248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Wrapper to eliminate autotest_lib from modpath.
4348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
4448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param modpath: name of module splitted on '.'
4548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param path: optional list of paths where module should be searched for.
4648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param context_file: path to file doing the importing.
4748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @return The path to the module as returned by the parent method invocation.
4848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @raises: ImportError if these is no such module.
4948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
5094a6493803ec2e113c96677dda995603dac4863ajadmanski    if modpath[0] == "autotest_lib":
5194a6493803ec2e113c96677dda995603dac4863ajadmanski        return _ffm(modpath[1:], path, context_file)
5294a6493803ec2e113c96677dda995603dac4863ajadmanski    else:
5394a6493803ec2e113c96677dda995603dac4863ajadmanski        return _ffm(modpath, path, context_file)
5494a6493803ec2e113c96677dda995603dac4863ajadmanskilogilab.common.modutils.file_from_modpath = file_from_modpath
5594a6493803ec2e113c96677dda995603dac4863ajadmanski
5694a6493803ec2e113c96677dda995603dac4863ajadmanski
5799d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighimport pylint.lint
582c6696426c57941c1b37331b28fb594e2a0beab7beepsfrom pylint.checkers import base, imports, variables
5999d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
6099d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh# need to put autotest root dir on sys.path so pylint will be happy
6199d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighautotest_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
6299d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighsys.path.insert(0, autotest_root)
6399d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
6499d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh# patch up pylint import checker to handle our importing magic
6548fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsROOT_MODULE = 'autotest_lib.'
66e19d303a6718cbd9995e27485554212b63a0a701beeps
67e19d303a6718cbd9995e27485554212b63a0a701beeps# A list of modules for pylint to ignore, specifically, these modules
68e19d303a6718cbd9995e27485554212b63a0a701beeps# are imported for their side-effects and are not meant to be used.
692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B_IGNORE_MODULES=['common', 'frontend_test_utils',
702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B                 'setup_django_environment',
71f9a36511e83c4ebeaa6192ca7219895597284681Keyar Hood                 'setup_django_lite_environment',
722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B                 'setup_django_readonly_environment', 'setup_test_environment',]
7348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
7448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
7598365d89774921bd065fe4bf2d273be7e100af43beepsclass pylint_error(Exception):
7698365d89774921bd065fe4bf2d273be7e100af43beeps    """
7798365d89774921bd065fe4bf2d273be7e100af43beeps    Error raised when pylint complains about a file.
7898365d89774921bd065fe4bf2d273be7e100af43beeps    """
7998365d89774921bd065fe4bf2d273be7e100af43beeps
8098365d89774921bd065fe4bf2d273be7e100af43beeps
8198365d89774921bd065fe4bf2d273be7e100af43beepsclass run_pylint_error(pylint_error):
8298365d89774921bd065fe4bf2d273be7e100af43beeps    """
8398365d89774921bd065fe4bf2d273be7e100af43beeps    Error raised when an assumption made in this file is violated.
8498365d89774921bd065fe4bf2d273be7e100af43beeps    """
8598365d89774921bd065fe4bf2d273be7e100af43beeps
8698365d89774921bd065fe4bf2d273be7e100af43beeps
8748fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef patch_modname(modname):
8848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
8948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Patches modname so we can make sense of autotest_lib modules.
9048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
9148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param modname: name of a module, contains '.'
9248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @return modified modname string.
9348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
9448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    if modname.startswith(ROOT_MODULE) or modname.startswith(ROOT_MODULE[:-1]):
9548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        modname = modname[len(ROOT_MODULE):]
9648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    return modname
9748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
9848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
9948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef patch_consumed_list(to_consume=None, consumed=None):
10048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
101e19d303a6718cbd9995e27485554212b63a0a701beeps    Patches the consumed modules list to ignore modules with side effects.
10248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
103e19d303a6718cbd9995e27485554212b63a0a701beeps    Autotest relies on importing certain modules solely for their side
104e19d303a6718cbd9995e27485554212b63a0a701beeps    effects. Pylint doesn't understand this and flags them as unused, since
105e19d303a6718cbd9995e27485554212b63a0a701beeps    they're not referenced anywhere in the code. To overcome this we need
106e19d303a6718cbd9995e27485554212b63a0a701beeps    to transplant said modules into the dictionary of modules pylint has
107e19d303a6718cbd9995e27485554212b63a0a701beeps    already seen, before pylint checks it.
10848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
10948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param to_consume: a dictionary of names pylint needs to see referenced.
11048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param consumed: a dictionary of names that pylint has seen referenced.
11148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
112e19d303a6718cbd9995e27485554212b63a0a701beeps    ignore_modules = []
113e19d303a6718cbd9995e27485554212b63a0a701beeps    if (to_consume is not None and consumed is not None):
114e19d303a6718cbd9995e27485554212b63a0a701beeps        ignore_modules = [module_name for module_name in _IGNORE_MODULES
115e19d303a6718cbd9995e27485554212b63a0a701beeps                          if module_name in to_consume]
116e19d303a6718cbd9995e27485554212b63a0a701beeps
117e19d303a6718cbd9995e27485554212b63a0a701beeps    for module_name in ignore_modules:
118e19d303a6718cbd9995e27485554212b63a0a701beeps        consumed[module_name] = to_consume[module_name]
119e19d303a6718cbd9995e27485554212b63a0a701beeps        del to_consume[module_name]
12048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
12199d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
12299d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighclass CustomImportsChecker(imports.ImportsChecker):
12348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """Modifies stock imports checker to suit autotest."""
12448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    def visit_from(self, node):
12548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        node.modname = patch_modname(node.modname)
12648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        return super(CustomImportsChecker, self).visit_from(node)
12748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
12848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
12948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsclass CustomVariablesChecker(variables.VariablesChecker):
13048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """Modifies stock variables checker to suit autotest."""
13148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
13248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    def visit_module(self, node):
13348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        """
13448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        Unflag 'import common'.
13548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
13648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        _to_consume eg: [({to reference}, {referenced}, 'scope type')]
13748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        Enteries are appended to this list as we drill deeper in scope.
138e19d303a6718cbd9995e27485554212b63a0a701beeps        If we ever come across a module to ignore,  we immediately move it
13948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        to the consumed list.
14048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
14148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        @param node: node of the ast we're currently checking.
14248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        """
14348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        super(CustomVariablesChecker, self).visit_module(node)
14448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        scoped_names = self._to_consume.pop()
14548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        patch_consumed_list(scoped_names[0],scoped_names[1])
14648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        self._to_consume.append(scoped_names)
14748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
14899d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh    def visit_from(self, node):
14948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        """Patches modnames so pylints understands autotest_lib."""
15048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        node.modname = patch_modname(node.modname)
15148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        return super(CustomVariablesChecker, self).visit_from(node)
15299d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
1532c6696426c57941c1b37331b28fb594e2a0beab7beeps
1542c6696426c57941c1b37331b28fb594e2a0beab7beepsclass CustomDocStringChecker(base.DocStringChecker):
1552c6696426c57941c1b37331b28fb594e2a0beab7beeps    """Modifies stock docstring checker to suit Autotest doxygen style."""
1562c6696426c57941c1b37331b28fb594e2a0beab7beeps
1571b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    def visit_module(self, node):
1581b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps        """
1591b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps        Don't visit imported modules when checking for docstrings.
1601b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps
1611b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps        @param node: the node we're visiting.
1621b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps        """
1631b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps        pass
1641b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps
1651b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps
16698365d89774921bd065fe4bf2d273be7e100af43beeps    def visit_function(self, node):
16798365d89774921bd065fe4bf2d273be7e100af43beeps        """
16898365d89774921bd065fe4bf2d273be7e100af43beeps        Don't request docstrings for commonly overridden autotest functions.
16998365d89774921bd065fe4bf2d273be7e100af43beeps
17098365d89774921bd065fe4bf2d273be7e100af43beeps        @param node: node of the ast we're currently checking.
17198365d89774921bd065fe4bf2d273be7e100af43beeps        """
172c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps
173c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps        # Even plain functions will have a parent, which is the
174c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps        # module they're in, and a frame, which is the context
175c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps        # of said module; They need not however, always have
176c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps        # ancestors.
17798365d89774921bd065fe4bf2d273be7e100af43beeps        if (node.name in ('run_once', 'initialize', 'cleanup') and
178c38decc40a5ed6e3ee0462388074f3dd80c250cbbeeps            hasattr(node.parent.frame(), 'ancestors') and
17998365d89774921bd065fe4bf2d273be7e100af43beeps            any(ancestor.name == 'base_test' for ancestor in
18098365d89774921bd065fe4bf2d273be7e100af43beeps                node.parent.frame().ancestors())):
18198365d89774921bd065fe4bf2d273be7e100af43beeps            return
18298365d89774921bd065fe4bf2d273be7e100af43beeps
18398365d89774921bd065fe4bf2d273be7e100af43beeps        super(CustomDocStringChecker, self).visit_function(node)
18498365d89774921bd065fe4bf2d273be7e100af43beeps
18598365d89774921bd065fe4bf2d273be7e100af43beeps
186ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa    @staticmethod
187ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa    def _should_skip_arg(arg):
18898365d89774921bd065fe4bf2d273be7e100af43beeps        """
18998365d89774921bd065fe4bf2d273be7e100af43beeps        @return: True if the argument given by arg is whitelisted, and does
19098365d89774921bd065fe4bf2d273be7e100af43beeps                 not require a "@param" docstring.
19198365d89774921bd065fe4bf2d273be7e100af43beeps        """
192ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa        return arg in ('self', 'cls', 'args', 'kwargs', 'dargs')
193ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa
194ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa
1952c6696426c57941c1b37331b28fb594e2a0beab7beeps    def _check_docstring(self, node_type, node):
1962c6696426c57941c1b37331b28fb594e2a0beab7beeps        """
1972c6696426c57941c1b37331b28fb594e2a0beab7beeps        Teaches pylint to look for @param with each argument in the
1982c6696426c57941c1b37331b28fb594e2a0beab7beeps        function/method signature.
1992c6696426c57941c1b37331b28fb594e2a0beab7beeps
2002c6696426c57941c1b37331b28fb594e2a0beab7beeps        @param node_type: type of the node we're currently checking.
2012c6696426c57941c1b37331b28fb594e2a0beab7beeps        @param node: node of the ast we're currently checking.
2022c6696426c57941c1b37331b28fb594e2a0beab7beeps        """
2032c6696426c57941c1b37331b28fb594e2a0beab7beeps        super(CustomDocStringChecker, self)._check_docstring(node_type, node)
2042c6696426c57941c1b37331b28fb594e2a0beab7beeps        docstring = node.doc
205a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi        if pylint_version >= 1.1:
206a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi            key = 'missing-docstring'
207a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi        else:
208a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi            key = 'C0111'
209a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi
2102c6696426c57941c1b37331b28fb594e2a0beab7beeps        if (docstring is not None and
2112c6696426c57941c1b37331b28fb594e2a0beab7beeps               (node_type is 'method' or
2122c6696426c57941c1b37331b28fb594e2a0beab7beeps                node_type is 'function')):
2132c6696426c57941c1b37331b28fb594e2a0beab7beeps            args = node.argnames()
214a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi            old_msg = self.linter._messages[key].msg
2152c6696426c57941c1b37331b28fb594e2a0beab7beeps            for arg in args:
2162c6696426c57941c1b37331b28fb594e2a0beab7beeps                arg_docstring_rgx = '.*@param '+arg+'.*'
2172c6696426c57941c1b37331b28fb594e2a0beab7beeps                line = re.search(arg_docstring_rgx, node.doc)
218ae6acd9b3e1c6f56310cdb4e05737122b7b2818cChris Sosa                if not line and not self._should_skip_arg(arg):
219a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi                    self.linter._messages[key].msg = ('Docstring needs '
220a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi                                                      '"@param '+arg+':"')
221a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi                    self.add_message(key, node=node)
222a528758835ee15acbdc06b4d7c56cb3f52d21b41Dan Shi            self.linter._messages[key].msg = old_msg
2232c6696426c57941c1b37331b28fb594e2a0beab7beeps
2242c6696426c57941c1b37331b28fb594e2a0beab7beepsbase.DocStringChecker = CustomDocStringChecker
22599d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighimports.ImportsChecker = CustomImportsChecker
22648fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsvariables.VariablesChecker = CustomVariablesChecker
22799d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
22848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
22998365d89774921bd065fe4bf2d273be7e100af43beepsdef batch_check_files(file_paths, base_opts):
23098365d89774921bd065fe4bf2d273be7e100af43beeps    """
23198365d89774921bd065fe4bf2d273be7e100af43beeps    Run pylint on a list of files so we get consolidated errors.
23298365d89774921bd065fe4bf2d273be7e100af43beeps
23398365d89774921bd065fe4bf2d273be7e100af43beeps    @param file_paths: a list of file paths.
23498365d89774921bd065fe4bf2d273be7e100af43beeps    @param base_opts: a list of pylint config options.
23598365d89774921bd065fe4bf2d273be7e100af43beeps
23698365d89774921bd065fe4bf2d273be7e100af43beeps    @raises: pylint_error if pylint finds problems with a file
23798365d89774921bd065fe4bf2d273be7e100af43beeps             in this commit.
23898365d89774921bd065fe4bf2d273be7e100af43beeps    """
23912a3c882b514db90fcadfd1e11d1af214d1e664abeeps    if not file_paths:
24012a3c882b514db90fcadfd1e11d1af214d1e664abeeps        return
24112a3c882b514db90fcadfd1e11d1af214d1e664abeeps
24298365d89774921bd065fe4bf2d273be7e100af43beeps    pylint_runner = pylint.lint.Run(list(base_opts) + list(file_paths),
24398365d89774921bd065fe4bf2d273be7e100af43beeps                                    exit=False)
24498365d89774921bd065fe4bf2d273be7e100af43beeps    if pylint_runner.linter.msg_status:
24598365d89774921bd065fe4bf2d273be7e100af43beeps        raise pylint_error(pylint_runner.linter.msg_status)
24698365d89774921bd065fe4bf2d273be7e100af43beeps
24798365d89774921bd065fe4bf2d273be7e100af43beeps
24898365d89774921bd065fe4bf2d273be7e100af43beepsdef should_check_file(file_path):
24998365d89774921bd065fe4bf2d273be7e100af43beeps    """
25098365d89774921bd065fe4bf2d273be7e100af43beeps    Don't check blacklisted or non .py files.
25198365d89774921bd065fe4bf2d273be7e100af43beeps
25298365d89774921bd065fe4bf2d273be7e100af43beeps    @param file_path: abs path of file to check.
25398365d89774921bd065fe4bf2d273be7e100af43beeps    @return: True if this file is a non-blacklisted python file.
25498365d89774921bd065fe4bf2d273be7e100af43beeps    """
25598365d89774921bd065fe4bf2d273be7e100af43beeps    file_path = os.path.abspath(file_path)
25698365d89774921bd065fe4bf2d273be7e100af43beeps    if file_path.endswith('.py'):
25798365d89774921bd065fe4bf2d273be7e100af43beeps        return all(not fnmatch.fnmatch(file_path, '*' + pattern)
25898365d89774921bd065fe4bf2d273be7e100af43beeps                   for pattern in BLACKLIST)
25998365d89774921bd065fe4bf2d273be7e100af43beeps    return False
26098365d89774921bd065fe4bf2d273be7e100af43beeps
26198365d89774921bd065fe4bf2d273be7e100af43beeps
26248fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef check_file(file_path, base_opts):
26348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
26448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Invokes pylint on files after confirming that they're not black listed.
26548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
2662c6696426c57941c1b37331b28fb594e2a0beab7beeps    @param base_opts: pylint base options.
26748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param file_path: path to the file we need to run pylint on.
26848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
26998365d89774921bd065fe4bf2d273be7e100af43beeps    if not isinstance(file_path, basestring):
27098365d89774921bd065fe4bf2d273be7e100af43beeps        raise TypeError('expected a string as filepath, got %s'%
27198365d89774921bd065fe4bf2d273be7e100af43beeps            type(file_path))
27298365d89774921bd065fe4bf2d273be7e100af43beeps
27398365d89774921bd065fe4bf2d273be7e100af43beeps    if should_check_file(file_path):
27498365d89774921bd065fe4bf2d273be7e100af43beeps        pylint_runner = pylint.lint.Run(base_opts + [file_path], exit=False)
27598365d89774921bd065fe4bf2d273be7e100af43beeps        if pylint_runner.linter.msg_status:
27698365d89774921bd065fe4bf2d273be7e100af43beeps            pylint_error(pylint_runner.linter.msg_status)
27799d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
27899d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
27999d2dedf4a3630cb3275a6860bb2762426c2e8cfmblighdef visit(arg, dirname, filenames):
28048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
28148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Visit function invoked in check_dir.
28248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
28348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param arg: arg from os.walk.path
28448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param dirname: dir from os.walk.path
28548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param filenames: files in dir from os.walk.path
28648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
28778ece92fc08f20e9d4f66c22b0bf034fe85ce9c2beeps    for filename in filenames:
28848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        check_file(os.path.join(dirname, filename), arg)
28948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
29048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
29148fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef check_dir(dir_path, base_opts):
29248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
29348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Calls visit on files in dir_path.
29448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
2952c6696426c57941c1b37331b28fb594e2a0beab7beeps    @param base_opts: pylint base options.
29648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param dir_path: path to directory.
29748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
29848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    os.path.walk(dir_path, visit, base_opts)
29948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
30048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
30148fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef extend_baseopts(base_opts, new_opt):
30248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
30348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Replaces an argument in base_opts with a cmd line argument.
30448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
30548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param base_opts: original pylint_base_opts.
30648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param new_opt: new cmd line option.
30748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
30848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    for args in base_opts:
30948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        if new_opt in args:
31048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps            base_opts.remove(args)
31148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    base_opts.append(new_opt)
31299d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
31399d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
31448fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef get_cmdline_options(args_list, pylint_base_opts, rcfile):
31548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
31648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Parses args_list and extends pylint_base_opts.
31799d2dedf4a3630cb3275a6860bb2762426c2e8cfmbligh
31848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Command line arguments might include options mixed with files.
31948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    Go through this list and filter out the options, if the options are
32048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    specified in the pylintrc file we cannot replace them and the file
32148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    needs to be edited. If the options are already a part of
32248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    pylint_base_opts we replace them, and if not we append to
32348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    pylint_base_opts.
32440f4744243db135888488d22d14277b739f4acf8Alex Miller
32548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param args_list: list of files/pylint args passed in through argv.
32648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param pylint_base_opts: default pylint options.
32748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    @param rcfile: text from pylint_rc.
32848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """
32948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    for args in args_list:
33048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        if args.startswith('--'):
33148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps            opt_name = args[2:].split('=')[0]
33248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps            if opt_name in rcfile and pylint_version >= 0.21:
33398365d89774921bd065fe4bf2d273be7e100af43beeps                raise run_pylint_error('The rcfile already contains the %s '
33498365d89774921bd065fe4bf2d273be7e100af43beeps                                        'option. Please edit pylintrc instead.'
33598365d89774921bd065fe4bf2d273be7e100af43beeps                                        % opt_name)
33648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps            else:
33748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                extend_baseopts(pylint_base_opts, args)
33848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                args_list.remove(args)
33948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
34098365d89774921bd065fe4bf2d273be7e100af43beeps
34198365d89774921bd065fe4bf2d273be7e100af43beepsdef git_show_to_temp_file(commit, original_file, new_temp_file):
34298365d89774921bd065fe4bf2d273be7e100af43beeps    """
34398365d89774921bd065fe4bf2d273be7e100af43beeps    'Git shows' the file in original_file to a tmp file with
34498365d89774921bd065fe4bf2d273be7e100af43beeps    the name new_temp_file. We need to preserve the filename
34598365d89774921bd065fe4bf2d273be7e100af43beeps    as it gets reflected in pylints error report.
34698365d89774921bd065fe4bf2d273be7e100af43beeps
34798365d89774921bd065fe4bf2d273be7e100af43beeps    @param commit: commit hash of the commit we're running repo upload on.
34898365d89774921bd065fe4bf2d273be7e100af43beeps    @param original_file: the path to the original file we'd like to run
34998365d89774921bd065fe4bf2d273be7e100af43beeps                          'git show' on.
35098365d89774921bd065fe4bf2d273be7e100af43beeps    @param new_temp_file: new_temp_file is the path to a temp file we write the
35198365d89774921bd065fe4bf2d273be7e100af43beeps                          output of 'git show' into.
35298365d89774921bd065fe4bf2d273be7e100af43beeps    """
35398365d89774921bd065fe4bf2d273be7e100af43beeps    git_repo = revision_control.GitRepo(common.autotest_dir, None, None,
35498365d89774921bd065fe4bf2d273be7e100af43beeps        common.autotest_dir)
35598365d89774921bd065fe4bf2d273be7e100af43beeps
35698365d89774921bd065fe4bf2d273be7e100af43beeps    with open(new_temp_file, 'w') as f:
35798365d89774921bd065fe4bf2d273be7e100af43beeps        output = git_repo.gitcmd('show --no-ext-diff %s:%s'
35898365d89774921bd065fe4bf2d273be7e100af43beeps                                 % (commit, original_file),
35998365d89774921bd065fe4bf2d273be7e100af43beeps                                 ignore_status=False).stdout
36098365d89774921bd065fe4bf2d273be7e100af43beeps        f.write(output)
36198365d89774921bd065fe4bf2d273be7e100af43beeps
36298365d89774921bd065fe4bf2d273be7e100af43beeps
36398365d89774921bd065fe4bf2d273be7e100af43beepsdef check_committed_files(work_tree_files, commit, pylint_base_opts):
36498365d89774921bd065fe4bf2d273be7e100af43beeps    """
36598365d89774921bd065fe4bf2d273be7e100af43beeps    Get a list of files corresponding to the commit hash.
36698365d89774921bd065fe4bf2d273be7e100af43beeps
36798365d89774921bd065fe4bf2d273be7e100af43beeps    The contents of a file in the git work tree can differ from the contents
36898365d89774921bd065fe4bf2d273be7e100af43beeps    of a file in the commit we mean to upload. To work around this we run
36998365d89774921bd065fe4bf2d273be7e100af43beeps    pylint on a temp file into which we've 'git show'n the committed version
37098365d89774921bd065fe4bf2d273be7e100af43beeps    of each file.
37198365d89774921bd065fe4bf2d273be7e100af43beeps
37298365d89774921bd065fe4bf2d273be7e100af43beeps    @param work_tree_files: list of files in this commit specified by their
37398365d89774921bd065fe4bf2d273be7e100af43beeps                            absolute path.
37498365d89774921bd065fe4bf2d273be7e100af43beeps    @param commit: hash of the commit this upload applies to.
37598365d89774921bd065fe4bf2d273be7e100af43beeps    @param pylint_base_opts: a list of pylint config options.
37698365d89774921bd065fe4bf2d273be7e100af43beeps    """
37798365d89774921bd065fe4bf2d273be7e100af43beeps    files_to_check = filter(should_check_file, work_tree_files)
37898365d89774921bd065fe4bf2d273be7e100af43beeps
37998365d89774921bd065fe4bf2d273be7e100af43beeps    # Map the absolute path of each file so it's relative to the autotest repo.
38098365d89774921bd065fe4bf2d273be7e100af43beeps    # All files that are a part of this commit should have an abs path within
38198365d89774921bd065fe4bf2d273be7e100af43beeps    # the autotest repo, so this regex should never fail.
38298365d89774921bd065fe4bf2d273be7e100af43beeps    work_tree_files = [re.search(r'%s/(.*)' % common.autotest_dir, f).group(1)
38398365d89774921bd065fe4bf2d273be7e100af43beeps                       for f in files_to_check]
38498365d89774921bd065fe4bf2d273be7e100af43beeps
38598365d89774921bd065fe4bf2d273be7e100af43beeps    tempdir = None
38698365d89774921bd065fe4bf2d273be7e100af43beeps    try:
38798365d89774921bd065fe4bf2d273be7e100af43beeps        tempdir = autotemp.tempdir()
38898365d89774921bd065fe4bf2d273be7e100af43beeps        temp_files = [os.path.join(tempdir.name, file_path.split('/')[-1:][0])
38998365d89774921bd065fe4bf2d273be7e100af43beeps                      for file_path in work_tree_files]
39098365d89774921bd065fe4bf2d273be7e100af43beeps
39198365d89774921bd065fe4bf2d273be7e100af43beeps        for file_tuple in zip(work_tree_files, temp_files):
39298365d89774921bd065fe4bf2d273be7e100af43beeps            git_show_to_temp_file(commit, *file_tuple)
39398365d89774921bd065fe4bf2d273be7e100af43beeps        # Only check if we successfully git showed all files in the commit.
39498365d89774921bd065fe4bf2d273be7e100af43beeps        batch_check_files(temp_files, pylint_base_opts)
39598365d89774921bd065fe4bf2d273be7e100af43beeps    finally:
39698365d89774921bd065fe4bf2d273be7e100af43beeps        if tempdir:
39798365d89774921bd065fe4bf2d273be7e100af43beeps            tempdir.clean()
39898365d89774921bd065fe4bf2d273be7e100af43beeps
39998365d89774921bd065fe4bf2d273be7e100af43beeps
40048fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsdef main():
40148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    """Main function checks each file in a commit for pylint violations."""
40248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
4031b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    # For now all error/warning/refactor/convention exceptions except those in
4041b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    # the enable string are disabled.
4051b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    # W0611: All imported modules (except common) need to be used.
4061b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    # W1201: Logging methods should take the form
4071b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    #   logging.<loggingmethod>(format_string, format_args...); and not
4081b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    #   logging.<loggingmethod>(format_string % (format_args...))
4091b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    # C0111: Docstring needed. Also checks @param for each arg.
4101b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps    # C0112: Non-empty Docstring needed.
41148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # Ideally we would like to enable as much as we can, but if we did so at
41248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # this stage anyone who makes a tiny change to a file will be tasked with
41348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # cleaning all the lint in it. See chromium-os:37364.
41448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
415e19d303a6718cbd9995e27485554212b63a0a701beeps    # Note:
416e19d303a6718cbd9995e27485554212b63a0a701beeps    # 1. There are three major sources of E1101/E1103/E1120 false positives:
417e19d303a6718cbd9995e27485554212b63a0a701beeps    #    * common_lib.enum.Enum objects
418e19d303a6718cbd9995e27485554212b63a0a701beeps    #    * DB model objects (scheduler models are the worst, but Django models
419e19d303a6718cbd9995e27485554212b63a0a701beeps    #      also generate some errors)
420e19d303a6718cbd9995e27485554212b63a0a701beeps    # 2. Docstrings are optional on private methods, and any methods that begin
421e19d303a6718cbd9995e27485554212b63a0a701beeps    #    with either 'set_' or 'get_'.
42248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    pylint_rc = os.path.join(os.path.dirname(os.path.abspath(__file__)),
42348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                             'pylintrc')
424e19d303a6718cbd9995e27485554212b63a0a701beeps
425e19d303a6718cbd9995e27485554212b63a0a701beeps    no_docstring_rgx = r'((_.*)|(set_.*)|(get_.*))'
42648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    if pylint_version >= 0.21:
42748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        pylint_base_opts = ['--rcfile=%s' % pylint_rc,
42848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                            '--reports=no',
4295553256ffaaa15b1bcaf92d58c14c06243c609fcbeeps                            '--disable=W,R,E,C,F',
4305e2bb4aa28611aaacaa8798fd07943ede1df46c6beeps                            '--enable=W0611,W1201,C0111,C0112,E0602,W0601',
431e19d303a6718cbd9995e27485554212b63a0a701beeps                            '--no-docstring-rgx=%s' % no_docstring_rgx,]
43248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    else:
43348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        all_failures = 'error,warning,refactor,convention'
43448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        pylint_base_opts = ['--disable-msg-cat=%s' % all_failures,
43548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                            '--reports=no',
43648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                            '--include-ids=y',
4371b6433bdf3ea1f1fffaa523c6aa0411e3d8f4ecebeeps                            '--ignore-docstrings=n',
438e19d303a6718cbd9995e27485554212b63a0a701beeps                            '--no-docstring-rgx=%s' % no_docstring_rgx,]
43948fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
44048fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # run_pylint can be invoked directly with command line arguments,
44148fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # or through a presubmit hook which uses the arguments in pylintrc. In the
44248fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # latter case no command line arguments are passed. If it is invoked
44348fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    # directly without any arguments, it should check all files in the cwd.
44448fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    args_list = sys.argv[1:]
44548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    if args_list:
44648fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps        get_cmdline_options(args_list,
44748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                            pylint_base_opts,
44848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps                            open(pylint_rc).read())
44998365d89774921bd065fe4bf2d273be7e100af43beeps        batch_check_files(args_list, pylint_base_opts)
45098365d89774921bd065fe4bf2d273be7e100af43beeps    elif os.environ.get('PRESUBMIT_FILES') is not None:
45198365d89774921bd065fe4bf2d273be7e100af43beeps        check_committed_files(
45298365d89774921bd065fe4bf2d273be7e100af43beeps                              os.environ.get('PRESUBMIT_FILES').split('\n'),
45398365d89774921bd065fe4bf2d273be7e100af43beeps                              os.environ.get('PRESUBMIT_COMMIT'),
45498365d89774921bd065fe4bf2d273be7e100af43beeps                              pylint_base_opts)
45548fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps    else:
45698365d89774921bd065fe4bf2d273be7e100af43beeps        check_dir('.', pylint_base_opts)
45748fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
45848fc6f50b87baa1ca6926c013e50b433f92d7aaebeeps
45948fc6f50b87baa1ca6926c013e50b433f92d7aaebeepsif __name__ == '__main__':
460baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian    try:
461baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian        main()
4627e10708628c1bd21d6a08adab3913c2275204df9Prathmesh Prabhu    except pylint_error as e:
463baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian        logging.error(e)
464baabef2eae533784f861b33ab41711616d399f04Prashanth Balasubramanian        sys.exit(1)
465