1#!/usr/bin/python -u
2"""
3Wrapper to patch pylint library functions to suit autotest.
4
5This script is invoked as part of the presubmit checks for autotest python
6files. It runs pylint on a list of files that it obtains either through
7the command line or from an environment variable set in pre-upload.py.
8
9Example:
10run_pylint.py filename.py
11"""
12
13import fnmatch
14import logging
15import os
16import re
17import sys
18
19import common
20from autotest_lib.client.common_lib import autotemp, revision_control
21
22# Do a basic check to see if pylint is even installed.
23try:
24    import pylint
25    from pylint.__pkginfo__ import version as pylint_version
26except ImportError:
27    print ("Unable to import pylint, it may need to be installed."
28           " Run 'sudo aptitude install pylint' if you haven't already.")
29    sys.exit(1)
30
31major, minor, release = pylint_version.split('.')
32pylint_version = float("%s.%s" % (major, minor))
33
34# some files make pylint blow up, so make sure we ignore them
35BLACKLIST = ['/site-packages/*', '/contrib/*', '/frontend/afe/management.py']
36
37# patch up the logilab module lookup tools to understand autotest_lib.* trash
38import logilab.common.modutils
39_ffm = logilab.common.modutils.file_from_modpath
40def file_from_modpath(modpath, path=None, context_file=None):
41    """
42    Wrapper to eliminate autotest_lib from modpath.
43
44    @param modpath: name of module splitted on '.'
45    @param path: optional list of paths where module should be searched for.
46    @param context_file: path to file doing the importing.
47    @return The path to the module as returned by the parent method invocation.
48    @raises: ImportError if these is no such module.
49    """
50    if modpath[0] == "autotest_lib":
51        return _ffm(modpath[1:], path, context_file)
52    else:
53        return _ffm(modpath, path, context_file)
54logilab.common.modutils.file_from_modpath = file_from_modpath
55
56
57import pylint.lint
58from pylint.checkers import base, imports, variables
59
60# need to put autotest root dir on sys.path so pylint will be happy
61autotest_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
62sys.path.insert(0, autotest_root)
63
64# patch up pylint import checker to handle our importing magic
65ROOT_MODULE = 'autotest_lib.'
66
67# A list of modules for pylint to ignore, specifically, these modules
68# are imported for their side-effects and are not meant to be used.
69_IGNORE_MODULES=['common', 'frontend_test_utils',
70                 'setup_django_environment',
71                 'setup_django_lite_environment',
72                 'setup_django_readonly_environment', 'setup_test_environment',]
73
74
75class pylint_error(Exception):
76    """
77    Error raised when pylint complains about a file.
78    """
79
80
81class run_pylint_error(pylint_error):
82    """
83    Error raised when an assumption made in this file is violated.
84    """
85
86
87def patch_modname(modname):
88    """
89    Patches modname so we can make sense of autotest_lib modules.
90
91    @param modname: name of a module, contains '.'
92    @return modified modname string.
93    """
94    if modname.startswith(ROOT_MODULE) or modname.startswith(ROOT_MODULE[:-1]):
95        modname = modname[len(ROOT_MODULE):]
96    return modname
97
98
99def patch_consumed_list(to_consume=None, consumed=None):
100    """
101    Patches the consumed modules list to ignore modules with side effects.
102
103    Autotest relies on importing certain modules solely for their side
104    effects. Pylint doesn't understand this and flags them as unused, since
105    they're not referenced anywhere in the code. To overcome this we need
106    to transplant said modules into the dictionary of modules pylint has
107    already seen, before pylint checks it.
108
109    @param to_consume: a dictionary of names pylint needs to see referenced.
110    @param consumed: a dictionary of names that pylint has seen referenced.
111    """
112    ignore_modules = []
113    if (to_consume is not None and consumed is not None):
114        ignore_modules = [module_name for module_name in _IGNORE_MODULES
115                          if module_name in to_consume]
116
117    for module_name in ignore_modules:
118        consumed[module_name] = to_consume[module_name]
119        del to_consume[module_name]
120
121
122class CustomImportsChecker(imports.ImportsChecker):
123    """Modifies stock imports checker to suit autotest."""
124    def visit_from(self, node):
125        node.modname = patch_modname(node.modname)
126        return super(CustomImportsChecker, self).visit_from(node)
127
128
129class CustomVariablesChecker(variables.VariablesChecker):
130    """Modifies stock variables checker to suit autotest."""
131
132    def visit_module(self, node):
133        """
134        Unflag 'import common'.
135
136        _to_consume eg: [({to reference}, {referenced}, 'scope type')]
137        Enteries are appended to this list as we drill deeper in scope.
138        If we ever come across a module to ignore,  we immediately move it
139        to the consumed list.
140
141        @param node: node of the ast we're currently checking.
142        """
143        super(CustomVariablesChecker, self).visit_module(node)
144        scoped_names = self._to_consume.pop()
145        patch_consumed_list(scoped_names[0],scoped_names[1])
146        self._to_consume.append(scoped_names)
147
148    def visit_from(self, node):
149        """Patches modnames so pylints understands autotest_lib."""
150        node.modname = patch_modname(node.modname)
151        return super(CustomVariablesChecker, self).visit_from(node)
152
153
154class CustomDocStringChecker(base.DocStringChecker):
155    """Modifies stock docstring checker to suit Autotest doxygen style."""
156
157    def visit_module(self, node):
158        """
159        Don't visit imported modules when checking for docstrings.
160
161        @param node: the node we're visiting.
162        """
163        pass
164
165
166    def visit_function(self, node):
167        """
168        Don't request docstrings for commonly overridden autotest functions.
169
170        @param node: node of the ast we're currently checking.
171        """
172
173        # Even plain functions will have a parent, which is the
174        # module they're in, and a frame, which is the context
175        # of said module; They need not however, always have
176        # ancestors.
177        if (node.name in ('run_once', 'initialize', 'cleanup') and
178            hasattr(node.parent.frame(), 'ancestors') and
179            any(ancestor.name == 'base_test' for ancestor in
180                node.parent.frame().ancestors())):
181            return
182
183        super(CustomDocStringChecker, self).visit_function(node)
184
185
186    @staticmethod
187    def _should_skip_arg(arg):
188        """
189        @return: True if the argument given by arg is whitelisted, and does
190                 not require a "@param" docstring.
191        """
192        return arg in ('self', 'cls', 'args', 'kwargs', 'dargs')
193
194
195    def _check_docstring(self, node_type, node):
196        """
197        Teaches pylint to look for @param with each argument in the
198        function/method signature.
199
200        @param node_type: type of the node we're currently checking.
201        @param node: node of the ast we're currently checking.
202        """
203        super(CustomDocStringChecker, self)._check_docstring(node_type, node)
204        docstring = node.doc
205        if pylint_version >= 1.1:
206            key = 'missing-docstring'
207        else:
208            key = 'C0111'
209
210        if (docstring is not None and
211               (node_type is 'method' or
212                node_type is 'function')):
213            args = node.argnames()
214            old_msg = self.linter._messages[key].msg
215            for arg in args:
216                arg_docstring_rgx = '.*@param '+arg+'.*'
217                line = re.search(arg_docstring_rgx, node.doc)
218                if not line and not self._should_skip_arg(arg):
219                    self.linter._messages[key].msg = ('Docstring needs '
220                                                      '"@param '+arg+':"')
221                    self.add_message(key, node=node)
222            self.linter._messages[key].msg = old_msg
223
224base.DocStringChecker = CustomDocStringChecker
225imports.ImportsChecker = CustomImportsChecker
226variables.VariablesChecker = CustomVariablesChecker
227
228
229def batch_check_files(file_paths, base_opts):
230    """
231    Run pylint on a list of files so we get consolidated errors.
232
233    @param file_paths: a list of file paths.
234    @param base_opts: a list of pylint config options.
235
236    @raises: pylint_error if pylint finds problems with a file
237             in this commit.
238    """
239    if not file_paths:
240        return
241
242    pylint_runner = pylint.lint.Run(list(base_opts) + list(file_paths),
243                                    exit=False)
244    if pylint_runner.linter.msg_status:
245        raise pylint_error(pylint_runner.linter.msg_status)
246
247
248def should_check_file(file_path):
249    """
250    Don't check blacklisted or non .py files.
251
252    @param file_path: abs path of file to check.
253    @return: True if this file is a non-blacklisted python file.
254    """
255    file_path = os.path.abspath(file_path)
256    if file_path.endswith('.py'):
257        return all(not fnmatch.fnmatch(file_path, '*' + pattern)
258                   for pattern in BLACKLIST)
259    return False
260
261
262def check_file(file_path, base_opts):
263    """
264    Invokes pylint on files after confirming that they're not black listed.
265
266    @param base_opts: pylint base options.
267    @param file_path: path to the file we need to run pylint on.
268    """
269    if not isinstance(file_path, basestring):
270        raise TypeError('expected a string as filepath, got %s'%
271            type(file_path))
272
273    if should_check_file(file_path):
274        pylint_runner = pylint.lint.Run(base_opts + [file_path], exit=False)
275        if pylint_runner.linter.msg_status:
276            pylint_error(pylint_runner.linter.msg_status)
277
278
279def visit(arg, dirname, filenames):
280    """
281    Visit function invoked in check_dir.
282
283    @param arg: arg from os.walk.path
284    @param dirname: dir from os.walk.path
285    @param filenames: files in dir from os.walk.path
286    """
287    for filename in filenames:
288        check_file(os.path.join(dirname, filename), arg)
289
290
291def check_dir(dir_path, base_opts):
292    """
293    Calls visit on files in dir_path.
294
295    @param base_opts: pylint base options.
296    @param dir_path: path to directory.
297    """
298    os.path.walk(dir_path, visit, base_opts)
299
300
301def extend_baseopts(base_opts, new_opt):
302    """
303    Replaces an argument in base_opts with a cmd line argument.
304
305    @param base_opts: original pylint_base_opts.
306    @param new_opt: new cmd line option.
307    """
308    for args in base_opts:
309        if new_opt in args:
310            base_opts.remove(args)
311    base_opts.append(new_opt)
312
313
314def get_cmdline_options(args_list, pylint_base_opts, rcfile):
315    """
316    Parses args_list and extends pylint_base_opts.
317
318    Command line arguments might include options mixed with files.
319    Go through this list and filter out the options, if the options are
320    specified in the pylintrc file we cannot replace them and the file
321    needs to be edited. If the options are already a part of
322    pylint_base_opts we replace them, and if not we append to
323    pylint_base_opts.
324
325    @param args_list: list of files/pylint args passed in through argv.
326    @param pylint_base_opts: default pylint options.
327    @param rcfile: text from pylint_rc.
328    """
329    for args in args_list:
330        if args.startswith('--'):
331            opt_name = args[2:].split('=')[0]
332            if opt_name in rcfile and pylint_version >= 0.21:
333                raise run_pylint_error('The rcfile already contains the %s '
334                                        'option. Please edit pylintrc instead.'
335                                        % opt_name)
336            else:
337                extend_baseopts(pylint_base_opts, args)
338                args_list.remove(args)
339
340
341def git_show_to_temp_file(commit, original_file, new_temp_file):
342    """
343    'Git shows' the file in original_file to a tmp file with
344    the name new_temp_file. We need to preserve the filename
345    as it gets reflected in pylints error report.
346
347    @param commit: commit hash of the commit we're running repo upload on.
348    @param original_file: the path to the original file we'd like to run
349                          'git show' on.
350    @param new_temp_file: new_temp_file is the path to a temp file we write the
351                          output of 'git show' into.
352    """
353    git_repo = revision_control.GitRepo(common.autotest_dir, None, None,
354        common.autotest_dir)
355
356    with open(new_temp_file, 'w') as f:
357        output = git_repo.gitcmd('show --no-ext-diff %s:%s'
358                                 % (commit, original_file),
359                                 ignore_status=False).stdout
360        f.write(output)
361
362
363def check_committed_files(work_tree_files, commit, pylint_base_opts):
364    """
365    Get a list of files corresponding to the commit hash.
366
367    The contents of a file in the git work tree can differ from the contents
368    of a file in the commit we mean to upload. To work around this we run
369    pylint on a temp file into which we've 'git show'n the committed version
370    of each file.
371
372    @param work_tree_files: list of files in this commit specified by their
373                            absolute path.
374    @param commit: hash of the commit this upload applies to.
375    @param pylint_base_opts: a list of pylint config options.
376    """
377    files_to_check = filter(should_check_file, work_tree_files)
378
379    # Map the absolute path of each file so it's relative to the autotest repo.
380    # All files that are a part of this commit should have an abs path within
381    # the autotest repo, so this regex should never fail.
382    work_tree_files = [re.search(r'%s/(.*)' % common.autotest_dir, f).group(1)
383                       for f in files_to_check]
384
385    tempdir = None
386    try:
387        tempdir = autotemp.tempdir()
388        temp_files = [os.path.join(tempdir.name, file_path.split('/')[-1:][0])
389                      for file_path in work_tree_files]
390
391        for file_tuple in zip(work_tree_files, temp_files):
392            git_show_to_temp_file(commit, *file_tuple)
393        # Only check if we successfully git showed all files in the commit.
394        batch_check_files(temp_files, pylint_base_opts)
395    finally:
396        if tempdir:
397            tempdir.clean()
398
399
400def main():
401    """Main function checks each file in a commit for pylint violations."""
402
403    # For now all error/warning/refactor/convention exceptions except those in
404    # the enable string are disabled.
405    # W0611: All imported modules (except common) need to be used.
406    # W1201: Logging methods should take the form
407    #   logging.<loggingmethod>(format_string, format_args...); and not
408    #   logging.<loggingmethod>(format_string % (format_args...))
409    # C0111: Docstring needed. Also checks @param for each arg.
410    # C0112: Non-empty Docstring needed.
411    # Ideally we would like to enable as much as we can, but if we did so at
412    # this stage anyone who makes a tiny change to a file will be tasked with
413    # cleaning all the lint in it. See chromium-os:37364.
414
415    # Note:
416    # 1. There are three major sources of E1101/E1103/E1120 false positives:
417    #    * common_lib.enum.Enum objects
418    #    * DB model objects (scheduler models are the worst, but Django models
419    #      also generate some errors)
420    # 2. Docstrings are optional on private methods, and any methods that begin
421    #    with either 'set_' or 'get_'.
422    pylint_rc = os.path.join(os.path.dirname(os.path.abspath(__file__)),
423                             'pylintrc')
424
425    no_docstring_rgx = r'((_.*)|(set_.*)|(get_.*))'
426    if pylint_version >= 0.21:
427        pylint_base_opts = ['--rcfile=%s' % pylint_rc,
428                            '--reports=no',
429                            '--disable=W,R,E,C,F',
430                            '--enable=W0611,W1201,C0111,C0112,E0602,W0601',
431                            '--no-docstring-rgx=%s' % no_docstring_rgx,]
432    else:
433        all_failures = 'error,warning,refactor,convention'
434        pylint_base_opts = ['--disable-msg-cat=%s' % all_failures,
435                            '--reports=no',
436                            '--include-ids=y',
437                            '--ignore-docstrings=n',
438                            '--no-docstring-rgx=%s' % no_docstring_rgx,]
439
440    # run_pylint can be invoked directly with command line arguments,
441    # or through a presubmit hook which uses the arguments in pylintrc. In the
442    # latter case no command line arguments are passed. If it is invoked
443    # directly without any arguments, it should check all files in the cwd.
444    args_list = sys.argv[1:]
445    if args_list:
446        get_cmdline_options(args_list,
447                            pylint_base_opts,
448                            open(pylint_rc).read())
449        batch_check_files(args_list, pylint_base_opts)
450    elif os.environ.get('PRESUBMIT_FILES') is not None:
451        check_committed_files(
452                              os.environ.get('PRESUBMIT_FILES').split('\n'),
453                              os.environ.get('PRESUBMIT_COMMIT'),
454                              pylint_base_opts)
455    else:
456        check_dir('.', pylint_base_opts)
457
458
459if __name__ == '__main__':
460    try:
461        main()
462    except pylint_error as e:
463        logging.error(e)
464        sys.exit(1)
465