1#!/usr/bin/python
2"""
3Script to verify errors on autotest code contributions (patches).
4The workflow is as follows:
5
6 * Patch will be applied and eventual problems will be notified.
7 * If there are new files created, remember user to add them to VCS.
8 * If any added file looks like a executable file, remember user to make them
9   executable.
10 * If any of the files added or modified introduces trailing whitespaces, tabs
11   or incorrect indentation, report problems.
12 * If any of the files have problems during pylint validation, report failures.
13 * If any of the files changed have a unittest suite, run the unittest suite
14   and report any failures.
15
16Usage: check_patch.py -p [/path/to/patch]
17       check_patch.py -i [patchwork id]
18
19@copyright: Red Hat Inc, 2009.
20@author: Lucas Meneghel Rodrigues <lmr@redhat.com>
21"""
22
23import os, stat, logging, sys, optparse, time
24import common
25from autotest_lib.client.common_lib import utils, error, logging_config
26from autotest_lib.client.common_lib import logging_manager
27
28
29class CheckPatchLoggingConfig(logging_config.LoggingConfig):
30    def configure_logging(self, results_dir=None, verbose=False):
31        super(CheckPatchLoggingConfig, self).configure_logging(use_console=True,
32                                                               verbose=verbose)
33
34
35class VCS(object):
36    """
37    Abstraction layer to the version control system.
38    """
39    def __init__(self):
40        """
41        Class constructor. Guesses the version control name and instantiates it
42        as a backend.
43        """
44        backend_name = self.guess_vcs_name()
45        if backend_name == "SVN":
46            self.backend = SubVersionBackend()
47
48
49    def guess_vcs_name(self):
50        if os.path.isdir(".svn"):
51            return "SVN"
52        else:
53            logging.error("Could not figure version control system. Are you "
54                          "on a working directory? Aborting.")
55            sys.exit(1)
56
57
58    def get_unknown_files(self):
59        """
60        Return a list of files unknown to the VCS.
61        """
62        return self.backend.get_unknown_files()
63
64
65    def get_modified_files(self):
66        """
67        Return a list of files that were modified, according to the VCS.
68        """
69        return self.backend.get_modified_files()
70
71
72    def add_untracked_file(self, file):
73        """
74        Add an untracked file to version control.
75        """
76        return self.backend.add_untracked_file(file)
77
78
79    def revert_file(self, file):
80        """
81        Restore file according to the latest state on the reference repo.
82        """
83        return self.backend.revert_file(file)
84
85
86    def apply_patch(self, patch):
87        """
88        Applies a patch using the most appropriate method to the particular VCS.
89        """
90        return self.backend.apply_patch(patch)
91
92
93    def update(self):
94        """
95        Updates the tree according to the latest state of the public tree
96        """
97        return self.backend.update()
98
99
100class SubVersionBackend(object):
101    """
102    Implementation of a subversion backend for use with the VCS abstraction
103    layer.
104    """
105    def __init__(self):
106        logging.debug("Subversion VCS backend initialized.")
107        self.ignored_extension_list = ['.orig', '.bak']
108
109
110    def get_unknown_files(self):
111        status = utils.system_output("svn status --ignore-externals")
112        unknown_files = []
113        for line in status.split("\n"):
114            status_flag = line[0]
115            if line and status_flag == "?":
116                for extension in self.ignored_extension_list:
117                    if not line.endswith(extension):
118                        unknown_files.append(line[1:].strip())
119        return unknown_files
120
121
122    def get_modified_files(self):
123        status = utils.system_output("svn status --ignore-externals")
124        modified_files = []
125        for line in status.split("\n"):
126            status_flag = line[0]
127            if line and status_flag == "M" or status_flag == "A":
128                modified_files.append(line[1:].strip())
129        return modified_files
130
131
132    def add_untracked_file(self, file):
133        """
134        Add an untracked file under revision control.
135
136        @param file: Path to untracked file.
137        """
138        try:
139            utils.run('svn add %s' % file)
140        except error.CmdError, e:
141            logging.error("Problem adding file %s to svn: %s", file, e)
142            sys.exit(1)
143
144
145    def revert_file(self, file):
146        """
147        Revert file against last revision.
148
149        @param file: Path to file to be reverted.
150        """
151        try:
152            utils.run('svn revert %s' % file)
153        except error.CmdError, e:
154            logging.error("Problem reverting file %s: %s", file, e)
155            sys.exit(1)
156
157
158    def apply_patch(self, patch):
159        """
160        Apply a patch to the code base. Patches are expected to be made using
161        level -p1, and taken according to the code base top level.
162
163        @param patch: Path to the patch file.
164        """
165        try:
166            utils.system_output("patch -p1 < %s" % patch)
167        except:
168            logging.error("Patch applied incorrectly. Possible causes: ")
169            logging.error("1 - Patch might not be -p1")
170            logging.error("2 - You are not at the top of the autotest tree")
171            logging.error("3 - Patch was made using an older tree")
172            logging.error("4 - Mailer might have messed the patch")
173            sys.exit(1)
174
175    def update(self):
176        try:
177            utils.system("svn update", ignore_status=True)
178        except error.CmdError, e:
179            logging.error("SVN tree update failed: %s" % e)
180
181
182class FileChecker(object):
183    """
184    Picks up a given file and performs various checks, looking after problems
185    and eventually suggesting solutions.
186    """
187    def __init__(self, path, confirm=False):
188        """
189        Class constructor, sets the path attribute.
190
191        @param path: Path to the file that will be checked.
192        @param confirm: Whether to answer yes to all questions asked without
193                prompting the user.
194        """
195        self.path = path
196        self.confirm = confirm
197        self.basename = os.path.basename(self.path)
198        if self.basename.endswith('.py'):
199            self.is_python = True
200        else:
201            self.is_python = False
202
203        mode = os.stat(self.path)[stat.ST_MODE]
204        if mode & stat.S_IXUSR:
205            self.is_executable = True
206        else:
207            self.is_executable = False
208
209        checked_file = open(self.path, "r")
210        self.first_line = checked_file.readline()
211        checked_file.close()
212        self.corrective_actions = []
213        self.indentation_exceptions = ['job_unittest.py']
214
215
216    def _check_indent(self):
217        """
218        Verifies the file with reindent.py. This tool performs the following
219        checks on python files:
220
221          * Trailing whitespaces
222          * Tabs
223          * End of line
224          * Incorrect indentation
225
226        For the purposes of checking, the dry run mode is used and no changes
227        are made. It is up to the user to decide if he wants to run reindent
228        to correct the issues.
229        """
230        reindent_raw = utils.system_output('reindent.py -v -d %s | head -1' %
231                                           self.path)
232        reindent_results = reindent_raw.split(" ")[-1].strip(".")
233        if reindent_results == "changed":
234            if self.basename not in self.indentation_exceptions:
235                self.corrective_actions.append("reindent.py -v %s" % self.path)
236
237
238    def _check_code(self):
239        """
240        Verifies the file with run_pylint.py. This tool will call the static
241        code checker pylint using the special autotest conventions and warn
242        only on problems. If problems are found, a report will be generated.
243        Some of the problems reported might be bogus, but it's allways good
244        to look at them.
245        """
246        c_cmd = 'run_pylint.py %s' % self.path
247        rc = utils.system(c_cmd, ignore_status=True)
248        if rc != 0:
249            logging.error("Syntax issues found during '%s'", c_cmd)
250
251
252    def _check_unittest(self):
253        """
254        Verifies if the file in question has a unittest suite, if so, run the
255        unittest and report on any failures. This is important to keep our
256        unit tests up to date.
257        """
258        if "unittest" not in self.basename:
259            stripped_name = self.basename.strip(".py")
260            unittest_name = stripped_name + "_unittest.py"
261            unittest_path = self.path.replace(self.basename, unittest_name)
262            if os.path.isfile(unittest_path):
263                unittest_cmd = 'python %s' % unittest_path
264                rc = utils.system(unittest_cmd, ignore_status=True)
265                if rc != 0:
266                    logging.error("Unittest issues found during '%s'",
267                                  unittest_cmd)
268
269
270    def _check_permissions(self):
271        """
272        Verifies the execution permissions, specifically:
273          * Files with no shebang and execution permissions are reported.
274          * Files with shebang and no execution permissions are reported.
275        """
276        if self.first_line.startswith("#!"):
277            if not self.is_executable:
278                self.corrective_actions.append("svn propset svn:executable ON %s" % self.path)
279        else:
280            if self.is_executable:
281                self.corrective_actions.append("svn propdel svn:executable %s" % self.path)
282
283
284    def report(self):
285        """
286        Executes all required checks, if problems are found, the possible
287        corrective actions are listed.
288        """
289        self._check_permissions()
290        if self.is_python:
291            self._check_indent()
292            self._check_code()
293            self._check_unittest()
294        if self.corrective_actions:
295            for action in self.corrective_actions:
296                answer = utils.ask("Would you like to execute %s?" % action,
297                                   auto=self.confirm)
298                if answer == "y":
299                    rc = utils.system(action, ignore_status=True)
300                    if rc != 0:
301                        logging.error("Error executing %s" % action)
302
303
304class PatchChecker(object):
305    def __init__(self, patch=None, patchwork_id=None, confirm=False):
306        self.confirm = confirm
307        self.base_dir = os.getcwd()
308        if patch:
309            self.patch = os.path.abspath(patch)
310        if patchwork_id:
311            self.patch = self._fetch_from_patchwork(patchwork_id)
312
313        if not os.path.isfile(self.patch):
314            logging.error("Invalid patch file %s provided. Aborting.",
315                          self.patch)
316            sys.exit(1)
317
318        self.vcs = VCS()
319        changed_files_before = self.vcs.get_modified_files()
320        if changed_files_before:
321            logging.error("Repository has changed files prior to patch "
322                          "application. ")
323            answer = utils.ask("Would you like to revert them?", auto=self.confirm)
324            if answer == "n":
325                logging.error("Not safe to proceed without reverting files.")
326                sys.exit(1)
327            else:
328                for changed_file in changed_files_before:
329                    self.vcs.revert_file(changed_file)
330
331        self.untracked_files_before = self.vcs.get_unknown_files()
332        self.vcs.update()
333
334
335    def _fetch_from_patchwork(self, id):
336        """
337        Gets a patch file from patchwork and puts it under the cwd so it can
338        be applied.
339
340        @param id: Patchwork patch id.
341        """
342        patch_url = "http://patchwork.test.kernel.org/patch/%s/mbox/" % id
343        patch_dest = os.path.join(self.base_dir, 'patchwork-%s.patch' % id)
344        patch = utils.get_file(patch_url, patch_dest)
345        # Patchwork sometimes puts garbage on the path, such as long
346        # sequences of underscores (_______). Get rid of those.
347        patch_ro = open(patch, 'r')
348        patch_contents = patch_ro.readlines()
349        patch_ro.close()
350        patch_rw = open(patch, 'w')
351        for line in patch_contents:
352            if not line.startswith("___"):
353                patch_rw.write(line)
354        patch_rw.close()
355        return patch
356
357
358    def _check_files_modified_patch(self):
359        untracked_files_after = self.vcs.get_unknown_files()
360        modified_files_after = self.vcs.get_modified_files()
361        add_to_vcs = []
362        for untracked_file in untracked_files_after:
363            if untracked_file not in self.untracked_files_before:
364                add_to_vcs.append(untracked_file)
365
366        if add_to_vcs:
367            logging.info("The files: ")
368            for untracked_file in add_to_vcs:
369                logging.info(untracked_file)
370            logging.info("Might need to be added to VCS")
371            answer = utils.ask("Would you like to add them to VCS ?")
372            if answer == "y":
373                for untracked_file in add_to_vcs:
374                    self.vcs.add_untracked_file(untracked_file)
375                    modified_files_after.append(untracked_file)
376            elif answer == "n":
377                pass
378
379        for modified_file in modified_files_after:
380            # Additional safety check, new commits might introduce
381            # new directories
382            if os.path.isfile(modified_file):
383                file_checker = FileChecker(modified_file)
384                file_checker.report()
385
386
387    def check(self):
388        self.vcs.apply_patch(self.patch)
389        self._check_files_modified_patch()
390
391
392if __name__ == "__main__":
393    parser = optparse.OptionParser()
394    parser.add_option('-p', '--patch', dest="local_patch", action='store',
395                      help='path to a patch file that will be checked')
396    parser.add_option('-i', '--patchwork-id', dest="id", action='store',
397                      help='id of a given patchwork patch')
398    parser.add_option('--verbose', dest="debug", action='store_true',
399                      help='include debug messages in console output')
400    parser.add_option('-f', '--full-check', dest="full_check",
401                      action='store_true',
402                      help='check the full tree for corrective actions')
403    parser.add_option('-y', '--yes', dest="confirm",
404                      action='store_true',
405                      help='Answer yes to all questions')
406
407    options, args = parser.parse_args()
408    local_patch = options.local_patch
409    id = options.id
410    debug = options.debug
411    full_check = options.full_check
412    confirm = options.confirm
413
414    logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
415
416    ignore_file_list = ['common.py']
417    if full_check:
418        for root, dirs, files in os.walk('.'):
419            if not '.svn' in root:
420                for file in files:
421                    if file not in ignore_file_list:
422                        path = os.path.join(root, file)
423                        file_checker = FileChecker(path, confirm=confirm)
424                        file_checker.report()
425    else:
426        if local_patch:
427            patch_checker = PatchChecker(patch=local_patch, confirm=confirm)
428        elif id:
429            patch_checker = PatchChecker(patchwork_id=id, confirm=confirm)
430        else:
431            logging.error('No patch or patchwork id specified. Aborting.')
432            sys.exit(1)
433        patch_checker.check()
434