unittest_suite.py revision 1f23b6918dd601f469eea2975f3e8dbda6659b58 
1#!/usr/bin/python -u
2
3import os, sys, unittest, optparse
4import common
5from autotest_lib.utils import parallel
6from autotest_lib.client.common_lib.test_utils import unittest as custom_unittest
7
8parser = optparse.OptionParser()
9parser.add_option("-r", action="store", type="string", dest="start",
10                  default='',
11                  help="root directory to start running unittests")
12parser.add_option("--full", action="store_true", dest="full", default=False,
13                  help="whether to run the shortened version of the test")
14parser.add_option("--debug", action="store_true", dest="debug", default=False,
15                  help="run in debug mode")
16parser.add_option("--skip-tests", dest="skip_tests",  default=[],
17                  help="A space separated list of tests to skip")
18
19parser.set_defaults(module_list=None)
20
21# Following sets are used to define a collection of modules that are optional
22# tests and do not need to be executed in unittest suite for various reasons.
23# Each entry can be file name or relative path that's relative to the parent
24# folder of the folder containing this file (unittest_suite.py). The list
25# will be used to filter any test file with matching name or matching full
26# path. If a file's name is too general and has a chance to collide with files
27# in other folder, it is recommended to specify its relative path here, e.g.,
28# using 'mirror/trigger_unittest.py', instead of 'trigger_unittest.py' only.
29
30REQUIRES_DJANGO = set((
31        'monitor_db_unittest.py',
32        'monitor_db_functional_test.py',
33        'monitor_db_cleanup_test.py',
34        'frontend_unittest.py',
35        'csv_encoder_unittest.py',
36        'rpc_interface_unittest.py',
37        'models_test.py',
38        'scheduler_models_unittest.py',
39        'metahost_scheduler_unittest.py',
40        'site_metahost_scheduler_unittest.py',
41        'rpc_utils_unittest.py',
42        'site_rpc_utils_unittest.py',
43        'execution_engine_unittest.py',
44        'service_proxy_lib_test.py',
45        ))
46
47REQUIRES_MYSQLDB = set((
48        'migrate_unittest.py',
49        'db_utils_unittest.py',
50        ))
51
52REQUIRES_GWT = set((
53        'client_compilation_unittest.py',
54        ))
55
56REQUIRES_SIMPLEJSON = set((
57        'resources_test.py',
58        'serviceHandler_unittest.py',
59        ))
60
61REQUIRES_AUTH = set ((
62    'trigger_unittest.py',
63    ))
64
65REQUIRES_HTTPLIB2 = set((
66        ))
67
68REQUIRES_PROTOBUFS = set((
69        'job_serializer_unittest.py',
70        ))
71
72REQUIRES_SELENIUM = set((
73        'ap_configurator_factory_unittest.py',
74        'chaos_base_test_unittest.py',
75        'ap_batch_locker_unittest.py'
76    ))
77
78LONG_RUNTIME = set((
79    'auth_server_unittest.py',
80    'base_barrier_unittest.py',
81    'logging_manager_test.py',
82    ))
83
84
85SKIP = set((
86    # This particular KVM autotest test is not a unittest
87    'guest_test.py',
88    'ap_configurator_test.py',
89    'chaos_base_test.py',
90    'chaos_interop_test.py',
91    'monitor_db_functional_test.py'
92    ))
93
94LONG_TESTS = (REQUIRES_MYSQLDB |
95              REQUIRES_GWT |
96              REQUIRES_HTTPLIB2 |
97              REQUIRES_AUTH |
98              REQUIRES_PROTOBUFS |
99              REQUIRES_SELENIUM |
100              LONG_RUNTIME)
101
102ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
103
104# The set of files in LONG_TESTS with its full path
105LONG_TESTS_FULL_PATH = {os.path.join(ROOT, t) for t in LONG_TESTS}
106
107class TestFailure(Exception):
108    """Exception type for any test failure."""
109    pass
110
111
112def run_test(mod_names, options):
113    """
114    @param mod_names: A list of individual parts of the module name to import
115            and run as a test suite.
116    @param options: optparse options.
117    """
118    if not options.debug:
119        parallel.redirect_io()
120
121    print "Running %s" % '.'.join(mod_names)
122    mod = common.setup_modules.import_module(mod_names[-1],
123                                             '.'.join(mod_names[:-1]))
124    for ut_module in [unittest, custom_unittest]:
125        test = ut_module.defaultTestLoader.loadTestsFromModule(mod)
126        suite = ut_module.TestSuite(test)
127        runner = ut_module.TextTestRunner(verbosity=2)
128        result = runner.run(suite)
129        if result.errors or result.failures:
130            msg = '%s had %d failures and %d errors.'
131            msg %= '.'.join(mod_names), len(result.failures), len(result.errors)
132            raise TestFailure(msg)
133
134
135def scan_for_modules(start, options):
136    """Scan folders and find all test modules that are not included in the
137    blacklist (defined in LONG_TESTS).
138
139    @param start: The absolute directory to look for tests under.
140    @param options: optparse options.
141    @return a list of modules to be executed.
142    """
143    modules = []
144
145    skip_tests = SKIP
146    if options.skip_tests:
147        skip_tests.update(options.skip_tests.split())
148    skip_tests_full_path = {os.path.join(ROOT, t) for t in skip_tests}
149
150    for dir_path, sub_dirs, file_names in os.walk(start):
151        # Only look in and below subdirectories that are python modules.
152        if '__init__.py' not in file_names:
153            if options.full:
154                for file_name in file_names:
155                    if file_name.endswith('.pyc'):
156                        os.unlink(os.path.join(dir_path, file_name))
157            # Skip all subdirectories below this one, it is not a module.
158            del sub_dirs[:]
159            if options.debug:
160                print 'Skipping', dir_path
161            continue  # Skip this directory.
162
163        # Look for unittest files.
164        for file_name in file_names:
165            if (file_name.endswith('_unittest.py') or
166                file_name.endswith('_test.py')):
167                file_path = os.path.join(dir_path, file_name)
168                if (not options.full and
169                    (file_name in LONG_TESTS or
170                     file_path in LONG_TESTS_FULL_PATH)):
171                    continue
172                if (file_name in skip_tests or
173                    file_path in skip_tests_full_path):
174                    continue
175                path_no_py = os.path.join(dir_path, file_name).rstrip('.py')
176                assert path_no_py.startswith(ROOT)
177                names = path_no_py[len(ROOT)+1:].split('/')
178                modules.append(['autotest_lib'] + names)
179                if options.debug:
180                    print 'testing', path_no_py
181    return modules
182
183def find_and_run_tests(start, options):
184    """
185    Find and run Python unittest suites below the given directory.  Only look
186    in subdirectories of start that are actual importable Python modules.
187
188    @param start: The absolute directory to look for tests under.
189    @param options: optparse options.
190    """
191    if options.module_list:
192        modules = []
193        for m in options.module_list:
194            modules.append(m.split('.'))
195    else:
196        modules = scan_for_modules(start, options)
197
198    if options.debug:
199        print 'Number of test modules found:', len(modules)
200
201    functions = {}
202    for module_names in modules:
203        # Create a function that'll test a particular module.  module=module
204        # is a hack to force python to evaluate the params now.  We then
205        # rename the function to make error reporting nicer.
206        run_module = lambda module=module_names: run_test(module, options)
207        name = '.'.join(module_names)
208        run_module.__name__ = name
209        functions[run_module] = set()
210
211    try:
212        dargs = {}
213        if options.debug:
214            dargs['max_simultaneous_procs'] = 1
215        pe = parallel.ParallelExecute(functions, **dargs)
216        pe.run_until_completion()
217    except parallel.ParallelError, err:
218        return err.errors
219    return []
220
221
222def main():
223    """Entry point for unittest_suite.py"""
224    options, args = parser.parse_args()
225    if args:
226        options.module_list = args
227
228    # Strip the arguments off the command line, so that the unit tests do not
229    # see them.
230    del sys.argv[1:]
231
232    absolute_start = os.path.join(ROOT, options.start)
233    errors = find_and_run_tests(absolute_start, options)
234    if errors:
235        print "%d tests resulted in an error/failure:" % len(errors)
236        for error in errors:
237            print "\t%s" % error
238        print "Rerun", sys.argv[0], "--debug to see the failure details."
239        sys.exit(1)
240    else:
241        print "All passed!"
242        sys.exit(0)
243
244
245if __name__ == "__main__":
246    main()
247