test_that.py revision 30322f95d61f6a2cffada92472729d3f2a926f6b 
1#!/usr/bin/python
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import argparse
7import logging
8import os
9import re
10import signal
11import stat
12import subprocess
13import sys
14import tempfile
15import threading
16
17import common
18from autotest_lib.client.common_lib.cros import dev_server, retry
19from autotest_lib.server.cros.dynamic_suite import suite
20from autotest_lib.server.cros.dynamic_suite import constants
21from autotest_lib.server import autoserv_utils
22
23try:
24    from chromite.lib import cros_build_lib
25except ImportError:
26    print 'Unable to import chromite.'
27    print 'This script must be either:'
28    print '  - Be run in the chroot.'
29    print '  - (not yet supported) be run after running '
30    print '    ../utils/build_externals.py'
31
32_autoserv_proc = None
33_sigint_handler_lock = threading.Lock()
34
35_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5
36_NO_BOARD = 'ad_hoc_board'
37_NO_BUILD = 'ad_hoc_build'
38
39_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge'
40
41_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report'
42
43
44def schedule_local_suite(autotest_path, suite_name, afe, build=_NO_BUILD,
45                         board=_NO_BOARD, results_directory=None):
46    """
47    Schedule a suite against a mock afe object, for a local suite run.
48    @param autotest_path: Absolute path to autotest (in sysroot).
49    @param suite_name: Name of suite to schedule.
50    @param afe: afe object to schedule against (typically a directAFE)
51    @param build: Build to schedule suite for.
52    @param board: Board to schedule suite for.
53    @param results_directory: Absolute path of directory to store results in.
54                              (results will be stored in subdirectory of this).
55    @returns: The number of tests scheduled.
56    """
57    fs_getter = suite.Suite.create_fs_getter(autotest_path)
58    devserver = dev_server.ImageServer('')
59    my_suite = suite.Suite.create_from_name(suite_name, build, board,
60            devserver, fs_getter, afe=afe, ignore_deps=True,
61            results_dir=results_directory)
62    if len(my_suite.tests) == 0:
63        raise ValueError('Suite named %s does not exist, or contains no '
64                         'tests.' % suite_name)
65    my_suite.schedule(lambda x: None) # Schedule tests, discard record calls.
66    return len(my_suite.tests)
67
68
69def schedule_local_test(autotest_path, test_name, afe, build=_NO_BUILD,
70                        board=_NO_BOARD, results_directory=None):
71    #temporarily disabling pylint
72    #pylint: disable-msg=C0111
73    """
74    Schedule an individual test against a mock afe object, for a local run.
75    @param autotest_path: Absolute path to autotest (in sysroot).
76    @param test_name: Name of test to schedule.
77    @param afe: afe object to schedule against (typically a directAFE)
78    @param build: Build to schedule suite for.
79    @param board: Board to schedule suite for.
80    @param results_directory: Absolute path of directory to store results in.
81                              (results will be stored in subdirectory of this).
82    @returns: The number of tests scheduled (may be >1 if there are
83              multiple tests with the same name).
84    """
85    fs_getter = suite.Suite.create_fs_getter(autotest_path)
86    devserver = dev_server.ImageServer('')
87    predicates = [suite.Suite.test_name_equals_predicate(test_name)]
88    suite_name = 'suite_' + test_name
89    my_suite = suite.Suite.create_from_predicates(predicates, build, board,
90            devserver, fs_getter, afe=afe, name=suite_name, ignore_deps=True,
91            results_dir=results_directory)
92    if len(my_suite.tests) == 0:
93        raise ValueError('No tests named %s.' % test_name)
94    my_suite.schedule(lambda x: None) # Schedule tests, discard record calls.
95    return len(my_suite.tests)
96
97
98def run_job(job, host, sysroot_autotest_path, results_directory, fast_mode,
99            id_digits=1, args=None):
100    """
101    Shell out to autoserv to run an individual test job.
102
103    @param job: A Job object containing the control file contents and other
104                relevent metadata for this test.
105    @param host: Hostname of DUT to run test against.
106    @param sysroot_autotest_path: Absolute path of autotest directory.
107    @param results_directory: Absolute path of directory to store results in.
108                              (results will be stored in subdirectory of this).
109    @param fast_mode: bool to use fast mode (disables slow autotest features).
110    @param id_digits: The minimum number of digits that job ids should be
111                      0-padded to when formatting as a string for results
112                      directory.
113    @param args: String that should be passed as args parameter to autoserv,
114                 and then ultimitely to test itself.
115    @returns: Absolute path of directory where results were stored.
116    """
117    with tempfile.NamedTemporaryFile() as temp_file:
118        temp_file.write(job.control_file)
119        temp_file.flush()
120        results_directory = os.path.join(results_directory,
121                                         'results-%0*d' % (id_digits, job.id))
122        extra_args = [temp_file.name]
123        if args:
124            extra_args.extend(['--args', args])
125
126        command = autoserv_utils.autoserv_run_job_command(
127                os.path.join(sysroot_autotest_path, 'server'),
128                machines=host, job=job, verbose=False,
129                results_directory=results_directory,
130                fast_mode=fast_mode,
131                extra_args=extra_args)
132        global _autoserv_proc
133        _autoserv_proc = subprocess.Popen(command)
134        _autoserv_proc.wait()
135        _autoserv_proc = None
136        return results_directory
137
138
139def setup_local_afe():
140    """
141    Setup a local afe database and return a direct_afe object to access it.
142
143    @returns: A autotest_lib.frontend.afe.direct_afe instance.
144    """
145    # This import statement is delayed until now rather than running at
146    # module load time, because it kicks off a local sqlite :memory: backed
147    # database, and we don't need that unless we are doing a local run.
148    from autotest_lib.frontend import setup_django_lite_environment
149    from autotest_lib.frontend.afe import direct_afe
150    return direct_afe.directAFE()
151
152
153def perform_local_run(afe, autotest_path, tests, remote, fast_mode,
154                      build=_NO_BUILD, board=_NO_BOARD, args=None):
155    """
156    @param afe: A direct_afe object used to interact with local afe database.
157    @param autotest_path: Absolute path of sysroot installed autotest.
158    @param tests: List of strings naming tests and suites to run. Suite strings
159                  should be formed like "suite:smoke".
160    @param remote: Remote hostname.
161    @param fast_mode: bool to use fast mode (disables slow autotest features).
162    @param build: String specifying build for local run.
163    @param board: String specifyinb board for local run.
164    @param args: String that should be passed as args parameter to autoserv,
165                 and then ultimitely to test itself.
166
167    @returns: directory in which results are stored.
168    """
169    afe.create_label(constants.VERSION_PREFIX + build)
170    afe.create_label(board)
171    afe.create_host(remote)
172
173    results_directory = tempfile.mkdtemp(prefix='test_that_results_')
174    os.chmod(results_directory, stat.S_IWOTH | stat.S_IROTH | stat.S_IXOTH)
175    logging.info('Running jobs. Results will be placed in %s',
176                 results_directory)
177    # Schedule tests / suites in local afe
178    for test in tests:
179        suitematch = re.match(r'suite:(.*)', test)
180        if suitematch:
181            suitename = suitematch.group(1)
182            logging.info('Scheduling suite %s...', suitename)
183            ntests = schedule_local_suite(autotest_path, suitename, afe,
184                                          build=build, board=board,
185                                          results_directory=results_directory)
186        else:
187            logging.info('Scheduling test %s...', test)
188            ntests = schedule_local_test(autotest_path, test, afe,
189                                         build=build, board=board,
190                                         results_directory=results_directory)
191        logging.info('... scheduled %s tests.', ntests)
192
193    if not afe.get_jobs():
194        logging.info('No jobs scheduled. End of local run.')
195        return results_directory
196
197    last_job_id = afe.get_jobs()[-1].id
198    job_id_digits=len(str(last_job_id))
199    for job in afe.get_jobs():
200        run_job(job, remote, autotest_path, results_directory, fast_mode,
201                job_id_digits, args)
202
203    return results_directory
204
205
206def validate_arguments(arguments):
207    """
208    Validates parsed arguments.
209
210    @param arguments: arguments object, as parsed by ParseArguments
211    @raises: ValueError if arguments were invalid.
212    """
213    if arguments.build:
214        raise ValueError('-i/--build flag not yet supported.')
215
216    if not arguments.board:
217        raise ValueError('Board autodetection not yet supported. '
218                         '--board required.')
219
220    if arguments.remote == ':lab:':
221        raise ValueError('Running tests in test lab not yet supported.')
222        if arguments.args:
223            raise ValueError('--args flag not supported when running against '
224                             ':lab:')
225
226
227def parse_arguments(argv):
228    """
229    Parse command line arguments
230
231    @param argv: argument list to parse
232    @returns:    parsed arguments.
233    """
234    parser = argparse.ArgumentParser(description='Run remote tests.')
235
236    parser.add_argument('remote', metavar='REMOTE',
237                        help='hostname[:port] for remote device. Specify '
238                        ':lab: to run in test lab, or :vm:PORT_NUMBER to '
239                        'run in vm.')
240    parser.add_argument('tests', nargs='+', metavar='TEST',
241                        help='Run given test(s). Use suite:SUITE to specify '
242                        'test suite.')
243    parser.add_argument('-b', '--board', metavar='BOARD',
244                        action='store',
245                        help='Board for which the test will run.')
246    parser.add_argument('-i', '--build', metavar='BUILD',
247                        help='Build to test. Device will be reimaged if '
248                        'necessary. Omit flag to skip reimage and test '
249                        'against already installed DUT image.')
250    parser.add_argument('--fast', action='store_true', dest='fast_mode',
251                        default=False,
252                        help='Enable fast mode.  This will cause test_that to '
253                             'skip time consuming steps like sysinfo and '
254                             'collecting crash information.')
255    parser.add_argument('--args', metavar='ARGS',
256                        help='Argument string to pass through to test. Only '
257                        'supported for runs against a local DUT.')
258
259    return parser.parse_args(argv)
260
261
262def sigint_handler(signum, stack_frame):
263    #pylint: disable-msg=C0111
264    """Handle SIGINT or SIGTERM to a local test_that run.
265
266    This handler sends a SIGINT to the running autoserv process,
267    if one is running, giving it up to 5 seconds to clean up and exit. After
268    the timeout elapses, autoserv is killed. In either case, after autoserv
269    exits then this process exits with status 1.
270    """
271    # If multiple signals arrive before handler is unset, ignore duplicates
272    if not _sigint_handler_lock.acquire(False):
273        return
274    try:
275        # Ignore future signals by unsetting handler.
276        signal.signal(signal.SIGINT, signal.SIG_IGN)
277        signal.signal(signal.SIGTERM, signal.SIG_IGN)
278
279        logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.')
280        if _autoserv_proc:
281            logging.warning('Sending SIGINT to autoserv process. Waiting up '
282                            'to %s seconds for cleanup.',
283                            _AUTOSERV_SIGINT_TIMEOUT_SECONDS)
284            _autoserv_proc.send_signal(signal.SIGINT)
285            timed_out, _ = retry.timeout(_autoserv_proc.wait,
286                    timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS)
287            if timed_out:
288                _autoserv_proc.kill()
289                logging.warning('Timed out waiting for autoserv to handle '
290                                'SIGINT. Killed autoserv.')
291    finally:
292        _sigint_handler_lock.release() # this is not really necessary?
293        sys.exit(1)
294
295
296def main(argv):
297    """
298    Entry point for test_that script.
299    @param argv: arguments list
300    """
301    if not cros_build_lib.IsInsideChroot():
302        logging.error('Script must be invoked inside the chroot.')
303        return 1
304
305    logging.getLogger('').setLevel(logging.INFO)
306
307    arguments = parse_arguments(argv)
308    try:
309        validate_arguments(arguments)
310    except ValueError as err:
311        logging.error('Invalid arguments. %s', err.message)
312        return 1
313
314    # TODO: Determine the following string programatically.
315    # (same TODO applied to autotest_quickmerge)
316    sysroot_path = os.path.join('/build', arguments.board, '')
317    sysroot_autotest_path = os.path.join(sysroot_path, 'usr', 'local',
318                                         'autotest', '')
319    sysroot_site_utils_path = os.path.join(sysroot_autotest_path,
320                                            'site_utils')
321
322    if not os.path.exists(sysroot_path):
323        logging.error('%s does not exist. Have you run setup_board?',
324                      sysroot_path)
325        return 1
326    if not os.path.exists(sysroot_autotest_path):
327        logging.error('%s does not exist. Have you run build_packages?',
328                      sysroot_autotest_path)
329        return 1
330
331    # If we are not running the sysroot version of script, perform
332    # a quickmerge if necessary and then re-execute
333    # the sysroot version of script with the same arguments.
334    realpath = os.path.realpath(__file__)
335    if os.path.dirname(realpath) != sysroot_site_utils_path:
336        subprocess.call([_QUICKMERGE_SCRIPTNAME, '--board='+arguments.board])
337
338        script_command = os.path.join(sysroot_site_utils_path,
339                                      os.path.basename(realpath))
340        proc = None
341        def resend_sig(signum, stack_frame):
342            #pylint: disable-msg=C0111
343            if proc:
344                proc.send_signal(signum)
345        signal.signal(signal.SIGINT, resend_sig)
346        signal.signal(signal.SIGTERM, resend_sig)
347
348        proc = subprocess.Popen([script_command] + argv)
349
350        return proc.wait()
351
352    # Hard coded to True temporarily. This will eventually be parsed to false
353    # if we are doing a run in the test lab.
354    local_run = True
355
356    signal.signal(signal.SIGINT, sigint_handler)
357    signal.signal(signal.SIGTERM, sigint_handler)
358
359    if local_run:
360        afe = setup_local_afe()
361        res_dir= perform_local_run(afe, sysroot_autotest_path, arguments.tests,
362                                   arguments.remote, arguments.fast_mode,
363                                   args=arguments.args)
364        final_result = subprocess.call([_TEST_REPORT_SCRIPTNAME, res_dir])
365        logging.info('Finished running tests. Results can be found in %s',
366                     res_dir)
367        return final_result
368
369
370if __name__ == '__main__':
371    sys.exit(main(sys.argv[1:]))
372