autotest.py revision 1b3b376d94edee8f235f8669a047dc00751bf97a 
1#!/usr/bin/python
2#
3# Copyright 2007 Google Inc. Released under the GPL v2
4
5"""
6This module defines the Autotest class
7
8        Autotest: software to run tests automatically
9"""
10
11__author__ = """
12mbligh@google.com (Martin J. Bligh),
13poirier@google.com (Benjamin Poirier),
14stutsman@google.com (Ryan Stutsman)
15"""
16
17import re, os, sys, traceback, subprocess, tempfile, shutil, time, pickle
18
19from autotest_lib.server import installable_object, utils, server_job
20from autotest_lib.client.common_lib import log
21from autotest_lib.client.common_lib import error, global_config, packages
22
23
24
25AUTOTEST_SVN  = 'svn://test.kernel.org/autotest/trunk/client'
26AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client'
27
28# Timeouts for powering down and up respectively
29HALT_TIME = 300
30BOOT_TIME = 1800
31CRASH_RECOVERY_TIME = 9000
32
33
34class BaseAutotest(installable_object.InstallableObject):
35    """
36    This class represents the Autotest program.
37
38    Autotest is used to run tests automatically and collect the results.
39    It also supports profilers.
40
41    Implementation details:
42    This is a leaf class in an abstract class hierarchy, it must
43    implement the unimplemented methods in parent classes.
44    """
45
46    def __init__(self, host = None):
47        self.host = host
48        self.got = False
49        self.installed = False
50        self.serverdir = utils.get_server_dir()
51        super(BaseAutotest, self).__init__()
52
53
54    @log.record
55    def install(self, host = None):
56        """
57        Install autotest.  If get() was not called previously, an
58        attempt will be made to install from the autotest svn
59        repository.
60
61        Args:
62                host: a Host instance on which autotest will be
63                        installed
64
65        Raises:
66                AutoservError: if a tarball was not specified and
67                        the target host does not have svn installed in its path
68
69        TODO(poirier): check dependencies
70        autotest needs:
71        bzcat
72        liboptdev (oprofile)
73        binutils-dev (oprofile)
74        make
75        psutils (netperf)
76        """
77        if not host:
78            host = self.host
79        if not self.got:
80            self.get()
81        host.wait_up(timeout=30)
82        host.setup()
83        print "Installing autotest on %s" % host.hostname
84
85        # Let's try to figure out where autotest is installed. If we can't,
86        # (autotest not installed) just assume '/usr/local/autotest' and
87        # proceed.
88        try:
89            autodir = _get_autodir(host)
90        except error.AutotestRunError:
91            autodir = '/usr/local/autotest'
92
93        # Make the host object know as to where autotest is installed
94        host.set_autodir(autodir)
95
96        host.run('mkdir -p "%s"' % utils.sh_escape(autodir))
97
98        # Fetch the autotest client from the nearest repository
99        try:
100            c = global_config.global_config
101            repos = c.get_config_value("PACKAGES", 'fetch_location', type=list)
102            pkgmgr = packages.PackageManager(
103                autodir, repo_urls=repos, do_locking=False,
104                run_function=host.run,
105                run_function_dargs=dict(timeout=600))
106            # The packages dir is used to store all the packages that
107            # are fetched on that client. (for the tests,deps etc.
108            # too apart from the client)
109            pkg_dir = os.path.join(autodir, 'packages')
110            # clean up the autodir except for the packages directory
111            host.run('cd %s && ls | grep -v "^packages$"'
112                     ' | xargs rm -rf && rm -rf .[^.]*' % autodir)
113            pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir,
114                               preserve_install_dir=True)
115            self.installed = True
116            return
117        except global_config.ConfigError, e:
118            print ("Could not install autotest using the"
119                   " packaging system %s" %  e)
120        except (packages.PackageInstallError, error.AutoservRunError), e:
121            print "Could not install autotest from %s : %s " % (repos, e)
122
123
124        # try to install from file or directory
125        if self.source_material:
126            if os.path.isdir(self.source_material):
127                # Copy autotest recursively
128                host.send_file(self.source_material, autodir)
129            else:
130                # Copy autotest via tarball
131                e_msg = 'Installation method not yet implemented!'
132                raise NotImplementedError(e_msg)
133            print "Installation of autotest completed"
134            self.installed = True
135            return
136
137        # if that fails try to install using svn
138        if utils.run('which svn').exit_status:
139            raise error.AutoservError('svn not found on target machine: %s'
140                                                                   % host.name)
141        try:
142            host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir))
143        except error.AutoservRunError, e:
144            host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir))
145        print "Installation of autotest completed"
146        self.installed = True
147
148
149    def get(self, location = None):
150        if not location:
151            location = os.path.join(self.serverdir, '../client')
152            location = os.path.abspath(location)
153        # If there's stuff run on our client directory already, it
154        # can cause problems. Try giving it a quick clean first.
155        cwd = os.getcwd()
156        os.chdir(location)
157        os.system('tools/make_clean')
158        os.chdir(cwd)
159        super(BaseAutotest, self).get(location)
160        self.got = True
161
162
163    def run(self, control_file, results_dir = '.', host = None,
164            timeout=None, tag=None, parallel_flag=False):
165        """
166        Run an autotest job on the remote machine.
167
168        Args:
169                control_file: an open file-like-obj of the control file
170                results_dir: a str path where the results should be stored
171                        on the local filesystem
172                host: a Host instance on which the control file should
173                        be run
174                tag: tag name for the client side instance of autotest
175                parallel_flag: flag set when multiple jobs are run at the
176                          same time
177        Raises:
178                AutotestRunError: if there is a problem executing
179                        the control file
180        """
181        host = self._get_host_and_setup(host)
182        results_dir = os.path.abspath(results_dir)
183
184        if tag:
185            results_dir = os.path.join(results_dir, tag)
186
187        atrun = _Run(host, results_dir, tag, parallel_flag)
188        self._do_run(control_file, results_dir, host, atrun, timeout)
189
190
191    def _get_host_and_setup(self, host):
192        if not host:
193            host = self.host
194        if not self.installed:
195            self.install(host)
196
197        host.wait_up(timeout=30)
198        return host
199
200
201    def _do_run(self, control_file, results_dir, host, atrun, timeout):
202        try:
203            atrun.verify_machine()
204        except:
205            print "Verify machine failed on %s. Reinstalling" % host.hostname
206            self.install(host)
207        atrun.verify_machine()
208        debug = os.path.join(results_dir, 'debug')
209        try:
210            os.makedirs(debug)
211        except:
212            pass
213
214        # Ready .... Aim ....
215        for control in [atrun.remote_control_file,
216                        atrun.remote_control_file + '.state',
217                        atrun.manual_control_file,
218                        atrun.manual_control_file + '.state']:
219            host.run('rm -f ' + control)
220
221        tmppath = utils.get(control_file)
222
223
224        # Prepend the control file with a job.disable_test_cleanup call
225        # if necessary
226        if not host.job.run_test_cleanup:
227            cfile = open(tmppath).read()
228            cfile = "job.disable_test_cleanup()\n" + cfile
229            open(tmppath, "w").write(cfile)
230
231        # Insert the job.add_repository() lines in the control file
232        # if there are any repos defined in global_config.ini
233        try:
234            cfile = open(tmppath, 'r')
235            cfile_orig = cfile.read()
236            cfile.close()
237            c = global_config.global_config
238            repos = c.get_config_value("PACKAGES", 'fetch_location', type=list)
239            control_file_new = []
240            control_file_new.append('job.add_repository(%s)\n' % repos)
241            control_file_new.append(cfile_orig)
242
243            # Overwrite the control file with the new one
244            cfile = open(tmppath, 'w')
245            cfile.write('\n'.join(control_file_new))
246            cfile.close()
247        except global_config.ConfigError, e:
248            pass
249
250
251        # Copy control_file to remote_control_file on the host
252        host.send_file(tmppath, atrun.remote_control_file)
253        if os.path.abspath(tmppath) != os.path.abspath(control_file):
254            os.remove(tmppath)
255
256        try:
257            atrun.execute_control(timeout=timeout)
258        finally:
259            collector = server_job.log_collector(host, atrun.tag, results_dir)
260            collector.collect_client_job_results()
261            self._process_client_state_file(host, atrun, results_dir)
262
263
264    def _process_client_state_file(self, host, atrun, results_dir):
265        state_file = os.path.basename(atrun.remote_control_file) + ".state"
266        state_path = os.path.join(results_dir, state_file)
267        try:
268            state_dict = pickle.load(open(state_path))
269        except Exception:
270            print >> sys.stderr, "Error while loading client job state file"
271            traceback.print_exc()
272            state_dict = {}
273
274        # clear out the state file
275        # TODO: stash the file away somewhere useful instead
276        try:
277            os.remove(state_path)
278        except Exception:
279            pass
280
281        msg = "Persistant state variables pulled back from %s: %s"
282        msg %= (host.hostname, state_dict)
283        print msg
284
285        if "__run_test_cleanup" in state_dict:
286            if state_dict["__run_test_cleanup"]:
287                host.job.enable_test_cleanup()
288            else:
289                host.job.disable_test_cleanup()
290
291
292    def run_timed_test(self, test_name, results_dir='.', host=None,
293                       timeout=None, tag=None, *args, **dargs):
294        """
295        Assemble a tiny little control file to just run one test,
296        and run it as an autotest client-side test
297        """
298        if not host:
299            host = self.host
300        if not self.installed:
301            self.install(host)
302        opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()]
303        cmd = ", ".join([repr(test_name)] + map(repr, args) + opts)
304        control = "job.run_test(%s)\n" % cmd
305        self.run(control, results_dir, host, timeout=timeout, tag=tag)
306
307
308    def run_test(self, test_name, results_dir='.', host=None, tag=None,
309                 *args, **dargs):
310        self.run_timed_test(test_name, results_dir, host, timeout=None,
311                            tag=tag, *args, **dargs)
312
313
314class _Run(object):
315    """
316    Represents a run of autotest control file.  This class maintains
317    all the state necessary as an autotest control file is executed.
318
319    It is not intended to be used directly, rather control files
320    should be run using the run method in Autotest.
321    """
322    def __init__(self, host, results_dir, tag, parallel_flag):
323        self.host = host
324        self.results_dir = results_dir
325        self.env = host.env
326        self.tag = tag
327        self.parallel_flag = parallel_flag
328        self.autodir = _get_autodir(self.host)
329        control = os.path.join(self.autodir, 'control')
330        if tag:
331            control += '.' + tag
332        self.manual_control_file = control
333        self.remote_control_file = control + '.autoserv'
334
335
336    def verify_machine(self):
337        binary = os.path.join(self.autodir, 'bin/autotest')
338        try:
339            self.host.run('ls %s > /dev/null 2>&1' % binary)
340        except:
341            raise "Autotest does not appear to be installed"
342
343        if not self.parallel_flag:
344            tmpdir = os.path.join(self.autodir, 'tmp')
345            download = os.path.join(self.autodir, 'tests/download')
346            self.host.run('umount %s' % tmpdir, ignore_status=True)
347            self.host.run('umount %s' % download, ignore_status=True)
348
349    def get_full_cmd(self, section):
350        # build up the full command we want to run over the host
351        cmd = [os.path.join(self.autodir, 'bin/autotest_client'),
352               '-H autoserv']
353        if section > 0:
354            cmd.append('-c')
355        if self.tag:
356            cmd.append('-t %s' % self.tag)
357        if self.host.job.use_external_logging():
358            cmd.append('-l')
359        cmd.append(self.remote_control_file)
360        return ' '.join(cmd)
361
362
363    def get_client_log(self, section):
364        # open up the files we need for our logging
365        client_log_file = os.path.join(self.results_dir, 'debug',
366                                       'client.log.%d' % section)
367        return open(client_log_file, 'w', 0)
368
369
370    def execute_section(self, section, timeout):
371        print "Executing %s/bin/autotest %s/control phase %d" % \
372                                (self.autodir, self.autodir, section)
373
374        full_cmd = self.get_full_cmd(section)
375        client_log = self.get_client_log(section)
376        redirector = server_job.client_logger(self.host, self.tag,
377                                              self.results_dir)
378
379        try:
380            old_resultdir = self.host.job.resultdir
381            self.host.job.resultdir = self.results_dir
382            result = self.host.run(full_cmd, ignore_status=True,
383                                   timeout=timeout,
384                                   stdout_tee=client_log,
385                                   stderr_tee=redirector)
386        finally:
387            redirector.close()
388            self.host.job.resultdir = old_resultdir
389
390        if result.exit_status == 1:
391            self.host.job.aborted = True
392            raise error.AutotestRunError("client job was aborted")
393        if not result.stderr:
394            raise error.AutotestRunError(
395                "execute_section: %s failed to return anything\n"
396                "stdout:%s\n" % (full_cmd, result.stdout))
397
398        return redirector.last_line
399
400
401    def execute_control(self, timeout=None):
402        section = 0
403        time_left = None
404        if timeout:
405            end_time = time.time() + timeout
406            time_left = end_time - time.time()
407        while not timeout or time_left > 0:
408            last = self.execute_section(section, time_left)
409            if timeout:
410                time_left = end_time - time.time()
411                if time_left <= 0:
412                    break
413            section += 1
414            if re.match(r'^END .*\t----\t----\t.*$', last):
415                print "Client complete"
416                return
417            elif re.match('^\t*GOOD\t----\treboot\.start.*$', last):
418                print "Client is rebooting"
419                print "Waiting for client to halt"
420                if not self.host.wait_down(HALT_TIME):
421                    err = "%s failed to shutdown after %d"
422                    err %= (self.host.hostname, HALT_TIME)
423                    raise error.AutotestRunError(err)
424                print "Client down, waiting for restart"
425                if not self.host.wait_up(BOOT_TIME):
426                    # since reboot failed
427                    # hardreset the machine once if possible
428                    # before failing this control file
429                    print "Hardresetting %s" % self.host.hostname
430                    try:
431                        self.host.hardreset(wait=False)
432                    except error.AutoservUnsupportedError:
433                        print "Hardreset unsupported on %s" % self.host.hostname
434                    raise error.AutotestRunError("%s failed to boot after %ds" %
435                                                (self.host.hostname, BOOT_TIME))
436                self.host.reboot_followup()
437                continue
438            self.host.job.record("END ABORT", None, None,
439                                 "Autotest client terminated unexpectedly")
440            # give the client machine a chance to recover from
441            # possible crash
442            self.host.wait_up(CRASH_RECOVERY_TIME)
443            raise error.AutotestRunError("Aborting - unexpected final status "
444                                         "message from client: %s\n" % last)
445
446        # should only get here if we timed out
447        assert timeout
448        raise error.AutotestTimeoutError()
449
450
451def _get_autodir(host):
452    autodir = host.get_autodir()
453    if autodir:
454        return autodir
455    try:
456        # There's no clean way to do this. readlink may not exist
457        cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf 2> /dev/null"
458        autodir = os.path.dirname(host.run(cmd).stdout)
459        if autodir:
460            return autodir
461    except error.AutoservRunError:
462        pass
463    for path in ['/usr/local/autotest', '/home/autotest']:
464        try:
465            host.run('ls %s > /dev/null 2>&1' %
466                     os.path.join(path, 'bin/autotest'))
467            return path
468        except error.AutoservRunError:
469            pass
470    raise error.AutotestRunError("Cannot figure out autotest directory")
471
472
473# site_autotest.py may be non-existant or empty, make sure that an appropriate
474# SiteAutotest class is created nevertheless
475try:
476    from site_autotest import SiteAutotest
477except ImportError:
478    class SiteAutotest(BaseAutotest):
479        pass
480
481
482class Autotest(SiteAutotest):
483    pass
484