autoserv.py revision 21baa459ea14f96e06212f1f35fcddab9442b3fc
1a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora#!/usr/bin/python -u
27c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora#
30406ce1417f76f2034833414dcecc9f56253640cVikas Arora# Copyright 2007 Google Inc. Released under the GPL v2
40406ce1417f76f2034833414dcecc9f56253640cVikas Arora
50406ce1417f76f2034833414dcecc9f56253640cVikas Arora"""
60406ce1417f76f2034833414dcecc9f56253640cVikas AroraRun an control file through the server side engine
70406ce1417f76f2034833414dcecc9f56253640cVikas Arora"""
87c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
97c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora__author__ = """\
10a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arorambligh@google.com (Martin J. Bligh)
117c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora"""
127c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
137c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Aroraimport sys, os, re, traceback, signal
148b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
15a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Aroraimport common
168b720228d581a84fd173b6dcb2fa295b59db489aVikas Arorafrom autotest_lib.server import server_job, utils, autoserv_parser
17a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora
18fa39824bb690c5806358871f46940d0450973d8aJames Zern
197c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Aroraclass PidFileManager(object):
201e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    pid_file = None
211e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    num_tests_failed = 0
22b6dbce6bfeaabde2a7b581c4c6888d532d32f3acDerek Sollenberger
23b6dbce6bfeaabde2a7b581c4c6888d532d32f3acDerek Sollenberger    def open_pid_file(self, results_dir):
241e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        pid_file_path = os.path.join(results_dir, '.autoserv_execute')
251e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        assert not os.path.exists(pid_file_path)
26b6dbce6bfeaabde2a7b581c4c6888d532d32f3acDerek Sollenberger        self.pid_file = open(pid_file_path, 'w')
27b6dbce6bfeaabde2a7b581c4c6888d532d32f3acDerek Sollenberger        self.pid_file.write(str(os.getpid()) + '\n')
281e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        self.pid_file.flush()
291e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
301e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
311e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    def close_pid_file(self, exit_code, signal_code=0):
32a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora        if not self.pid_file:
33a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora            return
34a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora        real_exit_code = (exit_code << 8) | (signal_code & 0xFF)
35a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora        self.pid_file.write(str(real_exit_code) + '\n')
36a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora        self.pid_file.write(str(self.num_tests_failed) + '\n')
37a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora        self.pid_file.close()
38466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora        self.pid_file = None
39a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora
40a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora
41a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Aroradef run_autoserv(pid_file_manager, results, parser):
42466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora    # send stdin to /dev/null
437c8da7ce66017295a65ec028084b90800be377f8James Zern    dev_null = os.open(os.devnull, os.O_RDONLY)
447c8da7ce66017295a65ec028084b90800be377f8James Zern    os.dup2(dev_null, sys.stdin.fileno())
457c8da7ce66017295a65ec028084b90800be377f8James Zern    os.close(dev_null)
467c8da7ce66017295a65ec028084b90800be377f8James Zern
477c8da7ce66017295a65ec028084b90800be377f8James Zern    # Create separate process group
487c8da7ce66017295a65ec028084b90800be377f8James Zern    os.setpgrp()
497c8da7ce66017295a65ec028084b90800be377f8James Zern
507c8da7ce66017295a65ec028084b90800be377f8James Zern    # Implement SIGTERM handler
517c8da7ce66017295a65ec028084b90800be377f8James Zern    def handle_sigint(signum, frame):
527c8da7ce66017295a65ec028084b90800be377f8James Zern        pid_file_manager.close_pid_file(1, signal.SIGTERM)
537c8da7ce66017295a65ec028084b90800be377f8James Zern        os.killpg(os.getpgrp(), signal.SIGKILL)
547c8da7ce66017295a65ec028084b90800be377f8James Zern
557c8da7ce66017295a65ec028084b90800be377f8James Zern    # Set signal handler
567c8da7ce66017295a65ec028084b90800be377f8James Zern    signal.signal(signal.SIGTERM, handle_sigint)
577c8da7ce66017295a65ec028084b90800be377f8James Zern
587c8da7ce66017295a65ec028084b90800be377f8James Zern    # Get a useful value for running 'USER'
591e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    realuser = os.environ.get('USER')
601e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    if not realuser:
611e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        realuser = 'anonymous'
621e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
637c8da7ce66017295a65ec028084b90800be377f8James Zern
64466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora
651e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    if parser.options.machines:
661e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        machines = parser.options.machines.replace(',', ' ').strip().split()
67228b1b1f024974d7832b51a3f266e5edc9110c02Vikas Arora    else:
681e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        machines = []
69b6dbce6bfeaabde2a7b581c4c6888d532d32f3acDerek Sollenberger    machines_file = parser.options.machines_file
701e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    label = parser.options.label
71b6dbce6bfeaabde2a7b581c4c6888d532d32f3acDerek Sollenberger    user = parser.options.user
720912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern    client = parser.options.client
731e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    server = parser.options.server
747c8da7ce66017295a65ec028084b90800be377f8James Zern    reboot = parser.options.reboot
75466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora    install_before = parser.options.install_before
76466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora    install_after = parser.options.install_after
777c8da7ce66017295a65ec028084b90800be377f8James Zern    verify = parser.options.verify
78466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora    repair = parser.options.repair
79466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora    no_tee = parser.options.no_tee
80a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora    parse_job = parser.options.parse_job
817c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    host_protection = parser.options.host_protection
827c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    ssh_user = parser.options.ssh_user
837c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    ssh_port = parser.options.ssh_port
847c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    ssh_pass = parser.options.ssh_pass
857c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
867c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    # can't be both a client and a server side test
877c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if client and server:
887c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        print "Can not specify a test as both server and client!"
897c8da7ce66017295a65ec028084b90800be377f8James Zern        sys.exit(1)
907c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
917c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if len(parser.args) < 1 and not verify and not repair:
927c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        print parser.parser.print_help()
931e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        sys.exit(1)
947c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
957c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    # We have a control file unless it's just a verify/repair job
967c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if len(parser.args) > 0:
977c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        control = parser.args[0]
987c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    else:
997c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        control = None
100a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora
1017c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if machines_file:
1027c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        machines = []
1037c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        for m in open(machines_file, 'r').readlines():
1047c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            # remove comments, spaces
1057c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            m = re.sub('#.*', '', m).strip()
1067c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            if m:
1077c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora                machines.append(m)
1087c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        print "Read list of machines from file: %s" % machines_file
1097c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        print ','.join(machines)
110a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora
111a2415724fb3466168b2af5b08bd94ba732c0e753Vikas Arora    if machines:
1127c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        for machine in machines:
1137c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            if not machine or re.search('\s', machine):
1147c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora                print "Invalid machine %s" % str(machine)
1157c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora                sys.exit(1)
1167c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        machines = list(set(machines))
1177c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        machines.sort()
1187c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1197c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    job = server_job.server_job(control, parser.args[1:], results, label,
1207c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora                                user, machines, client, parse_job,
1217c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora                                ssh_user, ssh_port, ssh_pass)
1227c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    debug_dir = os.path.join(results, 'debug')
1237c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    stdout = os.path.join(debug_dir, 'autoserv.stdout')
1247c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    stderr = os.path.join(debug_dir, 'autoserv.stderr')
1257c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if no_tee:
1267c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        job.stdout.redirect(stdout)
1277c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        job.stderr.redirect(stderr)
1287c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    else:
1297c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        job.stdout.tee_redirect(stdout)
1307c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        job.stderr.tee_redirect(stderr)
1317c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1327c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    # perform checks
1337c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    job.precheck()
1347c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1357c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    # run the job
1367c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    exit_code = 0
1377c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    try:
1387c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        if repair:
1397c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            job.repair(host_protection)
1407c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        elif verify:
1417c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            job.verify()
1427c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        else:
143466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora            try:
144466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora                job.run(reboot, install_before, install_after)
145466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora            finally:
146466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora                job.cleanup_parser()
147466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora    except:
148466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora        job.aborted = True
149466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora        traceback.print_exc()
150466727975bcc57c0c5597bcd0747a2fe4777b303Vikas Arora
1517c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    pid_file_manager.num_tests_failed = job.num_tests_failed
1527c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1537c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if getattr(job, 'aborted', False):
1547c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        sys.exit(1)
1551e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1567c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1577c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Aroradef main():
1587c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    pid_file_manager = PidFileManager()
1591e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1601e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    # grab the parser
1611e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    parser = autoserv_parser.autoserv_parser
1621e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1630406ce1417f76f2034833414dcecc9f56253640cVikas Arora    if len(sys.argv) == 1:
1641e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora        parser.parser.print_help()
1650406ce1417f76f2034833414dcecc9f56253640cVikas Arora        sys.exit(1)
1661e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1677c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    results  = parser.options.results
1687c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    results  = os.path.abspath(results)
1691e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    write_pidfile = parser.options.write_pidfile
1707c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    if write_pidfile:
1717c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        pid_file_manager.open_pid_file(results)
1727c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1731e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    exit_code = 0
1747c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora    try:
1757c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        try:
1767c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            run_autoserv(pid_file_manager, results, parser)
1777c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        except SystemExit, e:
1787c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora            exit_code = e.code
1797c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        except:
1807c8da7ce66017295a65ec028084b90800be377f8James Zern            traceback.print_exc()
1817c8da7ce66017295a65ec028084b90800be377f8James Zern            # If we don't know what happened, we'll classify it as
1827c8da7ce66017295a65ec028084b90800be377f8James Zern            # an 'abort' and return 1.
1837c8da7ce66017295a65ec028084b90800be377f8James Zern            exit_code = 1
1847c8da7ce66017295a65ec028084b90800be377f8James Zern    finally:
1857c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora        pid_file_manager.close_pid_file(exit_code)
1860406ce1417f76f2034833414dcecc9f56253640cVikas Arora    sys.exit(exit_code)
1878b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
1887c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Arora
1897c970a0a679089e416c5887cf7fcece15a70bfa4Vikas Aroraif __name__ == '__main__':
1900406ce1417f76f2034833414dcecc9f56253640cVikas Arora    main()
1910406ce1417f76f2034833414dcecc9f56253640cVikas Arora