autoserv.py revision 27bba96fa00d54b99867476f810c381a156af94e
1#!/usr/bin/python -u 2# Copyright 2007-2008 Martin J. Bligh <mbligh@google.com>, Google Inc. 3# Released under the GPL v2 4 5""" 6Run a control file through the server side engine 7""" 8 9import datetime 10import contextlib 11import getpass 12import logging 13import os 14import re 15import signal 16import socket 17import sys 18import traceback 19import time 20import urllib2 21 22 23import common 24from autotest_lib.client.common_lib import control_data 25from autotest_lib.client.common_lib import error 26from autotest_lib.client.common_lib import global_config 27from autotest_lib.client.common_lib import utils 28from autotest_lib.client.common_lib.cros.graphite import autotest_es 29 30from chromite.lib import metrics 31 32try: 33 from autotest_lib.puppylab import results_mocker 34except ImportError: 35 results_mocker = None 36 37_CONFIG = global_config.global_config 38 39require_atfork = _CONFIG.get_config_value( 40 'AUTOSERV', 'require_atfork_module', type=bool, default=True) 41 42 43# Number of seconds to wait before returning if testing mode is enabled 44TESTING_MODE_SLEEP_SECS = 1 45 46try: 47 import atfork 48 atfork.monkeypatch_os_fork_functions() 49 import atfork.stdlib_fixer 50 # Fix the Python standard library for threading+fork safety with its 51 # internal locks. http://code.google.com/p/python-atfork/ 52 import warnings 53 warnings.filterwarnings('ignore', 'logging module already imported') 54 atfork.stdlib_fixer.fix_logging_module() 55except ImportError, e: 56 from autotest_lib.client.common_lib import global_config 57 if _CONFIG.get_config_value( 58 'AUTOSERV', 'require_atfork_module', type=bool, default=False): 59 print >>sys.stderr, 'Please run utils/build_externals.py' 60 print e 61 sys.exit(1) 62 63from autotest_lib.server import frontend 64from autotest_lib.server import server_logging_config 65from autotest_lib.server import server_job, utils, autoserv_parser, autotest 66from autotest_lib.server import utils as server_utils 67from autotest_lib.server import site_utils 68from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 69from autotest_lib.site_utils import job_directories 70from autotest_lib.site_utils import job_overhead 71from autotest_lib.site_utils import lxc 72from autotest_lib.site_utils import lxc_utils 73from autotest_lib.client.common_lib import pidfile, logging_manager 74 75 76# Control segment to stage server-side package. 77STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path( 78 'stage_server_side_package') 79 80# Command line to start servod in a moblab. 81START_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s' 82STOP_SERVOD_CMD = 'sudo stop servod' 83 84def log_alarm(signum, frame): 85 logging.error("Received SIGALARM. Ignoring and continuing on.") 86 sys.exit(1) 87 88 89def _get_machines(parser): 90 """Get a list of machine names from command line arg -m or a file. 91 92 @param parser: Parser for the command line arguments. 93 94 @return: A list of machine names from command line arg -m or the 95 machines file specified in the command line arg -M. 96 """ 97 if parser.options.machines: 98 machines = parser.options.machines.replace(',', ' ').strip().split() 99 else: 100 machines = [] 101 machines_file = parser.options.machines_file 102 if machines_file: 103 machines = [] 104 for m in open(machines_file, 'r').readlines(): 105 # remove comments, spaces 106 m = re.sub('#.*', '', m).strip() 107 if m: 108 machines.append(m) 109 logging.debug('Read list of machines from file: %s', machines_file) 110 logging.debug('Machines: %s', ','.join(machines)) 111 112 if machines: 113 for machine in machines: 114 if not machine or re.search('\s', machine): 115 parser.parser.error("Invalid machine: %s" % str(machine)) 116 machines = list(set(machines)) 117 machines.sort() 118 return machines 119 120 121def _stage_ssp(parser): 122 """Stage server-side package. 123 124 This function calls a control segment to stage server-side package based on 125 the job and autoserv command line option. The detail implementation could 126 be different for each host type. Currently, only CrosHost has 127 stage_server_side_package function defined. 128 The script returns None if no server-side package is available. However, 129 it may raise exception if it failed for reasons other than artifact (the 130 server-side package) not found. 131 132 @param parser: Command line arguments parser passed in the autoserv process. 133 134 @return: (ssp_url, error_msg), where 135 ssp_url is a url to the autotest server-side package. None if 136 server-side package is not supported. 137 error_msg is a string indicating the failures. None if server- 138 side package is staged successfully. 139 """ 140 machines_list = _get_machines(parser) 141 machines_list = server_job.get_machine_dicts( 142 machines_list, parser.options.lab, parser.options.host_attributes) 143 144 # If test_source_build is not specified, default to use server-side test 145 # code from build specified in --image. 146 namespace = {'machines': machines_list, 147 'image': (parser.options.test_source_build or 148 parser.options.image),} 149 script_locals = {} 150 execfile(STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE, namespace, script_locals) 151 return script_locals['ssp_url'], script_locals['error_msg'] 152 153 154def _run_with_ssp(job, container_name, job_id, results, parser, ssp_url, 155 job_folder, machines): 156 """Run the server job with server-side packaging. 157 158 @param job: The server job object. 159 @param container_name: Name of the container to run the test. 160 @param job_id: ID of the test job. 161 @param results: Folder to store results. This could be different from 162 parser.options.results: 163 parser.options.results can be set to None for results to be 164 stored in a temp folder. 165 results can be None for autoserv run requires no logging. 166 @param parser: Command line parser that contains the options. 167 @param ssp_url: url of the staged server-side package. 168 @param job_folder: Name of the job result folder. 169 @param machines: A list of machines to run the test. 170 """ 171 bucket = lxc.ContainerBucket() 172 control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != '' 173 else None) 174 try: 175 dut_name = machines[0] if len(machines) >= 1 else None 176 test_container = bucket.setup_test(container_name, job_id, ssp_url, 177 results, control=control, 178 job_folder=job_folder, 179 dut_name=dut_name) 180 except Exception as e: 181 job.record('FAIL', None, None, 182 'Failed to setup container for test: %s. Check logs in ' 183 'ssp_logs folder for more details.' % e) 184 raise 185 186 args = sys.argv[:] 187 args.remove('--require-ssp') 188 # --parent_job_id is only useful in autoserv running in host, not in 189 # container. Include this argument will cause test to fail for builds before 190 # CL 286265 was merged. 191 if '--parent_job_id' in args: 192 index = args.index('--parent_job_id') 193 args.remove('--parent_job_id') 194 # Remove the actual parent job id in command line arg. 195 del args[index] 196 197 # A dictionary of paths to replace in the command line. Key is the path to 198 # be replaced with the one in value. 199 paths_to_replace = {} 200 # Replace the control file path with the one in container. 201 if control: 202 container_control_filename = os.path.join( 203 lxc.CONTROL_TEMP_PATH, os.path.basename(control)) 204 paths_to_replace[control] = container_control_filename 205 # Update result directory with the one in container. 206 container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % job_folder) 207 if parser.options.results: 208 paths_to_replace[parser.options.results] = container_result_dir 209 # Update parse_job directory with the one in container. The assumption is 210 # that the result folder to be parsed is always the same as the results_dir. 211 if parser.options.parse_job: 212 paths_to_replace[parser.options.parse_job] = container_result_dir 213 214 args = [paths_to_replace.get(arg, arg) for arg in args] 215 216 # Apply --use-existing-results, results directory is aready created and 217 # mounted in container. Apply this arg to avoid exception being raised. 218 if not '--use-existing-results' in args: 219 args.append('--use-existing-results') 220 221 # Make sure autoserv running in container using a different pid file. 222 if not '--pidfile-label' in args: 223 args.extend(['--pidfile-label', 'container_autoserv']) 224 225 cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args]) 226 logging.info('Run command in container: %s', cmd_line) 227 success = False 228 try: 229 test_container.attach_run(cmd_line) 230 success = True 231 except Exception as e: 232 # If the test run inside container fails without generating any log, 233 # write a message to status.log to help troubleshooting. 234 debug_files = os.listdir(os.path.join(results, 'debug')) 235 if not debug_files: 236 job.record('FAIL', None, None, 237 'Failed to run test inside the container: %s. Check ' 238 'logs in ssp_logs folder for more details.' % e) 239 raise 240 finally: 241 metrics.Counter( 242 'chromeos/autotest/experimental/execute_job_in_ssp').increment( 243 fields={'success': success}) 244 # metadata is uploaded separately so it can use http to upload. 245 metadata = {'drone': socket.gethostname(), 246 'job_id': job_id, 247 'success': success} 248 autotest_es.post(use_http=True, 249 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 250 metadata=metadata) 251 test_container.destroy() 252 253 254def correct_results_folder_permission(results): 255 """Make sure the results folder has the right permission settings. 256 257 For tests running with server-side packaging, the results folder has the 258 owner of root. This must be changed to the user running the autoserv 259 process, so parsing job can access the results folder. 260 TODO(dshi): crbug.com/459344 Remove this function when test container can be 261 unprivileged container. 262 263 @param results: Path to the results folder. 264 265 """ 266 if not results: 267 return 268 269 try: 270 utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results)) 271 utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results)) 272 except error.CmdError as e: 273 metadata = {'error': str(e), 274 'result_folder': results, 275 'drone': socket.gethostname()} 276 autotest_es.post(use_http=True, type_str='correct_results_folder_failure', 277 metadata=metadata) 278 raise 279 280 281def _start_servod(machine): 282 """Try to start servod in moblab if it's not already running or running with 283 different board or port. 284 285 @param machine: Name of the dut used for test. 286 """ 287 if not utils.is_moblab(): 288 return 289 290 logging.debug('Trying to start servod.') 291 try: 292 afe = frontend.AFE() 293 board = server_utils.get_board_from_afe(machine, afe) 294 hosts = afe.get_hosts(hostname=machine) 295 servo_host = hosts[0].attributes.get('servo_host', None) 296 servo_port = hosts[0].attributes.get('servo_port', 9999) 297 if not servo_host in ['localhost', '127.0.0.1']: 298 logging.warn('Starting servod is aborted. The dut\'s servo_host ' 299 'attribute is not set to localhost.') 300 return 301 except (urllib2.HTTPError, urllib2.URLError): 302 # Ignore error if RPC failed to get board 303 logging.error('Failed to get board name from AFE. Start servod is ' 304 'aborted') 305 return 306 307 try: 308 pid = utils.run('pgrep servod').stdout 309 cmd_line = utils.run('ps -fp %s' % pid).stdout 310 if ('--board %s' % board in cmd_line and 311 '--port %s' % servo_port in cmd_line): 312 logging.debug('Servod is already running with given board and port.' 313 ' There is no need to restart servod.') 314 return 315 logging.debug('Servod is running with different board or port. ' 316 'Stopping existing servod.') 317 utils.run('sudo stop servod') 318 except error.CmdError: 319 # servod is not running. 320 pass 321 322 try: 323 utils.run(START_SERVOD_CMD % (board, servo_port)) 324 logging.debug('Servod is started') 325 except error.CmdError as e: 326 logging.error('Servod failed to be started, error: %s', e) 327 328 329def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): 330 """Run server job with given options. 331 332 @param pid_file_manager: PidFileManager used to monitor the autoserv process 333 @param results: Folder to store results. 334 @param parser: Parser for the command line arguments. 335 @param ssp_url: Url to server-side package. 336 @param use_ssp: Set to True to run with server-side packaging. 337 """ 338 if parser.options.warn_no_ssp: 339 # Post a warning in the log. 340 logging.warn('Autoserv is required to run with server-side packaging. ' 341 'However, no drone is found to support server-side ' 342 'packaging. The test will be executed in a drone without ' 343 'server-side packaging supported.') 344 345 # send stdin to /dev/null 346 dev_null = os.open(os.devnull, os.O_RDONLY) 347 os.dup2(dev_null, sys.stdin.fileno()) 348 os.close(dev_null) 349 350 # Create separate process group if the process is not a process group 351 # leader. This allows autoserv process to keep running after the caller 352 # process (drone manager call) exits. 353 if os.getpid() != os.getpgid(0): 354 os.setsid() 355 356 # Container name is predefined so the container can be destroyed in 357 # handle_sigterm. 358 job_or_task_id = job_directories.get_job_id_or_task_id( 359 parser.options.results) 360 container_name = (lxc.TEST_CONTAINER_NAME_FMT % 361 (job_or_task_id, time.time(), os.getpid())) 362 job_folder = job_directories.get_job_folder_name(parser.options.results) 363 364 # Implement SIGTERM handler 365 def handle_sigterm(signum, frame): 366 logging.debug('Received SIGTERM') 367 if pid_file_manager: 368 pid_file_manager.close_file(1, signal.SIGTERM) 369 logging.debug('Finished writing to pid_file. Killing process.') 370 371 # Update results folder's file permission. This needs to be done ASAP 372 # before the parsing process tries to access the log. 373 if use_ssp and results: 374 correct_results_folder_permission(results) 375 376 # TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved. 377 # This sleep allows the pending output to be logged before the kill 378 # signal is sent. 379 time.sleep(.1) 380 if use_ssp: 381 logging.debug('Destroy container %s before aborting the autoserv ' 382 'process.', container_name) 383 metadata = {'drone': socket.gethostname(), 384 'job_id': job_or_task_id, 385 'container_name': container_name, 386 'action': 'abort', 387 'success': True} 388 try: 389 bucket = lxc.ContainerBucket() 390 container = bucket.get(container_name) 391 if container: 392 container.destroy() 393 else: 394 metadata['success'] = False 395 metadata['error'] = 'container not found' 396 logging.debug('Container %s is not found.', container_name) 397 except: 398 metadata['success'] = False 399 metadata['error'] = 'Exception: %s' % str(sys.exc_info()) 400 # Handle any exception so the autoserv process can be aborted. 401 logging.exception('Failed to destroy container %s.', 402 container_name) 403 autotest_es.post(use_http=True, 404 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 405 metadata=metadata) 406 # Try to correct the result file permission again after the 407 # container is destroyed, as the container might have created some 408 # new files in the result folder. 409 if results: 410 correct_results_folder_permission(results) 411 412 os.killpg(os.getpgrp(), signal.SIGKILL) 413 414 # Set signal handler 415 signal.signal(signal.SIGTERM, handle_sigterm) 416 417 # faulthandler is only needed to debug in the Lab and is not avaliable to 418 # be imported in the chroot as part of VMTest, so Try-Except it. 419 try: 420 import faulthandler 421 faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) 422 logging.debug('faulthandler registered on SIGTERM.') 423 except ImportError: 424 sys.exc_clear() 425 426 # Ignore SIGTTOU's generated by output from forked children. 427 signal.signal(signal.SIGTTOU, signal.SIG_IGN) 428 429 # If we received a SIGALARM, let's be loud about it. 430 signal.signal(signal.SIGALRM, log_alarm) 431 432 # Server side tests that call shell scripts often depend on $USER being set 433 # but depending on how you launch your autotest scheduler it may not be set. 434 os.environ['USER'] = getpass.getuser() 435 436 label = parser.options.label 437 group_name = parser.options.group_name 438 user = parser.options.user 439 client = parser.options.client 440 server = parser.options.server 441 install_before = parser.options.install_before 442 install_after = parser.options.install_after 443 verify = parser.options.verify 444 repair = parser.options.repair 445 cleanup = parser.options.cleanup 446 provision = parser.options.provision 447 reset = parser.options.reset 448 job_labels = parser.options.job_labels 449 no_tee = parser.options.no_tee 450 parse_job = parser.options.parse_job 451 execution_tag = parser.options.execution_tag 452 if not execution_tag: 453 execution_tag = parse_job 454 ssh_user = parser.options.ssh_user 455 ssh_port = parser.options.ssh_port 456 ssh_pass = parser.options.ssh_pass 457 collect_crashinfo = parser.options.collect_crashinfo 458 control_filename = parser.options.control_filename 459 test_retry = parser.options.test_retry 460 verify_job_repo_url = parser.options.verify_job_repo_url 461 skip_crash_collection = parser.options.skip_crash_collection 462 ssh_verbosity = int(parser.options.ssh_verbosity) 463 ssh_options = parser.options.ssh_options 464 no_use_packaging = parser.options.no_use_packaging 465 host_attributes = parser.options.host_attributes 466 in_lab = bool(parser.options.lab) 467 468 # can't be both a client and a server side test 469 if client and server: 470 parser.parser.error("Can not specify a test as both server and client!") 471 472 if provision and client: 473 parser.parser.error("Cannot specify provisioning and client!") 474 475 is_special_task = (verify or repair or cleanup or collect_crashinfo or 476 provision or reset) 477 if len(parser.args) < 1 and not is_special_task: 478 parser.parser.error("Missing argument: control file") 479 480 if ssh_verbosity > 0: 481 # ssh_verbosity is an integer between 0 and 3, inclusive 482 ssh_verbosity_flag = '-' + 'v' * ssh_verbosity 483 else: 484 ssh_verbosity_flag = '' 485 486 # We have a control file unless it's just a verify/repair/cleanup job 487 if len(parser.args) > 0: 488 control = parser.args[0] 489 else: 490 control = None 491 492 machines = _get_machines(parser) 493 if group_name and len(machines) < 2: 494 parser.parser.error('-G %r may only be supplied with more than one ' 495 'machine.' % group_name) 496 497 kwargs = {'group_name': group_name, 'tag': execution_tag, 498 'disable_sysinfo': parser.options.disable_sysinfo} 499 if parser.options.parent_job_id: 500 kwargs['parent_job_id'] = int(parser.options.parent_job_id) 501 if control_filename: 502 kwargs['control_filename'] = control_filename 503 if host_attributes: 504 kwargs['host_attributes'] = host_attributes 505 kwargs['in_lab'] = in_lab 506 job = server_job.server_job(control, parser.args[1:], results, label, 507 user, machines, client, parse_job, 508 ssh_user, ssh_port, ssh_pass, 509 ssh_verbosity_flag, ssh_options, 510 test_retry, **kwargs) 511 512 job.logging.start_logging() 513 job.init_parser() 514 515 # perform checks 516 job.precheck() 517 518 # run the job 519 exit_code = 0 520 auto_start_servod = _CONFIG.get_config_value( 521 'AUTOSERV', 'auto_start_servod', type=bool, default=False) 522 523 site_utils.SetupTsMonGlobalState('autoserv', indirect=False, 524 short_lived=True) 525 try: 526 try: 527 if repair: 528 if auto_start_servod and len(machines) == 1: 529 _start_servod(machines[0]) 530 job.repair(job_labels) 531 elif verify: 532 job.verify(job_labels) 533 elif provision: 534 job.provision(job_labels) 535 elif reset: 536 job.reset(job_labels) 537 elif cleanup: 538 job.cleanup(job_labels) 539 else: 540 if auto_start_servod and len(machines) == 1: 541 _start_servod(machines[0]) 542 if use_ssp: 543 try: 544 _run_with_ssp(job, container_name, job_or_task_id, 545 results, parser, ssp_url, job_folder, 546 machines) 547 finally: 548 # Update the ownership of files in result folder. 549 correct_results_folder_permission(results) 550 else: 551 if collect_crashinfo: 552 # Update the ownership of files in result folder. If the 553 # job to collect crashinfo was running inside container 554 # (SSP) and crashed before correcting folder permission, 555 # the result folder might have wrong permission setting. 556 try: 557 correct_results_folder_permission(results) 558 except: 559 # Ignore any error as the user may not have root 560 # permission to run sudo command. 561 pass 562 job.run(install_before, install_after, 563 verify_job_repo_url=verify_job_repo_url, 564 only_collect_crashinfo=collect_crashinfo, 565 skip_crash_collection=skip_crash_collection, 566 job_labels=job_labels, 567 use_packaging=(not no_use_packaging)) 568 finally: 569 while job.hosts: 570 host = job.hosts.pop() 571 host.close() 572 except: 573 exit_code = 1 574 traceback.print_exc() 575 finally: 576 metrics.Flush() 577 578 if pid_file_manager: 579 pid_file_manager.num_tests_failed = job.num_tests_failed 580 pid_file_manager.close_file(exit_code) 581 job.cleanup_parser() 582 583 sys.exit(exit_code) 584 585 586def record_autoserv(options, duration_secs): 587 """Record autoserv end-to-end time in metadata db. 588 589 @param options: parser options. 590 @param duration_secs: How long autoserv has taken, in secs. 591 """ 592 # Get machine hostname 593 machines = options.machines.replace( 594 ',', ' ').strip().split() if options.machines else [] 595 num_machines = len(machines) 596 if num_machines > 1: 597 # Skip the case where atomic group is used. 598 return 599 elif num_machines == 0: 600 machines.append('hostless') 601 602 # Determine the status that will be reported. 603 s = job_overhead.STATUS 604 task_mapping = { 605 'reset': s.RESETTING, 'verify': s.VERIFYING, 606 'provision': s.PROVISIONING, 'repair': s.REPAIRING, 607 'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING} 608 match = filter(lambda task: getattr(options, task, False) == True, 609 task_mapping) 610 status = task_mapping[match[0]] if match else s.RUNNING 611 is_special_task = status not in [s.RUNNING, s.GATHERING] 612 job_or_task_id = job_directories.get_job_id_or_task_id(options.results) 613 job_overhead.record_state_duration( 614 job_or_task_id, machines[0], status, duration_secs, 615 is_special_task=is_special_task) 616 617 618def main(): 619 start_time = datetime.datetime.now() 620 # grab the parser 621 parser = autoserv_parser.autoserv_parser 622 parser.parse_args() 623 624 if len(sys.argv) == 1: 625 parser.parser.print_help() 626 sys.exit(1) 627 628 # If the job requires to run with server-side package, try to stage server- 629 # side package first. If that fails with error that autotest server package 630 # does not exist, fall back to run the job without using server-side 631 # packaging. If option warn_no_ssp is specified, that means autoserv is 632 # running in a drone does not support SSP, thus no need to stage server-side 633 # package. 634 ssp_url = None 635 ssp_url_warning = False 636 if (not parser.options.warn_no_ssp and parser.options.require_ssp): 637 ssp_url, ssp_error_msg = _stage_ssp(parser) 638 # The build does not have autotest server package. Fall back to not 639 # to use server-side package. Logging is postponed until logging being 640 # set up. 641 ssp_url_warning = not ssp_url 642 643 if parser.options.no_logging: 644 results = None 645 else: 646 results = parser.options.results 647 if not results: 648 results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') 649 results = os.path.abspath(results) 650 resultdir_exists = False 651 for filename in ('control.srv', 'status.log', '.autoserv_execute'): 652 if os.path.exists(os.path.join(results, filename)): 653 resultdir_exists = True 654 if not parser.options.use_existing_results and resultdir_exists: 655 error = "Error: results directory already exists: %s\n" % results 656 sys.stderr.write(error) 657 sys.exit(1) 658 659 # Now that we certified that there's no leftover results dir from 660 # previous jobs, lets create the result dir since the logging system 661 # needs to create the log file in there. 662 if not os.path.isdir(results): 663 os.makedirs(results) 664 665 # Server-side packaging will only be used if it's required and the package 666 # is available. If warn_no_ssp is specified, it means that autoserv is 667 # running in a drone does not have SSP supported and a warning will be logs. 668 # Therefore, it should not run with SSP. 669 use_ssp = (not parser.options.warn_no_ssp and parser.options.require_ssp 670 and ssp_url) 671 if use_ssp: 672 log_dir = os.path.join(results, 'ssp_logs') if results else None 673 if log_dir and not os.path.exists(log_dir): 674 os.makedirs(log_dir) 675 else: 676 log_dir = results 677 678 logging_manager.configure_logging( 679 server_logging_config.ServerLoggingConfig(), 680 results_dir=log_dir, 681 use_console=not parser.options.no_tee, 682 verbose=parser.options.verbose, 683 no_console_prefix=parser.options.no_console_prefix) 684 685 if ssp_url_warning: 686 logging.warn( 687 'Autoserv is required to run with server-side packaging. ' 688 'However, no server-side package can be found based on ' 689 '`--image`, host attribute job_repo_url or host OS version ' 690 'label. It could be that the build to test is older than the ' 691 'minimum version that supports server-side packaging. The test ' 692 'will be executed without using erver-side packaging. ' 693 'Following is the detailed error:\n%s', ssp_error_msg) 694 695 if results: 696 logging.info("Results placed in %s" % results) 697 698 # wait until now to perform this check, so it get properly logged 699 if (parser.options.use_existing_results and not resultdir_exists and 700 not utils.is_in_container()): 701 logging.error("No existing results directory found: %s", results) 702 sys.exit(1) 703 704 logging.debug('autoserv is running in drone %s.', socket.gethostname()) 705 logging.debug('autoserv command was: %s', ' '.join(sys.argv)) 706 707 if parser.options.write_pidfile and results: 708 pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, 709 results) 710 pid_file_manager.open_file() 711 else: 712 pid_file_manager = None 713 714 autotest.BaseAutotest.set_install_in_tmpdir( 715 parser.options.install_in_tmpdir) 716 717 try: 718 # Take the first argument as control file name, get the test name from 719 # the control file. 720 if (len(parser.args) > 0 and parser.args[0] != '' and 721 parser.options.machines): 722 try: 723 test_name = control_data.parse_control(parser.args[0], 724 raise_warnings=True).name 725 except control_data.ControlVariableException: 726 logging.debug('Failed to retrieve test name from control file.') 727 test_name = None 728 except control_data.ControlVariableException as e: 729 logging.error(str(e)) 730 exit_code = 0 731 # TODO(beeps): Extend this to cover different failure modes. 732 # Testing exceptions are matched against labels sent to autoserv. Eg, 733 # to allow only the hostless job to run, specify 734 # testing_exceptions: test_suite in the shadow_config. To allow both 735 # the hostless job and dummy_Pass to run, specify 736 # testing_exceptions: test_suite,dummy_Pass. You can figure out 737 # what label autoserv is invoked with by looking through the logs of a test 738 # for the autoserv command's -l option. 739 testing_exceptions = _CONFIG.get_config_value( 740 'AUTOSERV', 'testing_exceptions', type=list, default=[]) 741 test_mode = _CONFIG.get_config_value( 742 'AUTOSERV', 'testing_mode', type=bool, default=False) 743 test_mode = (results_mocker and test_mode and not 744 any([ex in parser.options.label 745 for ex in testing_exceptions])) 746 is_task = (parser.options.verify or parser.options.repair or 747 parser.options.provision or parser.options.reset or 748 parser.options.cleanup or parser.options.collect_crashinfo) 749 try: 750 try: 751 if test_mode: 752 # The parser doesn't run on tasks anyway, so we can just return 753 # happy signals without faking results. 754 if not is_task: 755 machine = parser.options.results.split('/')[-1] 756 757 # TODO(beeps): The proper way to do this would be to 758 # refactor job creation so we can invoke job.record 759 # directly. To do that one needs to pipe the test_name 760 # through run_autoserv and bail just before invoking 761 # the server job. See the comment in 762 # puppylab/results_mocker for more context. 763 results_mocker.ResultsMocker( 764 test_name if test_name else 'unknown-test', 765 parser.options.results, machine 766 ).mock_results() 767 return 768 else: 769 run_autoserv(pid_file_manager, results, parser, ssp_url, 770 use_ssp) 771 except SystemExit as e: 772 exit_code = e.code 773 if exit_code: 774 logging.exception(e) 775 except Exception as e: 776 # If we don't know what happened, we'll classify it as 777 # an 'abort' and return 1. 778 logging.exception(e) 779 exit_code = 1 780 finally: 781 if pid_file_manager: 782 pid_file_manager.close_file(exit_code) 783 # Record the autoserv duration time. Must be called 784 # just before the system exits to ensure accuracy. 785 duration_secs = (datetime.datetime.now() - start_time).total_seconds() 786 record_autoserv(parser.options, duration_secs) 787 sys.exit(exit_code) 788 789 790if __name__ == '__main__': 791 main() 792