autoserv revision cbebd983db4c57366e7d666b4ad1447bc7c5e1a1
1#!/usr/bin/python -u 2# Copyright 2007-2008 Martin J. Bligh <mbligh@google.com>, Google Inc. 3# Released under the GPL v2 4 5""" 6Run a control file through the server side engine 7""" 8 9import datetime 10import contextlib 11import getpass 12import logging 13import os 14import re 15import signal 16import socket 17import sys 18import traceback 19import time 20import urllib2 21 22 23import common 24from autotest_lib.client.common_lib import control_data 25from autotest_lib.client.common_lib import error 26from autotest_lib.client.common_lib import global_config 27from autotest_lib.client.common_lib import utils 28from autotest_lib.client.common_lib.cros.graphite import autotest_es 29 30from chromite.lib import metrics 31 32try: 33 from autotest_lib.puppylab import results_mocker 34except ImportError: 35 results_mocker = None 36 37_CONFIG = global_config.global_config 38 39require_atfork = _CONFIG.get_config_value( 40 'AUTOSERV', 'require_atfork_module', type=bool, default=True) 41 42 43# Number of seconds to wait before returning if testing mode is enabled 44TESTING_MODE_SLEEP_SECS = 1 45 46try: 47 import atfork 48 atfork.monkeypatch_os_fork_functions() 49 import atfork.stdlib_fixer 50 # Fix the Python standard library for threading+fork safety with its 51 # internal locks. http://code.google.com/p/python-atfork/ 52 import warnings 53 warnings.filterwarnings('ignore', 'logging module already imported') 54 atfork.stdlib_fixer.fix_logging_module() 55except ImportError, e: 56 from autotest_lib.client.common_lib import global_config 57 if _CONFIG.get_config_value( 58 'AUTOSERV', 'require_atfork_module', type=bool, default=False): 59 print >>sys.stderr, 'Please run utils/build_externals.py' 60 print e 61 sys.exit(1) 62 63from autotest_lib.server import frontend 64from autotest_lib.server import server_logging_config 65from autotest_lib.server import server_job, utils, autoserv_parser, autotest 66from autotest_lib.server import utils as server_utils 67from autotest_lib.server import site_utils 68from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 69from autotest_lib.site_utils import job_directories 70from autotest_lib.site_utils import job_overhead 71from autotest_lib.site_utils import lxc 72from autotest_lib.site_utils import lxc_utils 73from autotest_lib.client.common_lib import pidfile, logging_manager 74 75 76# Control segment to stage server-side package. 77STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path( 78 'stage_server_side_package') 79 80# Command line to start servod in a moblab. 81START_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s' 82STOP_SERVOD_CMD = 'sudo stop servod' 83 84def log_alarm(signum, frame): 85 logging.error("Received SIGALARM. Ignoring and continuing on.") 86 sys.exit(1) 87 88 89def _get_machines(parser): 90 """Get a list of machine names from command line arg -m or a file. 91 92 @param parser: Parser for the command line arguments. 93 94 @return: A list of machine names from command line arg -m or the 95 machines file specified in the command line arg -M. 96 """ 97 if parser.options.machines: 98 machines = parser.options.machines.replace(',', ' ').strip().split() 99 else: 100 machines = [] 101 machines_file = parser.options.machines_file 102 if machines_file: 103 machines = [] 104 for m in open(machines_file, 'r').readlines(): 105 # remove comments, spaces 106 m = re.sub('#.*', '', m).strip() 107 if m: 108 machines.append(m) 109 logging.debug('Read list of machines from file: %s', machines_file) 110 logging.debug('Machines: %s', ','.join(machines)) 111 112 if machines: 113 for machine in machines: 114 if not machine or re.search('\s', machine): 115 parser.parser.error("Invalid machine: %s" % str(machine)) 116 machines = list(set(machines)) 117 machines.sort() 118 return machines 119 120 121def _stage_ssp(parser): 122 """Stage server-side package. 123 124 This function calls a control segment to stage server-side package based on 125 the job and autoserv command line option. The detail implementation could 126 be different for each host type. Currently, only CrosHost has 127 stage_server_side_package function defined. 128 The script returns None if no server-side package is available. However, 129 it may raise exception if it failed for reasons other than artifact (the 130 server-side package) not found. 131 132 @param parser: Command line arguments parser passed in the autoserv process. 133 134 @return: (ssp_url, error_msg), where 135 ssp_url is a url to the autotest server-side package. None if 136 server-side package is not supported. 137 error_msg is a string indicating the failures. None if server- 138 side package is staged successfully. 139 """ 140 machines_list = _get_machines(parser) 141 machines_list = server_job.get_machine_dicts( 142 machines_list, parser.options.lab, parser.options.host_attributes) 143 144 # If test_source_build is not specified, default to use server-side test 145 # code from build specified in --image. 146 namespace = {'machines': machines_list, 147 'image': (parser.options.test_source_build or 148 parser.options.image),} 149 script_locals = {} 150 execfile(STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE, namespace, script_locals) 151 return script_locals['ssp_url'], script_locals['error_msg'] 152 153 154def _run_with_ssp(job, container_name, job_id, results, parser, ssp_url, 155 job_folder, machines): 156 """Run the server job with server-side packaging. 157 158 @param job: The server job object. 159 @param container_name: Name of the container to run the test. 160 @param job_id: ID of the test job. 161 @param results: Folder to store results. This could be different from 162 parser.options.results: 163 parser.options.results can be set to None for results to be 164 stored in a temp folder. 165 results can be None for autoserv run requires no logging. 166 @param parser: Command line parser that contains the options. 167 @param ssp_url: url of the staged server-side package. 168 @param job_folder: Name of the job result folder. 169 @param machines: A list of machines to run the test. 170 """ 171 bucket = lxc.ContainerBucket() 172 control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != '' 173 else None) 174 try: 175 dut_name = machines[0] if len(machines) >= 1 else None 176 test_container = bucket.setup_test(container_name, job_id, ssp_url, 177 results, control=control, 178 job_folder=job_folder, 179 dut_name=dut_name) 180 except Exception as e: 181 job.record('FAIL', None, None, 182 'Failed to setup container for test: %s. Check logs in ' 183 'ssp_logs folder for more details.' % e) 184 raise 185 186 args = sys.argv[:] 187 args.remove('--require-ssp') 188 # --parent_job_id is only useful in autoserv running in host, not in 189 # container. Include this argument will cause test to fail for builds before 190 # CL 286265 was merged. 191 if '--parent_job_id' in args: 192 index = args.index('--parent_job_id') 193 args.remove('--parent_job_id') 194 # Remove the actual parent job id in command line arg. 195 del args[index] 196 197 # A dictionary of paths to replace in the command line. Key is the path to 198 # be replaced with the one in value. 199 paths_to_replace = {} 200 # Replace the control file path with the one in container. 201 if control: 202 container_control_filename = os.path.join( 203 lxc.CONTROL_TEMP_PATH, os.path.basename(control)) 204 paths_to_replace[control] = container_control_filename 205 # Update result directory with the one in container. 206 container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % job_folder) 207 if parser.options.results: 208 paths_to_replace[parser.options.results] = container_result_dir 209 # Update parse_job directory with the one in container. The assumption is 210 # that the result folder to be parsed is always the same as the results_dir. 211 if parser.options.parse_job: 212 paths_to_replace[parser.options.parse_job] = container_result_dir 213 214 args = [paths_to_replace.get(arg, arg) for arg in args] 215 216 # Apply --use-existing-results, results directory is aready created and 217 # mounted in container. Apply this arg to avoid exception being raised. 218 if not '--use-existing-results' in args: 219 args.append('--use-existing-results') 220 221 # Make sure autoserv running in container using a different pid file. 222 if not '--pidfile-label' in args: 223 args.extend(['--pidfile-label', 'container_autoserv']) 224 225 cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args]) 226 logging.info('Run command in container: %s', cmd_line) 227 success = False 228 try: 229 test_container.attach_run(cmd_line) 230 success = True 231 except Exception as e: 232 # If the test run inside container fails without generating any log, 233 # write a message to status.log to help troubleshooting. 234 debug_files = os.listdir(os.path.join(results, 'debug')) 235 if not debug_files: 236 job.record('FAIL', None, None, 237 'Failed to run test inside the container: %s. Check ' 238 'logs in ssp_logs folder for more details.' % e) 239 raise 240 finally: 241 metrics.Counter( 242 'chromeos/autotest/experimental/execute_job_in_ssp').increment( 243 fields={'success': success}) 244 # metadata is uploaded separately so it can use http to upload. 245 metadata = {'drone': socket.gethostname(), 246 'job_id': job_id, 247 'success': success} 248 autotest_es.post(use_http=True, 249 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 250 metadata=metadata) 251 test_container.destroy() 252 253 254def correct_results_folder_permission(results): 255 """Make sure the results folder has the right permission settings. 256 257 For tests running with server-side packaging, the results folder has the 258 owner of root. This must be changed to the user running the autoserv 259 process, so parsing job can access the results folder. 260 TODO(dshi): crbug.com/459344 Remove this function when test container can be 261 unprivileged container. 262 263 @param results: Path to the results folder. 264 265 """ 266 if not results: 267 return 268 269 try: 270 utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results)) 271 utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results)) 272 except error.CmdError as e: 273 metadata = {'error': str(e), 274 'result_folder': results, 275 'drone': socket.gethostname()} 276 autotest_es.post(use_http=True, type_str='correct_results_folder_failure', 277 metadata=metadata) 278 raise 279 280 281def _start_servod(machine): 282 """Try to start servod in moblab if it's not already running or running with 283 different board or port. 284 285 @param machine: Name of the dut used for test. 286 """ 287 if not utils.is_moblab(): 288 return 289 290 logging.debug('Trying to start servod.') 291 try: 292 afe = frontend.AFE() 293 board = server_utils.get_board_from_afe(machine, afe) 294 hosts = afe.get_hosts(hostname=machine) 295 servo_host = hosts[0].attributes.get('servo_host', None) 296 servo_port = hosts[0].attributes.get('servo_port', 9999) 297 if not servo_host in ['localhost', '127.0.0.1']: 298 logging.warn('Starting servod is aborted. The dut\'s servo_host ' 299 'attribute is not set to localhost.') 300 return 301 except (urllib2.HTTPError, urllib2.URLError): 302 # Ignore error if RPC failed to get board 303 logging.error('Failed to get board name from AFE. Start servod is ' 304 'aborted') 305 return 306 307 try: 308 pid = utils.run('pgrep servod').stdout 309 cmd_line = utils.run('ps -fp %s' % pid).stdout 310 if ('--board %s' % board in cmd_line and 311 '--port %s' % servo_port in cmd_line): 312 logging.debug('Servod is already running with given board and port.' 313 ' There is no need to restart servod.') 314 return 315 logging.debug('Servod is running with different board or port. ' 316 'Stopping existing servod.') 317 utils.run('sudo stop servod') 318 except error.CmdError: 319 # servod is not running. 320 pass 321 322 try: 323 utils.run(START_SERVOD_CMD % (board, servo_port)) 324 logging.debug('Servod is started') 325 except error.CmdError as e: 326 logging.error('Servod failed to be started, error: %s', e) 327 328 329def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): 330 """Run server job with given options. 331 332 @param pid_file_manager: PidFileManager used to monitor the autoserv process 333 @param results: Folder to store results. 334 @param parser: Parser for the command line arguments. 335 @param ssp_url: Url to server-side package. 336 @param use_ssp: Set to True to run with server-side packaging. 337 """ 338 if parser.options.warn_no_ssp: 339 # Post a warning in the log. 340 logging.warn('Autoserv is required to run with server-side packaging. ' 341 'However, no drone is found to support server-side ' 342 'packaging. The test will be executed in a drone without ' 343 'server-side packaging supported.') 344 345 # send stdin to /dev/null 346 dev_null = os.open(os.devnull, os.O_RDONLY) 347 os.dup2(dev_null, sys.stdin.fileno()) 348 os.close(dev_null) 349 350 # Create separate process group if the process is not a process group 351 # leader. This allows autoserv process to keep running after the caller 352 # process (drone manager call) exits. 353 if os.getpid() != os.getpgid(0): 354 os.setsid() 355 356 # Container name is predefined so the container can be destroyed in 357 # handle_sigterm. 358 job_or_task_id = job_directories.get_job_id_or_task_id( 359 parser.options.results) 360 container_name = (lxc.TEST_CONTAINER_NAME_FMT % 361 (job_or_task_id, time.time(), os.getpid())) 362 job_folder = job_directories.get_job_folder_name(parser.options.results) 363 364 # Implement SIGTERM handler 365 def handle_sigterm(signum, frame): 366 logging.debug('Received SIGTERM') 367 if pid_file_manager: 368 pid_file_manager.close_file(1, signal.SIGTERM) 369 logging.debug('Finished writing to pid_file. Killing process.') 370 371 # Update results folder's file permission. This needs to be done ASAP 372 # before the parsing process tries to access the log. 373 if use_ssp and results: 374 correct_results_folder_permission(results) 375 376 # TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved. 377 # This sleep allows the pending output to be logged before the kill 378 # signal is sent. 379 time.sleep(.1) 380 if use_ssp: 381 logging.debug('Destroy container %s before aborting the autoserv ' 382 'process.', container_name) 383 metadata = {'drone': socket.gethostname(), 384 'job_id': job_or_task_id, 385 'container_name': container_name, 386 'action': 'abort', 387 'success': True} 388 try: 389 bucket = lxc.ContainerBucket() 390 container = bucket.get(container_name) 391 if container: 392 container.destroy() 393 else: 394 metadata['success'] = False 395 metadata['error'] = 'container not found' 396 logging.debug('Container %s is not found.', container_name) 397 except: 398 metadata['success'] = False 399 metadata['error'] = 'Exception: %s' % str(sys.exc_info()) 400 # Handle any exception so the autoserv process can be aborted. 401 logging.exception('Failed to destroy container %s.', 402 container_name) 403 autotest_es.post(use_http=True, 404 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 405 metadata=metadata) 406 # Try to correct the result file permission again after the 407 # container is destroyed, as the container might have created some 408 # new files in the result folder. 409 if results: 410 correct_results_folder_permission(results) 411 412 os.killpg(os.getpgrp(), signal.SIGKILL) 413 414 # Set signal handler 415 signal.signal(signal.SIGTERM, handle_sigterm) 416 417 # faulthandler is only needed to debug in the Lab and is not avaliable to 418 # be imported in the chroot as part of VMTest, so Try-Except it. 419 try: 420 import faulthandler 421 faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) 422 logging.debug('faulthandler registered on SIGTERM.') 423 except ImportError: 424 sys.exc_clear() 425 426 # Ignore SIGTTOU's generated by output from forked children. 427 signal.signal(signal.SIGTTOU, signal.SIG_IGN) 428 429 # If we received a SIGALARM, let's be loud about it. 430 signal.signal(signal.SIGALRM, log_alarm) 431 432 # Server side tests that call shell scripts often depend on $USER being set 433 # but depending on how you launch your autotest scheduler it may not be set. 434 os.environ['USER'] = getpass.getuser() 435 436 label = parser.options.label 437 group_name = parser.options.group_name 438 user = parser.options.user 439 client = parser.options.client 440 server = parser.options.server 441 install_before = parser.options.install_before 442 install_after = parser.options.install_after 443 verify = parser.options.verify 444 repair = parser.options.repair 445 cleanup = parser.options.cleanup 446 provision = parser.options.provision 447 reset = parser.options.reset 448 job_labels = parser.options.job_labels 449 no_tee = parser.options.no_tee 450 parse_job = parser.options.parse_job 451 execution_tag = parser.options.execution_tag 452 if not execution_tag: 453 execution_tag = parse_job 454 ssh_user = parser.options.ssh_user 455 ssh_port = parser.options.ssh_port 456 ssh_pass = parser.options.ssh_pass 457 collect_crashinfo = parser.options.collect_crashinfo 458 control_filename = parser.options.control_filename 459 test_retry = parser.options.test_retry 460 verify_job_repo_url = parser.options.verify_job_repo_url 461 skip_crash_collection = parser.options.skip_crash_collection 462 ssh_verbosity = int(parser.options.ssh_verbosity) 463 ssh_options = parser.options.ssh_options 464 no_use_packaging = parser.options.no_use_packaging 465 host_attributes = parser.options.host_attributes 466 in_lab = bool(parser.options.lab) 467 468 # can't be both a client and a server side test 469 if client and server: 470 parser.parser.error("Can not specify a test as both server and client!") 471 472 if provision and client: 473 parser.parser.error("Cannot specify provisioning and client!") 474 475 is_special_task = (verify or repair or cleanup or collect_crashinfo or 476 provision or reset) 477 if len(parser.args) < 1 and not is_special_task: 478 parser.parser.error("Missing argument: control file") 479 480 if ssh_verbosity > 0: 481 # ssh_verbosity is an integer between 0 and 3, inclusive 482 ssh_verbosity_flag = '-' + 'v' * ssh_verbosity 483 else: 484 ssh_verbosity_flag = '' 485 486 # We have a control file unless it's just a verify/repair/cleanup job 487 if len(parser.args) > 0: 488 control = parser.args[0] 489 else: 490 control = None 491 492 machines = _get_machines(parser) 493 if group_name and len(machines) < 2: 494 parser.parser.error('-G %r may only be supplied with more than one ' 495 'machine.' % group_name) 496 497 kwargs = {'group_name': group_name, 'tag': execution_tag, 498 'disable_sysinfo': parser.options.disable_sysinfo} 499 if parser.options.parent_job_id: 500 kwargs['parent_job_id'] = int(parser.options.parent_job_id) 501 if control_filename: 502 kwargs['control_filename'] = control_filename 503 if host_attributes: 504 kwargs['host_attributes'] = host_attributes 505 kwargs['in_lab'] = in_lab 506 job = server_job.server_job(control, parser.args[1:], results, label, 507 user, machines, client, parse_job, 508 ssh_user, ssh_port, ssh_pass, 509 ssh_verbosity_flag, ssh_options, 510 test_retry, **kwargs) 511 512 job.logging.start_logging() 513 job.init_parser() 514 515 # perform checks 516 job.precheck() 517 518 # run the job 519 exit_code = 0 520 auto_start_servod = _CONFIG.get_config_value( 521 'AUTOSERV', 'auto_start_servod', type=bool, default=False) 522 523 try: 524 with site_utils.SetupTsMonGlobalState('autoserv', indirect=True, short_lived=True): 525 try: 526 if repair: 527 if auto_start_servod and len(machines) == 1: 528 _start_servod(machines[0]) 529 job.repair(job_labels) 530 elif verify: 531 job.verify(job_labels) 532 elif provision: 533 job.provision(job_labels) 534 elif reset: 535 job.reset(job_labels) 536 elif cleanup: 537 job.cleanup(job_labels) 538 else: 539 if auto_start_servod and len(machines) == 1: 540 _start_servod(machines[0]) 541 if use_ssp: 542 try: 543 _run_with_ssp(job, container_name, job_or_task_id, 544 results, parser, ssp_url, job_folder, 545 machines) 546 finally: 547 # Update the ownership of files in result folder. 548 correct_results_folder_permission(results) 549 else: 550 if collect_crashinfo: 551 # Update the ownership of files in result folder. If the 552 # job to collect crashinfo was running inside container 553 # (SSP) and crashed before correcting folder permission, 554 # the result folder might have wrong permission setting. 555 try: 556 correct_results_folder_permission(results) 557 except: 558 # Ignore any error as the user may not have root 559 # permission to run sudo command. 560 pass 561 job.run(install_before, install_after, 562 verify_job_repo_url=verify_job_repo_url, 563 only_collect_crashinfo=collect_crashinfo, 564 skip_crash_collection=skip_crash_collection, 565 job_labels=job_labels, 566 use_packaging=(not no_use_packaging)) 567 finally: 568 while job.hosts: 569 host = job.hosts.pop() 570 host.close() 571 except: 572 exit_code = 1 573 traceback.print_exc() 574 575 if pid_file_manager: 576 pid_file_manager.num_tests_failed = job.num_tests_failed 577 pid_file_manager.close_file(exit_code) 578 job.cleanup_parser() 579 580 sys.exit(exit_code) 581 582 583def record_autoserv(options, duration_secs): 584 """Record autoserv end-to-end time in metadata db. 585 586 @param options: parser options. 587 @param duration_secs: How long autoserv has taken, in secs. 588 """ 589 # Get machine hostname 590 machines = options.machines.replace( 591 ',', ' ').strip().split() if options.machines else [] 592 num_machines = len(machines) 593 if num_machines > 1: 594 # Skip the case where atomic group is used. 595 return 596 elif num_machines == 0: 597 machines.append('hostless') 598 599 # Determine the status that will be reported. 600 s = job_overhead.STATUS 601 task_mapping = { 602 'reset': s.RESETTING, 'verify': s.VERIFYING, 603 'provision': s.PROVISIONING, 'repair': s.REPAIRING, 604 'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING} 605 match = filter(lambda task: getattr(options, task, False) == True, 606 task_mapping) 607 status = task_mapping[match[0]] if match else s.RUNNING 608 is_special_task = status not in [s.RUNNING, s.GATHERING] 609 job_or_task_id = job_directories.get_job_id_or_task_id(options.results) 610 job_overhead.record_state_duration( 611 job_or_task_id, machines[0], status, duration_secs, 612 is_special_task=is_special_task) 613 614 615def main(): 616 start_time = datetime.datetime.now() 617 # grab the parser 618 parser = autoserv_parser.autoserv_parser 619 parser.parse_args() 620 621 if len(sys.argv) == 1: 622 parser.parser.print_help() 623 sys.exit(1) 624 625 # If the job requires to run with server-side package, try to stage server- 626 # side package first. If that fails with error that autotest server package 627 # does not exist, fall back to run the job without using server-side 628 # packaging. If option warn_no_ssp is specified, that means autoserv is 629 # running in a drone does not support SSP, thus no need to stage server-side 630 # package. 631 ssp_url = None 632 ssp_url_warning = False 633 if (not parser.options.warn_no_ssp and parser.options.require_ssp): 634 ssp_url, ssp_error_msg = _stage_ssp(parser) 635 # The build does not have autotest server package. Fall back to not 636 # to use server-side package. Logging is postponed until logging being 637 # set up. 638 ssp_url_warning = not ssp_url 639 640 if parser.options.no_logging: 641 results = None 642 else: 643 results = parser.options.results 644 if not results: 645 results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') 646 results = os.path.abspath(results) 647 resultdir_exists = False 648 for filename in ('control.srv', 'status.log', '.autoserv_execute'): 649 if os.path.exists(os.path.join(results, filename)): 650 resultdir_exists = True 651 if not parser.options.use_existing_results and resultdir_exists: 652 error = "Error: results directory already exists: %s\n" % results 653 sys.stderr.write(error) 654 sys.exit(1) 655 656 # Now that we certified that there's no leftover results dir from 657 # previous jobs, lets create the result dir since the logging system 658 # needs to create the log file in there. 659 if not os.path.isdir(results): 660 os.makedirs(results) 661 662 # Server-side packaging will only be used if it's required and the package 663 # is available. If warn_no_ssp is specified, it means that autoserv is 664 # running in a drone does not have SSP supported and a warning will be logs. 665 # Therefore, it should not run with SSP. 666 use_ssp = (not parser.options.warn_no_ssp and parser.options.require_ssp 667 and ssp_url) 668 if use_ssp: 669 log_dir = os.path.join(results, 'ssp_logs') if results else None 670 if log_dir and not os.path.exists(log_dir): 671 os.makedirs(log_dir) 672 else: 673 log_dir = results 674 675 logging_manager.configure_logging( 676 server_logging_config.ServerLoggingConfig(), 677 results_dir=log_dir, 678 use_console=not parser.options.no_tee, 679 verbose=parser.options.verbose, 680 no_console_prefix=parser.options.no_console_prefix) 681 682 if ssp_url_warning: 683 logging.warn( 684 'Autoserv is required to run with server-side packaging. ' 685 'However, no server-side package can be found based on ' 686 '`--image`, host attribute job_repo_url or host OS version ' 687 'label. It could be that the build to test is older than the ' 688 'minimum version that supports server-side packaging. The test ' 689 'will be executed without using erver-side packaging. ' 690 'Following is the detailed error:\n%s', ssp_error_msg) 691 692 if results: 693 logging.info("Results placed in %s" % results) 694 695 # wait until now to perform this check, so it get properly logged 696 if (parser.options.use_existing_results and not resultdir_exists and 697 not utils.is_in_container()): 698 logging.error("No existing results directory found: %s", results) 699 sys.exit(1) 700 701 logging.debug('autoserv is running in drone %s.', socket.gethostname()) 702 logging.debug('autoserv command was: %s', ' '.join(sys.argv)) 703 704 if parser.options.write_pidfile and results: 705 pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, 706 results) 707 pid_file_manager.open_file() 708 else: 709 pid_file_manager = None 710 711 autotest.BaseAutotest.set_install_in_tmpdir( 712 parser.options.install_in_tmpdir) 713 714 try: 715 # Take the first argument as control file name, get the test name from 716 # the control file. 717 if (len(parser.args) > 0 and parser.args[0] != '' and 718 parser.options.machines): 719 try: 720 test_name = control_data.parse_control(parser.args[0], 721 raise_warnings=True).name 722 except control_data.ControlVariableException: 723 logging.debug('Failed to retrieve test name from control file.') 724 test_name = None 725 except control_data.ControlVariableException as e: 726 logging.error(str(e)) 727 exit_code = 0 728 # TODO(beeps): Extend this to cover different failure modes. 729 # Testing exceptions are matched against labels sent to autoserv. Eg, 730 # to allow only the hostless job to run, specify 731 # testing_exceptions: test_suite in the shadow_config. To allow both 732 # the hostless job and dummy_Pass to run, specify 733 # testing_exceptions: test_suite,dummy_Pass. You can figure out 734 # what label autoserv is invoked with by looking through the logs of a test 735 # for the autoserv command's -l option. 736 testing_exceptions = _CONFIG.get_config_value( 737 'AUTOSERV', 'testing_exceptions', type=list, default=[]) 738 test_mode = _CONFIG.get_config_value( 739 'AUTOSERV', 'testing_mode', type=bool, default=False) 740 test_mode = (results_mocker and test_mode and not 741 any([ex in parser.options.label 742 for ex in testing_exceptions])) 743 is_task = (parser.options.verify or parser.options.repair or 744 parser.options.provision or parser.options.reset or 745 parser.options.cleanup or parser.options.collect_crashinfo) 746 try: 747 try: 748 if test_mode: 749 # The parser doesn't run on tasks anyway, so we can just return 750 # happy signals without faking results. 751 if not is_task: 752 machine = parser.options.results.split('/')[-1] 753 754 # TODO(beeps): The proper way to do this would be to 755 # refactor job creation so we can invoke job.record 756 # directly. To do that one needs to pipe the test_name 757 # through run_autoserv and bail just before invoking 758 # the server job. See the comment in 759 # puppylab/results_mocker for more context. 760 results_mocker.ResultsMocker( 761 test_name if test_name else 'unknown-test', 762 parser.options.results, machine 763 ).mock_results() 764 return 765 else: 766 run_autoserv(pid_file_manager, results, parser, ssp_url, 767 use_ssp) 768 except SystemExit as e: 769 exit_code = e.code 770 if exit_code: 771 logging.exception(e) 772 except Exception as e: 773 # If we don't know what happened, we'll classify it as 774 # an 'abort' and return 1. 775 logging.exception(e) 776 exit_code = 1 777 finally: 778 if pid_file_manager: 779 pid_file_manager.close_file(exit_code) 780 # Record the autoserv duration time. Must be called 781 # just before the system exits to ensure accuracy. 782 duration_secs = (datetime.datetime.now() - start_time).total_seconds() 783 record_autoserv(parser.options, duration_secs) 784 sys.exit(exit_code) 785 786 787if __name__ == '__main__': 788 main() 789