autoserv.py revision 5ae41397f4e633a12453faadeea4dbe1cc5e65bc
1#!/usr/bin/python -u 2# Copyright 2007-2008 Martin J. Bligh <mbligh@google.com>, Google Inc. 3# Released under the GPL v2 4 5""" 6Run a control file through the server side engine 7""" 8 9import datetime 10import contextlib 11import getpass 12import logging 13import os 14import re 15import signal 16import socket 17import sys 18import traceback 19import time 20import urllib2 21 22 23import common 24from autotest_lib.client.common_lib import control_data 25from autotest_lib.client.common_lib import error 26from autotest_lib.client.common_lib import global_config 27from autotest_lib.client.common_lib import utils 28from autotest_lib.client.common_lib.cros.graphite import autotest_es 29 30from chromite.lib import metrics 31 32try: 33 from autotest_lib.puppylab import results_mocker 34except ImportError: 35 results_mocker = None 36 37_CONFIG = global_config.global_config 38 39require_atfork = _CONFIG.get_config_value( 40 'AUTOSERV', 'require_atfork_module', type=bool, default=True) 41 42 43# Number of seconds to wait before returning if testing mode is enabled 44TESTING_MODE_SLEEP_SECS = 1 45 46try: 47 import atfork 48 atfork.monkeypatch_os_fork_functions() 49 import atfork.stdlib_fixer 50 # Fix the Python standard library for threading+fork safety with its 51 # internal locks. http://code.google.com/p/python-atfork/ 52 import warnings 53 warnings.filterwarnings('ignore', 'logging module already imported') 54 atfork.stdlib_fixer.fix_logging_module() 55except ImportError, e: 56 from autotest_lib.client.common_lib import global_config 57 if _CONFIG.get_config_value( 58 'AUTOSERV', 'require_atfork_module', type=bool, default=False): 59 print >>sys.stderr, 'Please run utils/build_externals.py' 60 print e 61 sys.exit(1) 62 63from autotest_lib.server import frontend 64from autotest_lib.server import server_logging_config 65from autotest_lib.server import server_job, utils, autoserv_parser, autotest 66from autotest_lib.server import utils as server_utils 67from autotest_lib.server import site_utils 68from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 69from autotest_lib.site_utils import job_directories 70from autotest_lib.site_utils import job_overhead 71from autotest_lib.site_utils import lxc 72from autotest_lib.site_utils import lxc_utils 73from autotest_lib.client.common_lib import pidfile, logging_manager 74 75 76# Control segment to stage server-side package. 77STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path( 78 'stage_server_side_package') 79 80# Command line to start servod in a moblab. 81START_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s' 82STOP_SERVOD_CMD = 'sudo stop servod' 83 84def log_alarm(signum, frame): 85 logging.error("Received SIGALARM. Ignoring and continuing on.") 86 sys.exit(1) 87 88 89def _get_machines(parser): 90 """Get a list of machine names from command line arg -m or a file. 91 92 @param parser: Parser for the command line arguments. 93 94 @return: A list of machine names from command line arg -m or the 95 machines file specified in the command line arg -M. 96 """ 97 if parser.options.machines: 98 machines = parser.options.machines.replace(',', ' ').strip().split() 99 else: 100 machines = [] 101 machines_file = parser.options.machines_file 102 if machines_file: 103 machines = [] 104 for m in open(machines_file, 'r').readlines(): 105 # remove comments, spaces 106 m = re.sub('#.*', '', m).strip() 107 if m: 108 machines.append(m) 109 logging.debug('Read list of machines from file: %s', machines_file) 110 logging.debug('Machines: %s', ','.join(machines)) 111 112 if machines: 113 for machine in machines: 114 if not machine or re.search('\s', machine): 115 parser.parser.error("Invalid machine: %s" % str(machine)) 116 machines = list(set(machines)) 117 machines.sort() 118 return machines 119 120 121def _stage_ssp(parser): 122 """Stage server-side package. 123 124 This function calls a control segment to stage server-side package based on 125 the job and autoserv command line option. The detail implementation could 126 be different for each host type. Currently, only CrosHost has 127 stage_server_side_package function defined. 128 The script returns None if no server-side package is available. However, 129 it may raise exception if it failed for reasons other than artifact (the 130 server-side package) not found. 131 132 @param parser: Command line arguments parser passed in the autoserv process. 133 134 @return: (ssp_url, error_msg), where 135 ssp_url is a url to the autotest server-side package. None if 136 server-side package is not supported. 137 error_msg is a string indicating the failures. None if server- 138 side package is staged successfully. 139 """ 140 machines_list = _get_machines(parser) 141 if bool(parser.options.lab): 142 machines_list = server_job.get_machine_dicts( 143 machines_list, parser.options.lab) 144 145 # If test_source_build is not specified, default to use server-side test 146 # code from build specified in --image. 147 namespace = {'machines': machines_list, 148 'image': (parser.options.test_source_build or 149 parser.options.image),} 150 script_locals = {} 151 execfile(STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE, namespace, script_locals) 152 return script_locals['ssp_url'], script_locals['error_msg'] 153 154 155def _run_with_ssp(job, container_name, job_id, results, parser, ssp_url, 156 job_folder, machines): 157 """Run the server job with server-side packaging. 158 159 @param job: The server job object. 160 @param container_name: Name of the container to run the test. 161 @param job_id: ID of the test job. 162 @param results: Folder to store results. This could be different from 163 parser.options.results: 164 parser.options.results can be set to None for results to be 165 stored in a temp folder. 166 results can be None for autoserv run requires no logging. 167 @param parser: Command line parser that contains the options. 168 @param ssp_url: url of the staged server-side package. 169 @param job_folder: Name of the job result folder. 170 @param machines: A list of machines to run the test. 171 """ 172 bucket = lxc.ContainerBucket() 173 control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != '' 174 else None) 175 try: 176 dut_name = machines[0] if len(machines) >= 1 else None 177 test_container = bucket.setup_test(container_name, job_id, ssp_url, 178 results, control=control, 179 job_folder=job_folder, 180 dut_name=dut_name) 181 except Exception as e: 182 job.record('FAIL', None, None, 183 'Failed to setup container for test: %s. Check logs in ' 184 'ssp_logs folder for more details.' % e) 185 raise 186 187 args = sys.argv[:] 188 args.remove('--require-ssp') 189 # --parent_job_id is only useful in autoserv running in host, not in 190 # container. Include this argument will cause test to fail for builds before 191 # CL 286265 was merged. 192 if '--parent_job_id' in args: 193 index = args.index('--parent_job_id') 194 args.remove('--parent_job_id') 195 # Remove the actual parent job id in command line arg. 196 del args[index] 197 198 # A dictionary of paths to replace in the command line. Key is the path to 199 # be replaced with the one in value. 200 paths_to_replace = {} 201 # Replace the control file path with the one in container. 202 if control: 203 container_control_filename = os.path.join( 204 lxc.CONTROL_TEMP_PATH, os.path.basename(control)) 205 paths_to_replace[control] = container_control_filename 206 # Update result directory with the one in container. 207 container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % job_folder) 208 if parser.options.results: 209 paths_to_replace[parser.options.results] = container_result_dir 210 # Update parse_job directory with the one in container. The assumption is 211 # that the result folder to be parsed is always the same as the results_dir. 212 if parser.options.parse_job: 213 paths_to_replace[parser.options.parse_job] = container_result_dir 214 215 args = [paths_to_replace.get(arg, arg) for arg in args] 216 217 # Apply --use-existing-results, results directory is aready created and 218 # mounted in container. Apply this arg to avoid exception being raised. 219 if not '--use-existing-results' in args: 220 args.append('--use-existing-results') 221 222 # Make sure autoserv running in container using a different pid file. 223 if not '--pidfile-label' in args: 224 args.extend(['--pidfile-label', 'container_autoserv']) 225 226 cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args]) 227 logging.info('Run command in container: %s', cmd_line) 228 success = False 229 try: 230 test_container.attach_run(cmd_line) 231 success = True 232 except Exception as e: 233 # If the test run inside container fails without generating any log, 234 # write a message to status.log to help troubleshooting. 235 debug_files = os.listdir(os.path.join(results, 'debug')) 236 if not debug_files: 237 job.record('FAIL', None, None, 238 'Failed to run test inside the container: %s. Check ' 239 'logs in ssp_logs folder for more details.' % e) 240 raise 241 finally: 242 metrics.Counter( 243 'chromeos/autotest/experimental/execute_job_in_ssp').increment( 244 fields={'success': success}) 245 # metadata is uploaded separately so it can use http to upload. 246 metadata = {'drone': socket.gethostname(), 247 'job_id': job_id, 248 'success': success} 249 autotest_es.post(use_http=True, 250 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 251 metadata=metadata) 252 test_container.destroy() 253 254 255def correct_results_folder_permission(results): 256 """Make sure the results folder has the right permission settings. 257 258 For tests running with server-side packaging, the results folder has the 259 owner of root. This must be changed to the user running the autoserv 260 process, so parsing job can access the results folder. 261 TODO(dshi): crbug.com/459344 Remove this function when test container can be 262 unprivileged container. 263 264 @param results: Path to the results folder. 265 266 """ 267 if not results: 268 return 269 270 try: 271 utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results)) 272 utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results)) 273 except error.CmdError as e: 274 metadata = {'error': str(e), 275 'result_folder': results, 276 'drone': socket.gethostname()} 277 autotest_es.post(use_http=True, type_str='correct_results_folder_failure', 278 metadata=metadata) 279 raise 280 281 282def _start_servod(machine): 283 """Try to start servod in moblab if it's not already running or running with 284 different board or port. 285 286 @param machine: Name of the dut used for test. 287 """ 288 if not utils.is_moblab(): 289 return 290 291 logging.debug('Trying to start servod.') 292 try: 293 afe = frontend.AFE() 294 board = server_utils.get_board_from_afe(machine, afe) 295 hosts = afe.get_hosts(hostname=machine) 296 servo_host = hosts[0].attributes.get('servo_host', None) 297 servo_port = hosts[0].attributes.get('servo_port', 9999) 298 if not servo_host in ['localhost', '127.0.0.1']: 299 logging.warn('Starting servod is aborted. The dut\'s servo_host ' 300 'attribute is not set to localhost.') 301 return 302 except (urllib2.HTTPError, urllib2.URLError): 303 # Ignore error if RPC failed to get board 304 logging.error('Failed to get board name from AFE. Start servod is ' 305 'aborted') 306 return 307 308 try: 309 pid = utils.run('pgrep servod').stdout 310 cmd_line = utils.run('ps -fp %s' % pid).stdout 311 if ('--board %s' % board in cmd_line and 312 '--port %s' % servo_port in cmd_line): 313 logging.debug('Servod is already running with given board and port.' 314 ' There is no need to restart servod.') 315 return 316 logging.debug('Servod is running with different board or port. ' 317 'Stopping existing servod.') 318 utils.run('sudo stop servod') 319 except error.CmdError: 320 # servod is not running. 321 pass 322 323 try: 324 utils.run(START_SERVOD_CMD % (board, servo_port)) 325 logging.debug('Servod is started') 326 except error.CmdError as e: 327 logging.error('Servod failed to be started, error: %s', e) 328 329 330def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): 331 """Run server job with given options. 332 333 @param pid_file_manager: PidFileManager used to monitor the autoserv process 334 @param results: Folder to store results. 335 @param parser: Parser for the command line arguments. 336 @param ssp_url: Url to server-side package. 337 @param use_ssp: Set to True to run with server-side packaging. 338 """ 339 if parser.options.warn_no_ssp: 340 # Post a warning in the log. 341 logging.warn('Autoserv is required to run with server-side packaging. ' 342 'However, no drone is found to support server-side ' 343 'packaging. The test will be executed in a drone without ' 344 'server-side packaging supported.') 345 346 # send stdin to /dev/null 347 dev_null = os.open(os.devnull, os.O_RDONLY) 348 os.dup2(dev_null, sys.stdin.fileno()) 349 os.close(dev_null) 350 351 # Create separate process group if the process is not a process group 352 # leader. This allows autoserv process to keep running after the caller 353 # process (drone manager call) exits. 354 if os.getpid() != os.getpgid(0): 355 os.setsid() 356 357 # Container name is predefined so the container can be destroyed in 358 # handle_sigterm. 359 job_or_task_id = job_directories.get_job_id_or_task_id( 360 parser.options.results) 361 container_name = (lxc.TEST_CONTAINER_NAME_FMT % 362 (job_or_task_id, time.time(), os.getpid())) 363 job_folder = job_directories.get_job_folder_name(parser.options.results) 364 365 # Implement SIGTERM handler 366 def handle_sigterm(signum, frame): 367 logging.debug('Received SIGTERM') 368 if pid_file_manager: 369 pid_file_manager.close_file(1, signal.SIGTERM) 370 logging.debug('Finished writing to pid_file. Killing process.') 371 372 # Update results folder's file permission. This needs to be done ASAP 373 # before the parsing process tries to access the log. 374 if use_ssp and results: 375 correct_results_folder_permission(results) 376 377 # TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved. 378 # This sleep allows the pending output to be logged before the kill 379 # signal is sent. 380 time.sleep(.1) 381 if use_ssp: 382 logging.debug('Destroy container %s before aborting the autoserv ' 383 'process.', container_name) 384 metadata = {'drone': socket.gethostname(), 385 'job_id': job_or_task_id, 386 'container_name': container_name, 387 'action': 'abort', 388 'success': True} 389 try: 390 bucket = lxc.ContainerBucket() 391 container = bucket.get(container_name) 392 if container: 393 container.destroy() 394 else: 395 metadata['success'] = False 396 metadata['error'] = 'container not found' 397 logging.debug('Container %s is not found.', container_name) 398 except: 399 metadata['success'] = False 400 metadata['error'] = 'Exception: %s' % str(sys.exc_info()) 401 # Handle any exception so the autoserv process can be aborted. 402 logging.exception('Failed to destroy container %s.', 403 container_name) 404 autotest_es.post(use_http=True, 405 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 406 metadata=metadata) 407 # Try to correct the result file permission again after the 408 # container is destroyed, as the container might have created some 409 # new files in the result folder. 410 if results: 411 correct_results_folder_permission(results) 412 413 os.killpg(os.getpgrp(), signal.SIGKILL) 414 415 # Set signal handler 416 signal.signal(signal.SIGTERM, handle_sigterm) 417 418 # faulthandler is only needed to debug in the Lab and is not avaliable to 419 # be imported in the chroot as part of VMTest, so Try-Except it. 420 try: 421 import faulthandler 422 faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) 423 logging.debug('faulthandler registered on SIGTERM.') 424 except ImportError: 425 sys.exc_clear() 426 427 # Ignore SIGTTOU's generated by output from forked children. 428 signal.signal(signal.SIGTTOU, signal.SIG_IGN) 429 430 # If we received a SIGALARM, let's be loud about it. 431 signal.signal(signal.SIGALRM, log_alarm) 432 433 # Server side tests that call shell scripts often depend on $USER being set 434 # but depending on how you launch your autotest scheduler it may not be set. 435 os.environ['USER'] = getpass.getuser() 436 437 label = parser.options.label 438 group_name = parser.options.group_name 439 user = parser.options.user 440 client = parser.options.client 441 server = parser.options.server 442 install_before = parser.options.install_before 443 install_after = parser.options.install_after 444 verify = parser.options.verify 445 repair = parser.options.repair 446 cleanup = parser.options.cleanup 447 provision = parser.options.provision 448 reset = parser.options.reset 449 job_labels = parser.options.job_labels 450 no_tee = parser.options.no_tee 451 parse_job = parser.options.parse_job 452 execution_tag = parser.options.execution_tag 453 if not execution_tag: 454 execution_tag = parse_job 455 ssh_user = parser.options.ssh_user 456 ssh_port = parser.options.ssh_port 457 ssh_pass = parser.options.ssh_pass 458 collect_crashinfo = parser.options.collect_crashinfo 459 control_filename = parser.options.control_filename 460 test_retry = parser.options.test_retry 461 verify_job_repo_url = parser.options.verify_job_repo_url 462 skip_crash_collection = parser.options.skip_crash_collection 463 ssh_verbosity = int(parser.options.ssh_verbosity) 464 ssh_options = parser.options.ssh_options 465 no_use_packaging = parser.options.no_use_packaging 466 host_attributes = parser.options.host_attributes 467 in_lab = bool(parser.options.lab) 468 469 # can't be both a client and a server side test 470 if client and server: 471 parser.parser.error("Can not specify a test as both server and client!") 472 473 if provision and client: 474 parser.parser.error("Cannot specify provisioning and client!") 475 476 is_special_task = (verify or repair or cleanup or collect_crashinfo or 477 provision or reset) 478 if len(parser.args) < 1 and not is_special_task: 479 parser.parser.error("Missing argument: control file") 480 481 if ssh_verbosity > 0: 482 # ssh_verbosity is an integer between 0 and 3, inclusive 483 ssh_verbosity_flag = '-' + 'v' * ssh_verbosity 484 else: 485 ssh_verbosity_flag = '' 486 487 # We have a control file unless it's just a verify/repair/cleanup job 488 if len(parser.args) > 0: 489 control = parser.args[0] 490 else: 491 control = None 492 493 machines = _get_machines(parser) 494 if group_name and len(machines) < 2: 495 parser.parser.error('-G %r may only be supplied with more than one ' 496 'machine.' % group_name) 497 498 kwargs = {'group_name': group_name, 'tag': execution_tag, 499 'disable_sysinfo': parser.options.disable_sysinfo} 500 if parser.options.parent_job_id: 501 kwargs['parent_job_id'] = int(parser.options.parent_job_id) 502 if control_filename: 503 kwargs['control_filename'] = control_filename 504 if host_attributes: 505 kwargs['host_attributes'] = host_attributes 506 kwargs['in_lab'] = in_lab 507 job = server_job.server_job(control, parser.args[1:], results, label, 508 user, machines, client, parse_job, 509 ssh_user, ssh_port, ssh_pass, 510 ssh_verbosity_flag, ssh_options, 511 test_retry, **kwargs) 512 513 job.logging.start_logging() 514 job.init_parser() 515 516 # perform checks 517 job.precheck() 518 519 # run the job 520 exit_code = 0 521 auto_start_servod = _CONFIG.get_config_value( 522 'AUTOSERV', 'auto_start_servod', type=bool, default=False) 523 524 try: 525 with site_utils.SetupTsMonGlobalState('autoserv', indirect=True, short_lived=True): 526 try: 527 if repair: 528 if auto_start_servod and len(machines) == 1: 529 _start_servod(machines[0]) 530 job.repair(job_labels) 531 elif verify: 532 job.verify(job_labels) 533 elif provision: 534 job.provision(job_labels) 535 elif reset: 536 job.reset(job_labels) 537 elif cleanup: 538 job.cleanup(job_labels) 539 else: 540 if auto_start_servod and len(machines) == 1: 541 _start_servod(machines[0]) 542 if use_ssp: 543 try: 544 _run_with_ssp(job, container_name, job_or_task_id, 545 results, parser, ssp_url, job_folder, 546 machines) 547 finally: 548 # Update the ownership of files in result folder. 549 correct_results_folder_permission(results) 550 else: 551 if collect_crashinfo: 552 # Update the ownership of files in result folder. If the 553 # job to collect crashinfo was running inside container 554 # (SSP) and crashed before correcting folder permission, 555 # the result folder might have wrong permission setting. 556 try: 557 correct_results_folder_permission(results) 558 except: 559 # Ignore any error as the user may not have root 560 # permission to run sudo command. 561 pass 562 job.run(install_before, install_after, 563 verify_job_repo_url=verify_job_repo_url, 564 only_collect_crashinfo=collect_crashinfo, 565 skip_crash_collection=skip_crash_collection, 566 job_labels=job_labels, 567 use_packaging=(not no_use_packaging)) 568 finally: 569 while job.hosts: 570 host = job.hosts.pop() 571 host.close() 572 except: 573 exit_code = 1 574 traceback.print_exc() 575 576 if pid_file_manager: 577 pid_file_manager.num_tests_failed = job.num_tests_failed 578 pid_file_manager.close_file(exit_code) 579 job.cleanup_parser() 580 581 sys.exit(exit_code) 582 583 584def record_autoserv(options, duration_secs): 585 """Record autoserv end-to-end time in metadata db. 586 587 @param options: parser options. 588 @param duration_secs: How long autoserv has taken, in secs. 589 """ 590 # Get machine hostname 591 machines = options.machines.replace( 592 ',', ' ').strip().split() if options.machines else [] 593 num_machines = len(machines) 594 if num_machines > 1: 595 # Skip the case where atomic group is used. 596 return 597 elif num_machines == 0: 598 machines.append('hostless') 599 600 # Determine the status that will be reported. 601 s = job_overhead.STATUS 602 task_mapping = { 603 'reset': s.RESETTING, 'verify': s.VERIFYING, 604 'provision': s.PROVISIONING, 'repair': s.REPAIRING, 605 'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING} 606 match = filter(lambda task: getattr(options, task, False) == True, 607 task_mapping) 608 status = task_mapping[match[0]] if match else s.RUNNING 609 is_special_task = status not in [s.RUNNING, s.GATHERING] 610 job_or_task_id = job_directories.get_job_id_or_task_id(options.results) 611 job_overhead.record_state_duration( 612 job_or_task_id, machines[0], status, duration_secs, 613 is_special_task=is_special_task) 614 615 616def main(): 617 start_time = datetime.datetime.now() 618 # grab the parser 619 parser = autoserv_parser.autoserv_parser 620 parser.parse_args() 621 622 if len(sys.argv) == 1: 623 parser.parser.print_help() 624 sys.exit(1) 625 626 # If the job requires to run with server-side package, try to stage server- 627 # side package first. If that fails with error that autotest server package 628 # does not exist, fall back to run the job without using server-side 629 # packaging. If option warn_no_ssp is specified, that means autoserv is 630 # running in a drone does not support SSP, thus no need to stage server-side 631 # package. 632 ssp_url = None 633 ssp_url_warning = False 634 if (not parser.options.warn_no_ssp and parser.options.require_ssp): 635 ssp_url, ssp_error_msg = _stage_ssp(parser) 636 # The build does not have autotest server package. Fall back to not 637 # to use server-side package. Logging is postponed until logging being 638 # set up. 639 ssp_url_warning = not ssp_url 640 641 if parser.options.no_logging: 642 results = None 643 else: 644 results = parser.options.results 645 if not results: 646 results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') 647 results = os.path.abspath(results) 648 resultdir_exists = False 649 for filename in ('control.srv', 'status.log', '.autoserv_execute'): 650 if os.path.exists(os.path.join(results, filename)): 651 resultdir_exists = True 652 if not parser.options.use_existing_results and resultdir_exists: 653 error = "Error: results directory already exists: %s\n" % results 654 sys.stderr.write(error) 655 sys.exit(1) 656 657 # Now that we certified that there's no leftover results dir from 658 # previous jobs, lets create the result dir since the logging system 659 # needs to create the log file in there. 660 if not os.path.isdir(results): 661 os.makedirs(results) 662 663 # Server-side packaging will only be used if it's required and the package 664 # is available. If warn_no_ssp is specified, it means that autoserv is 665 # running in a drone does not have SSP supported and a warning will be logs. 666 # Therefore, it should not run with SSP. 667 use_ssp = (not parser.options.warn_no_ssp and parser.options.require_ssp 668 and ssp_url) 669 if use_ssp: 670 log_dir = os.path.join(results, 'ssp_logs') if results else None 671 if log_dir and not os.path.exists(log_dir): 672 os.makedirs(log_dir) 673 else: 674 log_dir = results 675 676 logging_manager.configure_logging( 677 server_logging_config.ServerLoggingConfig(), 678 results_dir=log_dir, 679 use_console=not parser.options.no_tee, 680 verbose=parser.options.verbose, 681 no_console_prefix=parser.options.no_console_prefix) 682 683 if ssp_url_warning: 684 logging.warn( 685 'Autoserv is required to run with server-side packaging. ' 686 'However, no server-side package can be found based on ' 687 '`--image`, host attribute job_repo_url or host OS version ' 688 'label. It could be that the build to test is older than the ' 689 'minimum version that supports server-side packaging. The test ' 690 'will be executed without using erver-side packaging. ' 691 'Following is the detailed error:\n%s', ssp_error_msg) 692 693 if results: 694 logging.info("Results placed in %s" % results) 695 696 # wait until now to perform this check, so it get properly logged 697 if (parser.options.use_existing_results and not resultdir_exists and 698 not utils.is_in_container()): 699 logging.error("No existing results directory found: %s", results) 700 sys.exit(1) 701 702 logging.debug('autoserv is running in drone %s.', socket.gethostname()) 703 logging.debug('autoserv command was: %s', ' '.join(sys.argv)) 704 705 if parser.options.write_pidfile and results: 706 pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, 707 results) 708 pid_file_manager.open_file() 709 else: 710 pid_file_manager = None 711 712 autotest.BaseAutotest.set_install_in_tmpdir( 713 parser.options.install_in_tmpdir) 714 715 try: 716 # Take the first argument as control file name, get the test name from 717 # the control file. 718 if (len(parser.args) > 0 and parser.args[0] != '' and 719 parser.options.machines): 720 try: 721 test_name = control_data.parse_control(parser.args[0], 722 raise_warnings=True).name 723 except control_data.ControlVariableException: 724 logging.debug('Failed to retrieve test name from control file.') 725 test_name = None 726 except control_data.ControlVariableException as e: 727 logging.error(str(e)) 728 exit_code = 0 729 # TODO(beeps): Extend this to cover different failure modes. 730 # Testing exceptions are matched against labels sent to autoserv. Eg, 731 # to allow only the hostless job to run, specify 732 # testing_exceptions: test_suite in the shadow_config. To allow both 733 # the hostless job and dummy_Pass to run, specify 734 # testing_exceptions: test_suite,dummy_Pass. You can figure out 735 # what label autoserv is invoked with by looking through the logs of a test 736 # for the autoserv command's -l option. 737 testing_exceptions = _CONFIG.get_config_value( 738 'AUTOSERV', 'testing_exceptions', type=list, default=[]) 739 test_mode = _CONFIG.get_config_value( 740 'AUTOSERV', 'testing_mode', type=bool, default=False) 741 test_mode = (results_mocker and test_mode and not 742 any([ex in parser.options.label 743 for ex in testing_exceptions])) 744 is_task = (parser.options.verify or parser.options.repair or 745 parser.options.provision or parser.options.reset or 746 parser.options.cleanup or parser.options.collect_crashinfo) 747 try: 748 try: 749 if test_mode: 750 # The parser doesn't run on tasks anyway, so we can just return 751 # happy signals without faking results. 752 if not is_task: 753 machine = parser.options.results.split('/')[-1] 754 755 # TODO(beeps): The proper way to do this would be to 756 # refactor job creation so we can invoke job.record 757 # directly. To do that one needs to pipe the test_name 758 # through run_autoserv and bail just before invoking 759 # the server job. See the comment in 760 # puppylab/results_mocker for more context. 761 results_mocker.ResultsMocker( 762 test_name if test_name else 'unknown-test', 763 parser.options.results, machine 764 ).mock_results() 765 return 766 else: 767 run_autoserv(pid_file_manager, results, parser, ssp_url, 768 use_ssp) 769 except SystemExit as e: 770 exit_code = e.code 771 if exit_code: 772 logging.exception(e) 773 except Exception as e: 774 # If we don't know what happened, we'll classify it as 775 # an 'abort' and return 1. 776 logging.exception(e) 777 exit_code = 1 778 finally: 779 if pid_file_manager: 780 pid_file_manager.close_file(exit_code) 781 # Record the autoserv duration time. Must be called 782 # just before the system exits to ensure accuracy. 783 duration_secs = (datetime.datetime.now() - start_time).total_seconds() 784 record_autoserv(parser.options, duration_secs) 785 sys.exit(exit_code) 786 787 788if __name__ == '__main__': 789 main() 790