autoserv revision 92bf7b67c855085727ee3e630003da8278dd50c8
1#!/usr/bin/python -u 2# Copyright 2007-2008 Martin J. Bligh <mbligh@google.com>, Google Inc. 3# Released under the GPL v2 4 5""" 6Run a control file through the server side engine 7""" 8 9import datetime 10import contextlib 11import getpass 12import logging 13import os 14import re 15import signal 16import socket 17import sys 18import traceback 19import time 20import urllib2 21 22 23import common 24from autotest_lib.client.common_lib import control_data 25from autotest_lib.client.common_lib import error 26from autotest_lib.client.common_lib import global_config 27from autotest_lib.client.common_lib import utils 28from autotest_lib.client.common_lib.cros.graphite import autotest_es 29 30try: 31 from chromite.lib import metrics 32except ImportError: 33 metrics = utils.metrics_mock 34 35try: 36 from autotest_lib.puppylab import results_mocker 37except ImportError: 38 results_mocker = None 39 40_CONFIG = global_config.global_config 41 42require_atfork = _CONFIG.get_config_value( 43 'AUTOSERV', 'require_atfork_module', type=bool, default=True) 44 45 46# Number of seconds to wait before returning if testing mode is enabled 47TESTING_MODE_SLEEP_SECS = 1 48 49try: 50 import atfork 51 atfork.monkeypatch_os_fork_functions() 52 import atfork.stdlib_fixer 53 # Fix the Python standard library for threading+fork safety with its 54 # internal locks. http://code.google.com/p/python-atfork/ 55 import warnings 56 warnings.filterwarnings('ignore', 'logging module already imported') 57 atfork.stdlib_fixer.fix_logging_module() 58except ImportError, e: 59 from autotest_lib.client.common_lib import global_config 60 if _CONFIG.get_config_value( 61 'AUTOSERV', 'require_atfork_module', type=bool, default=False): 62 print >>sys.stderr, 'Please run utils/build_externals.py' 63 print e 64 sys.exit(1) 65 66from autotest_lib.server import frontend 67from autotest_lib.server import server_logging_config 68from autotest_lib.server import server_job, utils, autoserv_parser, autotest 69from autotest_lib.server import utils as server_utils 70from autotest_lib.server import site_utils 71from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 72from autotest_lib.site_utils import job_directories 73from autotest_lib.site_utils import job_overhead 74from autotest_lib.site_utils import lxc 75from autotest_lib.site_utils import lxc_utils 76from autotest_lib.client.common_lib import pidfile, logging_manager 77 78 79# Control segment to stage server-side package. 80STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path( 81 'stage_server_side_package') 82 83# Command line to start servod in a moblab. 84START_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s' 85STOP_SERVOD_CMD = 'sudo stop servod' 86 87def log_alarm(signum, frame): 88 logging.error("Received SIGALARM. Ignoring and continuing on.") 89 sys.exit(1) 90 91 92def _get_machines(parser): 93 """Get a list of machine names from command line arg -m or a file. 94 95 @param parser: Parser for the command line arguments. 96 97 @return: A list of machine names from command line arg -m or the 98 machines file specified in the command line arg -M. 99 """ 100 if parser.options.machines: 101 machines = parser.options.machines.replace(',', ' ').strip().split() 102 else: 103 machines = [] 104 machines_file = parser.options.machines_file 105 if machines_file: 106 machines = [] 107 for m in open(machines_file, 'r').readlines(): 108 # remove comments, spaces 109 m = re.sub('#.*', '', m).strip() 110 if m: 111 machines.append(m) 112 logging.debug('Read list of machines from file: %s', machines_file) 113 logging.debug('Machines: %s', ','.join(machines)) 114 115 if machines: 116 for machine in machines: 117 if not machine or re.search('\s', machine): 118 parser.parser.error("Invalid machine: %s" % str(machine)) 119 machines = list(set(machines)) 120 machines.sort() 121 return machines 122 123 124def _stage_ssp(parser): 125 """Stage server-side package. 126 127 This function calls a control segment to stage server-side package based on 128 the job and autoserv command line option. The detail implementation could 129 be different for each host type. Currently, only CrosHost has 130 stage_server_side_package function defined. 131 The script returns None if no server-side package is available. However, 132 it may raise exception if it failed for reasons other than artifact (the 133 server-side package) not found. 134 135 @param parser: Command line arguments parser passed in the autoserv process. 136 137 @return: (ssp_url, error_msg), where 138 ssp_url is a url to the autotest server-side package. None if 139 server-side package is not supported. 140 error_msg is a string indicating the failures. None if server- 141 side package is staged successfully. 142 """ 143 machines_list = _get_machines(parser) 144 machines_list = server_job.get_machine_dicts( 145 machines_list, parser.options.lab, parser.options.host_attributes) 146 147 # If test_source_build is not specified, default to use server-side test 148 # code from build specified in --image. 149 namespace = {'machines': machines_list, 150 'image': (parser.options.test_source_build or 151 parser.options.image),} 152 script_locals = {} 153 execfile(STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE, namespace, script_locals) 154 return script_locals['ssp_url'], script_locals['error_msg'] 155 156 157def _run_with_ssp(job, container_name, job_id, results, parser, ssp_url, 158 job_folder, machines): 159 """Run the server job with server-side packaging. 160 161 @param job: The server job object. 162 @param container_name: Name of the container to run the test. 163 @param job_id: ID of the test job. 164 @param results: Folder to store results. This could be different from 165 parser.options.results: 166 parser.options.results can be set to None for results to be 167 stored in a temp folder. 168 results can be None for autoserv run requires no logging. 169 @param parser: Command line parser that contains the options. 170 @param ssp_url: url of the staged server-side package. 171 @param job_folder: Name of the job result folder. 172 @param machines: A list of machines to run the test. 173 """ 174 bucket = lxc.ContainerBucket() 175 control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != '' 176 else None) 177 try: 178 dut_name = machines[0] if len(machines) >= 1 else None 179 test_container = bucket.setup_test(container_name, job_id, ssp_url, 180 results, control=control, 181 job_folder=job_folder, 182 dut_name=dut_name) 183 except Exception as e: 184 job.record('FAIL', None, None, 185 'Failed to setup container for test: %s. Check logs in ' 186 'ssp_logs folder for more details.' % e) 187 raise 188 189 args = sys.argv[:] 190 args.remove('--require-ssp') 191 # --parent_job_id is only useful in autoserv running in host, not in 192 # container. Include this argument will cause test to fail for builds before 193 # CL 286265 was merged. 194 if '--parent_job_id' in args: 195 index = args.index('--parent_job_id') 196 args.remove('--parent_job_id') 197 # Remove the actual parent job id in command line arg. 198 del args[index] 199 200 # A dictionary of paths to replace in the command line. Key is the path to 201 # be replaced with the one in value. 202 paths_to_replace = {} 203 # Replace the control file path with the one in container. 204 if control: 205 container_control_filename = os.path.join( 206 lxc.CONTROL_TEMP_PATH, os.path.basename(control)) 207 paths_to_replace[control] = container_control_filename 208 # Update result directory with the one in container. 209 container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % job_folder) 210 if parser.options.results: 211 paths_to_replace[parser.options.results] = container_result_dir 212 # Update parse_job directory with the one in container. The assumption is 213 # that the result folder to be parsed is always the same as the results_dir. 214 if parser.options.parse_job: 215 paths_to_replace[parser.options.parse_job] = container_result_dir 216 217 args = [paths_to_replace.get(arg, arg) for arg in args] 218 219 # Apply --use-existing-results, results directory is aready created and 220 # mounted in container. Apply this arg to avoid exception being raised. 221 if not '--use-existing-results' in args: 222 args.append('--use-existing-results') 223 224 # Make sure autoserv running in container using a different pid file. 225 if not '--pidfile-label' in args: 226 args.extend(['--pidfile-label', 'container_autoserv']) 227 228 cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args]) 229 logging.info('Run command in container: %s', cmd_line) 230 success = False 231 try: 232 test_container.attach_run(cmd_line) 233 success = True 234 except Exception as e: 235 # If the test run inside container fails without generating any log, 236 # write a message to status.log to help troubleshooting. 237 debug_files = os.listdir(os.path.join(results, 'debug')) 238 if not debug_files: 239 job.record('FAIL', None, None, 240 'Failed to run test inside the container: %s. Check ' 241 'logs in ssp_logs folder for more details.' % e) 242 raise 243 finally: 244 metrics.Counter( 245 'chromeos/autotest/experimental/execute_job_in_ssp').increment( 246 fields={'success': success}) 247 # metadata is uploaded separately so it can use http to upload. 248 metadata = {'drone': socket.gethostname(), 249 'job_id': job_id, 250 'success': success} 251 autotest_es.post(use_http=True, 252 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 253 metadata=metadata) 254 test_container.destroy() 255 256 257def correct_results_folder_permission(results): 258 """Make sure the results folder has the right permission settings. 259 260 For tests running with server-side packaging, the results folder has the 261 owner of root. This must be changed to the user running the autoserv 262 process, so parsing job can access the results folder. 263 TODO(dshi): crbug.com/459344 Remove this function when test container can be 264 unprivileged container. 265 266 @param results: Path to the results folder. 267 268 """ 269 if not results: 270 return 271 272 try: 273 utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results)) 274 utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results)) 275 except error.CmdError as e: 276 metadata = {'error': str(e), 277 'result_folder': results, 278 'drone': socket.gethostname()} 279 autotest_es.post(use_http=True, type_str='correct_results_folder_failure', 280 metadata=metadata) 281 raise 282 283 284def _start_servod(machine): 285 """Try to start servod in moblab if it's not already running or running with 286 different board or port. 287 288 @param machine: Name of the dut used for test. 289 """ 290 if not utils.is_moblab(): 291 return 292 293 logging.debug('Trying to start servod.') 294 try: 295 afe = frontend.AFE() 296 board = server_utils.get_board_from_afe(machine, afe) 297 hosts = afe.get_hosts(hostname=machine) 298 servo_host = hosts[0].attributes.get('servo_host', None) 299 servo_port = hosts[0].attributes.get('servo_port', 9999) 300 if not servo_host in ['localhost', '127.0.0.1']: 301 logging.warn('Starting servod is aborted. The dut\'s servo_host ' 302 'attribute is not set to localhost.') 303 return 304 except (urllib2.HTTPError, urllib2.URLError): 305 # Ignore error if RPC failed to get board 306 logging.error('Failed to get board name from AFE. Start servod is ' 307 'aborted') 308 return 309 310 try: 311 pid = utils.run('pgrep servod').stdout 312 cmd_line = utils.run('ps -fp %s' % pid).stdout 313 if ('--board %s' % board in cmd_line and 314 '--port %s' % servo_port in cmd_line): 315 logging.debug('Servod is already running with given board and port.' 316 ' There is no need to restart servod.') 317 return 318 logging.debug('Servod is running with different board or port. ' 319 'Stopping existing servod.') 320 utils.run('sudo stop servod') 321 except error.CmdError: 322 # servod is not running. 323 pass 324 325 try: 326 utils.run(START_SERVOD_CMD % (board, servo_port)) 327 logging.debug('Servod is started') 328 except error.CmdError as e: 329 logging.error('Servod failed to be started, error: %s', e) 330 331 332def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): 333 """Run server job with given options. 334 335 @param pid_file_manager: PidFileManager used to monitor the autoserv process 336 @param results: Folder to store results. 337 @param parser: Parser for the command line arguments. 338 @param ssp_url: Url to server-side package. 339 @param use_ssp: Set to True to run with server-side packaging. 340 """ 341 if parser.options.warn_no_ssp: 342 # Post a warning in the log. 343 logging.warn('Autoserv is required to run with server-side packaging. ' 344 'However, no drone is found to support server-side ' 345 'packaging. The test will be executed in a drone without ' 346 'server-side packaging supported.') 347 348 # send stdin to /dev/null 349 dev_null = os.open(os.devnull, os.O_RDONLY) 350 os.dup2(dev_null, sys.stdin.fileno()) 351 os.close(dev_null) 352 353 # Create separate process group if the process is not a process group 354 # leader. This allows autoserv process to keep running after the caller 355 # process (drone manager call) exits. 356 if os.getpid() != os.getpgid(0): 357 os.setsid() 358 359 # Container name is predefined so the container can be destroyed in 360 # handle_sigterm. 361 job_or_task_id = job_directories.get_job_id_or_task_id( 362 parser.options.results) 363 container_name = (lxc.TEST_CONTAINER_NAME_FMT % 364 (job_or_task_id, time.time(), os.getpid())) 365 job_folder = job_directories.get_job_folder_name(parser.options.results) 366 367 # Implement SIGTERM handler 368 def handle_sigterm(signum, frame): 369 logging.debug('Received SIGTERM') 370 if pid_file_manager: 371 pid_file_manager.close_file(1, signal.SIGTERM) 372 logging.debug('Finished writing to pid_file. Killing process.') 373 374 # Update results folder's file permission. This needs to be done ASAP 375 # before the parsing process tries to access the log. 376 if use_ssp and results: 377 correct_results_folder_permission(results) 378 379 # TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved. 380 # This sleep allows the pending output to be logged before the kill 381 # signal is sent. 382 time.sleep(.1) 383 if use_ssp: 384 logging.debug('Destroy container %s before aborting the autoserv ' 385 'process.', container_name) 386 metadata = {'drone': socket.gethostname(), 387 'job_id': job_or_task_id, 388 'container_name': container_name, 389 'action': 'abort', 390 'success': True} 391 try: 392 bucket = lxc.ContainerBucket() 393 container = bucket.get(container_name) 394 if container: 395 container.destroy() 396 else: 397 metadata['success'] = False 398 metadata['error'] = 'container not found' 399 logging.debug('Container %s is not found.', container_name) 400 except: 401 metadata['success'] = False 402 metadata['error'] = 'Exception: %s' % str(sys.exc_info()) 403 # Handle any exception so the autoserv process can be aborted. 404 logging.exception('Failed to destroy container %s.', 405 container_name) 406 autotest_es.post(use_http=True, 407 type_str=lxc.CONTAINER_RUN_TEST_METADB_TYPE, 408 metadata=metadata) 409 # Try to correct the result file permission again after the 410 # container is destroyed, as the container might have created some 411 # new files in the result folder. 412 if results: 413 correct_results_folder_permission(results) 414 415 os.killpg(os.getpgrp(), signal.SIGKILL) 416 417 # Set signal handler 418 signal.signal(signal.SIGTERM, handle_sigterm) 419 420 # faulthandler is only needed to debug in the Lab and is not avaliable to 421 # be imported in the chroot as part of VMTest, so Try-Except it. 422 try: 423 import faulthandler 424 faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) 425 logging.debug('faulthandler registered on SIGTERM.') 426 except ImportError: 427 sys.exc_clear() 428 429 # Ignore SIGTTOU's generated by output from forked children. 430 signal.signal(signal.SIGTTOU, signal.SIG_IGN) 431 432 # If we received a SIGALARM, let's be loud about it. 433 signal.signal(signal.SIGALRM, log_alarm) 434 435 # Server side tests that call shell scripts often depend on $USER being set 436 # but depending on how you launch your autotest scheduler it may not be set. 437 os.environ['USER'] = getpass.getuser() 438 439 label = parser.options.label 440 group_name = parser.options.group_name 441 user = parser.options.user 442 client = parser.options.client 443 server = parser.options.server 444 install_before = parser.options.install_before 445 install_after = parser.options.install_after 446 verify = parser.options.verify 447 repair = parser.options.repair 448 cleanup = parser.options.cleanup 449 provision = parser.options.provision 450 reset = parser.options.reset 451 job_labels = parser.options.job_labels 452 no_tee = parser.options.no_tee 453 parse_job = parser.options.parse_job 454 execution_tag = parser.options.execution_tag 455 if not execution_tag: 456 execution_tag = parse_job 457 ssh_user = parser.options.ssh_user 458 ssh_port = parser.options.ssh_port 459 ssh_pass = parser.options.ssh_pass 460 collect_crashinfo = parser.options.collect_crashinfo 461 control_filename = parser.options.control_filename 462 test_retry = parser.options.test_retry 463 verify_job_repo_url = parser.options.verify_job_repo_url 464 skip_crash_collection = parser.options.skip_crash_collection 465 ssh_verbosity = int(parser.options.ssh_verbosity) 466 ssh_options = parser.options.ssh_options 467 no_use_packaging = parser.options.no_use_packaging 468 host_attributes = parser.options.host_attributes 469 in_lab = bool(parser.options.lab) 470 471 # can't be both a client and a server side test 472 if client and server: 473 parser.parser.error("Can not specify a test as both server and client!") 474 475 if provision and client: 476 parser.parser.error("Cannot specify provisioning and client!") 477 478 is_special_task = (verify or repair or cleanup or collect_crashinfo or 479 provision or reset) 480 if len(parser.args) < 1 and not is_special_task: 481 parser.parser.error("Missing argument: control file") 482 483 if ssh_verbosity > 0: 484 # ssh_verbosity is an integer between 0 and 3, inclusive 485 ssh_verbosity_flag = '-' + 'v' * ssh_verbosity 486 else: 487 ssh_verbosity_flag = '' 488 489 # We have a control file unless it's just a verify/repair/cleanup job 490 if len(parser.args) > 0: 491 control = parser.args[0] 492 else: 493 control = None 494 495 machines = _get_machines(parser) 496 if group_name and len(machines) < 2: 497 parser.parser.error('-G %r may only be supplied with more than one ' 498 'machine.' % group_name) 499 500 kwargs = {'group_name': group_name, 'tag': execution_tag, 501 'disable_sysinfo': parser.options.disable_sysinfo} 502 if parser.options.parent_job_id: 503 kwargs['parent_job_id'] = int(parser.options.parent_job_id) 504 if control_filename: 505 kwargs['control_filename'] = control_filename 506 if host_attributes: 507 kwargs['host_attributes'] = host_attributes 508 kwargs['in_lab'] = in_lab 509 job = server_job.server_job(control, parser.args[1:], results, label, 510 user, machines, client, parse_job, 511 ssh_user, ssh_port, ssh_pass, 512 ssh_verbosity_flag, ssh_options, 513 test_retry, **kwargs) 514 515 job.logging.start_logging() 516 job.init_parser() 517 518 # perform checks 519 job.precheck() 520 521 # run the job 522 exit_code = 0 523 auto_start_servod = _CONFIG.get_config_value( 524 'AUTOSERV', 'auto_start_servod', type=bool, default=False) 525 526 site_utils.SetupTsMonGlobalState('autoserv', indirect=False, 527 short_lived=True) 528 try: 529 try: 530 if repair: 531 if auto_start_servod and len(machines) == 1: 532 _start_servod(machines[0]) 533 job.repair(job_labels) 534 elif verify: 535 job.verify(job_labels) 536 elif provision: 537 job.provision(job_labels) 538 elif reset: 539 job.reset(job_labels) 540 elif cleanup: 541 job.cleanup(job_labels) 542 else: 543 if auto_start_servod and len(machines) == 1: 544 _start_servod(machines[0]) 545 if use_ssp: 546 try: 547 _run_with_ssp(job, container_name, job_or_task_id, 548 results, parser, ssp_url, job_folder, 549 machines) 550 finally: 551 # Update the ownership of files in result folder. 552 correct_results_folder_permission(results) 553 else: 554 if collect_crashinfo: 555 # Update the ownership of files in result folder. If the 556 # job to collect crashinfo was running inside container 557 # (SSP) and crashed before correcting folder permission, 558 # the result folder might have wrong permission setting. 559 try: 560 correct_results_folder_permission(results) 561 except: 562 # Ignore any error as the user may not have root 563 # permission to run sudo command. 564 pass 565 metric_name = ('chromeos/autotest/experimental/' 566 'autoserv_job_run_duration') 567 f = {'in_container': utils.is_in_container(), 568 'success': False} 569 with metrics.SecondsTimer(metric_name, fields=f) as c: 570 job.run(install_before, install_after, 571 verify_job_repo_url=verify_job_repo_url, 572 only_collect_crashinfo=collect_crashinfo, 573 skip_crash_collection=skip_crash_collection, 574 job_labels=job_labels, 575 use_packaging=(not no_use_packaging)) 576 c['success'] = True 577 578 finally: 579 while job.hosts: 580 host = job.hosts.pop() 581 host.close() 582 except: 583 exit_code = 1 584 traceback.print_exc() 585 finally: 586 metrics.Flush() 587 588 if pid_file_manager: 589 pid_file_manager.num_tests_failed = job.num_tests_failed 590 pid_file_manager.close_file(exit_code) 591 job.cleanup_parser() 592 593 sys.exit(exit_code) 594 595 596def record_autoserv(options, duration_secs): 597 """Record autoserv end-to-end time in metadata db. 598 599 @param options: parser options. 600 @param duration_secs: How long autoserv has taken, in secs. 601 """ 602 # Get machine hostname 603 machines = options.machines.replace( 604 ',', ' ').strip().split() if options.machines else [] 605 num_machines = len(machines) 606 if num_machines > 1: 607 # Skip the case where atomic group is used. 608 return 609 elif num_machines == 0: 610 machines.append('hostless') 611 612 # Determine the status that will be reported. 613 s = job_overhead.STATUS 614 task_mapping = { 615 'reset': s.RESETTING, 'verify': s.VERIFYING, 616 'provision': s.PROVISIONING, 'repair': s.REPAIRING, 617 'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING} 618 match = filter(lambda task: getattr(options, task, False) == True, 619 task_mapping) 620 status = task_mapping[match[0]] if match else s.RUNNING 621 is_special_task = status not in [s.RUNNING, s.GATHERING] 622 job_or_task_id = job_directories.get_job_id_or_task_id(options.results) 623 job_overhead.record_state_duration( 624 job_or_task_id, machines[0], status, duration_secs, 625 is_special_task=is_special_task) 626 627 628def main(): 629 start_time = datetime.datetime.now() 630 # grab the parser 631 parser = autoserv_parser.autoserv_parser 632 parser.parse_args() 633 634 if len(sys.argv) == 1: 635 parser.parser.print_help() 636 sys.exit(1) 637 638 # If the job requires to run with server-side package, try to stage server- 639 # side package first. If that fails with error that autotest server package 640 # does not exist, fall back to run the job without using server-side 641 # packaging. If option warn_no_ssp is specified, that means autoserv is 642 # running in a drone does not support SSP, thus no need to stage server-side 643 # package. 644 ssp_url = None 645 ssp_url_warning = False 646 if (not parser.options.warn_no_ssp and parser.options.require_ssp): 647 ssp_url, ssp_error_msg = _stage_ssp(parser) 648 # The build does not have autotest server package. Fall back to not 649 # to use server-side package. Logging is postponed until logging being 650 # set up. 651 ssp_url_warning = not ssp_url 652 653 if parser.options.no_logging: 654 results = None 655 else: 656 results = parser.options.results 657 if not results: 658 results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') 659 results = os.path.abspath(results) 660 resultdir_exists = False 661 for filename in ('control.srv', 'status.log', '.autoserv_execute'): 662 if os.path.exists(os.path.join(results, filename)): 663 resultdir_exists = True 664 if not parser.options.use_existing_results and resultdir_exists: 665 error = "Error: results directory already exists: %s\n" % results 666 sys.stderr.write(error) 667 sys.exit(1) 668 669 # Now that we certified that there's no leftover results dir from 670 # previous jobs, lets create the result dir since the logging system 671 # needs to create the log file in there. 672 if not os.path.isdir(results): 673 os.makedirs(results) 674 675 # Server-side packaging will only be used if it's required and the package 676 # is available. If warn_no_ssp is specified, it means that autoserv is 677 # running in a drone does not have SSP supported and a warning will be logs. 678 # Therefore, it should not run with SSP. 679 use_ssp = (not parser.options.warn_no_ssp and parser.options.require_ssp 680 and ssp_url) 681 if use_ssp: 682 log_dir = os.path.join(results, 'ssp_logs') if results else None 683 if log_dir and not os.path.exists(log_dir): 684 os.makedirs(log_dir) 685 else: 686 log_dir = results 687 688 logging_manager.configure_logging( 689 server_logging_config.ServerLoggingConfig(), 690 results_dir=log_dir, 691 use_console=not parser.options.no_tee, 692 verbose=parser.options.verbose, 693 no_console_prefix=parser.options.no_console_prefix) 694 695 if ssp_url_warning: 696 logging.warn( 697 'Autoserv is required to run with server-side packaging. ' 698 'However, no server-side package can be found based on ' 699 '`--image`, host attribute job_repo_url or host OS version ' 700 'label. It could be that the build to test is older than the ' 701 'minimum version that supports server-side packaging. The test ' 702 'will be executed without using erver-side packaging. ' 703 'Following is the detailed error:\n%s', ssp_error_msg) 704 705 if results: 706 logging.info("Results placed in %s" % results) 707 708 # wait until now to perform this check, so it get properly logged 709 if (parser.options.use_existing_results and not resultdir_exists and 710 not utils.is_in_container()): 711 logging.error("No existing results directory found: %s", results) 712 sys.exit(1) 713 714 logging.debug('autoserv is running in drone %s.', socket.gethostname()) 715 logging.debug('autoserv command was: %s', ' '.join(sys.argv)) 716 717 if parser.options.write_pidfile and results: 718 pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, 719 results) 720 pid_file_manager.open_file() 721 else: 722 pid_file_manager = None 723 724 autotest.BaseAutotest.set_install_in_tmpdir( 725 parser.options.install_in_tmpdir) 726 727 try: 728 # Take the first argument as control file name, get the test name from 729 # the control file. 730 if (len(parser.args) > 0 and parser.args[0] != '' and 731 parser.options.machines): 732 try: 733 test_name = control_data.parse_control(parser.args[0], 734 raise_warnings=True).name 735 except control_data.ControlVariableException: 736 logging.debug('Failed to retrieve test name from control file.') 737 test_name = None 738 except control_data.ControlVariableException as e: 739 logging.error(str(e)) 740 exit_code = 0 741 # TODO(beeps): Extend this to cover different failure modes. 742 # Testing exceptions are matched against labels sent to autoserv. Eg, 743 # to allow only the hostless job to run, specify 744 # testing_exceptions: test_suite in the shadow_config. To allow both 745 # the hostless job and dummy_Pass to run, specify 746 # testing_exceptions: test_suite,dummy_Pass. You can figure out 747 # what label autoserv is invoked with by looking through the logs of a test 748 # for the autoserv command's -l option. 749 testing_exceptions = _CONFIG.get_config_value( 750 'AUTOSERV', 'testing_exceptions', type=list, default=[]) 751 test_mode = _CONFIG.get_config_value( 752 'AUTOSERV', 'testing_mode', type=bool, default=False) 753 test_mode = (results_mocker and test_mode and not 754 any([ex in parser.options.label 755 for ex in testing_exceptions])) 756 is_task = (parser.options.verify or parser.options.repair or 757 parser.options.provision or parser.options.reset or 758 parser.options.cleanup or parser.options.collect_crashinfo) 759 try: 760 try: 761 if test_mode: 762 # The parser doesn't run on tasks anyway, so we can just return 763 # happy signals without faking results. 764 if not is_task: 765 machine = parser.options.results.split('/')[-1] 766 767 # TODO(beeps): The proper way to do this would be to 768 # refactor job creation so we can invoke job.record 769 # directly. To do that one needs to pipe the test_name 770 # through run_autoserv and bail just before invoking 771 # the server job. See the comment in 772 # puppylab/results_mocker for more context. 773 results_mocker.ResultsMocker( 774 test_name if test_name else 'unknown-test', 775 parser.options.results, machine 776 ).mock_results() 777 return 778 else: 779 run_autoserv(pid_file_manager, results, parser, ssp_url, 780 use_ssp) 781 except SystemExit as e: 782 exit_code = e.code 783 if exit_code: 784 logging.exception(e) 785 except Exception as e: 786 # If we don't know what happened, we'll classify it as 787 # an 'abort' and return 1. 788 logging.exception(e) 789 exit_code = 1 790 finally: 791 if pid_file_manager: 792 pid_file_manager.close_file(exit_code) 793 # Record the autoserv duration time. Must be called 794 # just before the system exits to ensure accuracy. 795 duration_secs = (datetime.datetime.now() - start_time).total_seconds() 796 record_autoserv(parser.options, duration_secs) 797 sys.exit(exit_code) 798 799 800if __name__ == '__main__': 801 main() 802