test_that.py revision 1a6ce6e485e44536f705dfc9a0bd66c3b0b294eb
1#!/usr/bin/python 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import errno 8import os 9import pipes 10import re 11import shutil 12import signal 13import stat 14import subprocess 15import sys 16import tempfile 17import threading 18 19import logging 20# Turn the logging level to INFO before importing other autotest 21# code, to avoid having failed import logging messages confuse the 22# test_that user. 23logging.basicConfig(level=logging.INFO) 24 25import common 26from autotest_lib.client.common_lib.cros import dev_server, retry 27from autotest_lib.client.common_lib import error, logging_manager 28from autotest_lib.server.cros.dynamic_suite import suite, constants 29from autotest_lib.server.cros import provision 30from autotest_lib.server.hosts import factory 31from autotest_lib.server import autoserv_utils 32from autotest_lib.server import server_logging_config 33from autotest_lib.server import utils 34 35 36try: 37 from chromite.lib import cros_build_lib 38except ImportError: 39 print 'Unable to import chromite.' 40 print 'This script must be either:' 41 print ' - Be run in the chroot.' 42 print ' - (not yet supported) be run after running ' 43 print ' ../utils/build_externals.py' 44 45_autoserv_proc = None 46_sigint_handler_lock = threading.Lock() 47 48_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 49_NO_BOARD = 'ad_hoc_board' 50_NO_BUILD = 'ad_hoc_build' 51_SUITE_REGEX = r'suite:(.*)' 52 53_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' 54_TEST_KEY_FILENAME = 'testing_rsa' 55_TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 56 'ssh_keys/%s' % _TEST_KEY_FILENAME) 57 58_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' 59 60_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 61 62 63class TestThatRunError(Exception): 64 """Raised if test_that encounters something unexpected while running.""" 65 66 67class TestThatProvisioningError(Exception): 68 """Raised when it fails to provision the DUT to the requested build.""" 69 70 71def schedule_local_suite(autotest_path, suite_predicate, afe, remote, 72 build=_NO_BUILD, board=_NO_BOARD, 73 results_directory=None, no_experimental=False, 74 ignore_deps=True): 75 """Schedule a suite against a mock afe object, for a local suite run. 76 77 Satisfaction of dependencies is enforced by Suite.schedule() if 78 ignore_deps is False. Note that this method assumes only one host, 79 i.e. |remote|, was added to afe. Suite.schedule() will not 80 schedule a job if none of the hosts in the afe (in our case, 81 just one host |remote|) has a label that matches a requested 82 test dependency. 83 84 @param autotest_path: Absolute path to autotest (in sysroot or 85 custom autotest directory set by --autotest_dir). 86 @param suite_predicate: callable that takes ControlData objects, and 87 returns True on those that should be in suite 88 @param afe: afe object to schedule against (typically a directAFE) 89 @param remote: String representing the IP of the remote host. 90 @param build: Build to schedule suite for. 91 @param board: Board to schedule suite for. 92 @param results_directory: Absolute path of directory to store results in. 93 (results will be stored in subdirectory of this). 94 @param no_experimental: Skip experimental tests when scheduling a suite. 95 @param ignore_deps: If True, test dependencies will be ignored. 96 97 @returns: The number of tests scheduled. 98 99 """ 100 fs_getter = suite.Suite.create_fs_getter(autotest_path) 101 devserver = dev_server.ImageServer('') 102 my_suite = suite.Suite.create_from_predicates([suite_predicate], 103 build, constants.BOARD_PREFIX + board, 104 devserver, fs_getter, afe=afe, 105 ignore_deps=ignore_deps, 106 results_dir=results_directory, forgiving_parser=False) 107 if len(my_suite.tests) == 0: 108 raise ValueError('Suite contained no tests.') 109 110 if not ignore_deps: 111 # Log tests whose dependencies can't be satisfied. 112 labels = [label.name for label in 113 afe.get_labels(host__hostname=remote)] 114 for test in my_suite.tests: 115 if test.experimental and no_experimental: 116 continue 117 unsatisfiable_deps = set(test.dependencies).difference(labels) 118 if unsatisfiable_deps: 119 logging.warn('%s will be skipped, unsatisfiable ' 120 'test dependencies: %s', test.name, 121 unsatisfiable_deps) 122 # Schedule tests, discard record calls. 123 return my_suite.schedule(lambda x: None, 124 add_experimental=not no_experimental) 125 126 127def _run_autoserv(command, pretend=False): 128 """Run autoserv command. 129 130 Run the autoserv command and wait on it. Log the stdout. 131 Ensure that SIGINT signals are passed along to autoserv. 132 133 @param command: the autoserv command to run. 134 @returns: exit code of the command. 135 136 """ 137 if not pretend: 138 logging.debug('Running autoserv command: %s', command) 139 global _autoserv_proc 140 _autoserv_proc = subprocess.Popen(command, 141 stdout=subprocess.PIPE, 142 stderr=subprocess.STDOUT) 143 # This incantation forces unbuffered reading from stdout, 144 # so that autoserv output can be displayed to the user 145 # immediately. 146 for message in iter(_autoserv_proc.stdout.readline, b''): 147 logging.info('autoserv| %s', message.strip()) 148 149 _autoserv_proc.wait() 150 returncode = _autoserv_proc.returncode 151 _autoserv_proc = None 152 else: 153 logging.info('Pretend mode. Would run autoserv command: %s', 154 command) 155 returncode = 0 156 return returncode 157 158 159def run_provisioning_job(provision_label, host, autotest_path, 160 results_directory, fast_mode, 161 ssh_verbosity=0, ssh_options=None, 162 pretend=False, autoserv_verbose=False): 163 """Shell out to autoserv to run provisioning job. 164 165 @param provision_label: Label to provision the machine to. 166 @param host: Hostname of DUT. 167 @param autotest_path: Absolute path of autotest directory. 168 @param results_directory: Absolute path of directory to store results in. 169 (results will be stored in subdirectory of this). 170 @param fast_mode: bool to use fast mode (disables slow autotest features). 171 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 172 @param ssh_options: Additional ssh options to be passed to autoserv_utils 173 @param pretend: If True, will print out autoserv commands rather than 174 running them. 175 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 176 177 @returns: Absolute path of directory where results were stored. 178 179 """ 180 # TODO(fdeng): When running against a local DUT, autoserv 181 # is still hitting the AFE in the lab. 182 # provision_AutoUpdate checks the current build of DUT by 183 # retrieving build info from AFE. crosbug.com/295178 184 results_directory = os.path.join(results_directory, 'results-provision') 185 provision_arg = '='.join(['--provision', provision_label]) 186 command = autoserv_utils.autoserv_run_job_command( 187 os.path.join(autotest_path, 'server'), 188 machines=host, job=None, verbose=autoserv_verbose, 189 results_directory=results_directory, 190 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 191 ssh_options=ssh_options, extra_args=[provision_arg], 192 no_console_prefix=True) 193 if _run_autoserv(command, pretend) != 0: 194 raise TestThatProvisioningError('Command returns non-zero code: %s ' % 195 command) 196 return results_directory 197 198 199def run_job(job, host, autotest_path, results_directory, fast_mode, 200 id_digits=1, ssh_verbosity=0, ssh_options=None, 201 args=None, pretend=False, 202 autoserv_verbose=False): 203 """ 204 Shell out to autoserv to run an individual test job. 205 206 @param job: A Job object containing the control file contents and other 207 relevent metadata for this test. 208 @param host: Hostname of DUT to run test against. 209 @param autotest_path: Absolute path of autotest directory. 210 @param results_directory: Absolute path of directory to store results in. 211 (results will be stored in subdirectory of this). 212 @param fast_mode: bool to use fast mode (disables slow autotest features). 213 @param id_digits: The minimum number of digits that job ids should be 214 0-padded to when formatting as a string for results 215 directory. 216 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 217 @param ssh_options: Additional ssh options to be passed to autoserv_utils 218 @param args: String that should be passed as args parameter to autoserv, 219 and then ultimitely to test itself. 220 @param pretend: If True, will print out autoserv commands rather than 221 running them. 222 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 223 @returns: Absolute path of directory where results were stored. 224 """ 225 with tempfile.NamedTemporaryFile() as temp_file: 226 temp_file.write(job.control_file) 227 temp_file.flush() 228 name_tail = job.name.split('/')[-1] 229 results_directory = os.path.join(results_directory, 230 'results-%0*d-%s' % (id_digits, job.id, 231 name_tail)) 232 # Drop experimental keyval in the keval file in the job result folder. 233 os.makedirs(results_directory) 234 utils.write_keyval(results_directory, 235 {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ 236 constants.JOB_EXPERIMENTAL_KEY]}) 237 extra_args = [temp_file.name] 238 if args: 239 extra_args.extend(['--args', args]) 240 241 command = autoserv_utils.autoserv_run_job_command( 242 os.path.join(autotest_path, 'server'), 243 machines=host, job=job, verbose=autoserv_verbose, 244 results_directory=results_directory, 245 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 246 ssh_options=ssh_options, 247 extra_args=extra_args, 248 no_console_prefix=True) 249 250 _run_autoserv(command, pretend) 251 return results_directory 252 253 254def setup_local_afe(): 255 """ 256 Setup a local afe database and return a direct_afe object to access it. 257 258 @returns: A autotest_lib.frontend.afe.direct_afe instance. 259 """ 260 # This import statement is delayed until now rather than running at 261 # module load time, because it kicks off a local sqlite :memory: backed 262 # database, and we don't need that unless we are doing a local run. 263 from autotest_lib.frontend import setup_django_lite_environment 264 from autotest_lib.frontend.afe import direct_afe 265 return direct_afe.directAFE() 266 267 268def get_predicate_for_test_arg(test): 269 """ 270 Gets a suite predicte function for a given command-line argument. 271 272 @param test: String. An individual TEST command line argument, e.g. 273 'login_CryptohomeMounted' or 'suite:smoke' 274 @returns: A (predicate, string) tuple with the necessary suite 275 predicate, and a description string of the suite that 276 this predicate will produce. 277 """ 278 suitematch = re.match(_SUITE_REGEX, test) 279 name_pattern_match = re.match(r'e:(.*)', test) 280 file_pattern_match = re.match(r'f:(.*)', test) 281 if suitematch: 282 suitename = suitematch.group(1) 283 return (suite.Suite.name_in_tag_predicate(suitename), 284 'suite named %s' % suitename) 285 if name_pattern_match: 286 pattern = '^%s$' % name_pattern_match.group(1) 287 return (suite.Suite.test_name_matches_pattern_predicate(pattern), 288 'suite to match name pattern %s' % pattern) 289 if file_pattern_match: 290 pattern = '^%s$' % file_pattern_match.group(1) 291 return (suite.Suite.test_file_matches_pattern_predicate(pattern), 292 'suite to match file name pattern %s' % pattern) 293 return (suite.Suite.test_name_equals_predicate(test), 294 'job named %s' % test) 295 296 297def _add_ssh_identity(temp_directory): 298 """Add an ssh identity to the agent. 299 300 @param temp_directory: A directory to copy the testing_rsa into. 301 """ 302 # Add the testing key to the current ssh agent. 303 if os.environ.has_key('SSH_AGENT_PID'): 304 # Copy the testing key to the temp directory and make it NOT 305 # world-readable. Otherwise, ssh-add complains. 306 shutil.copy(_TEST_KEY_PATH, temp_directory) 307 key_copy_path = os.path.join(temp_directory, _TEST_KEY_FILENAME) 308 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 309 p = subprocess.Popen(['ssh-add', key_copy_path], 310 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 311 p_out, _ = p.communicate() 312 for line in p_out.splitlines(): 313 logging.info(line) 314 else: 315 logging.warning('There appears to be no running ssh-agent. Attempting ' 316 'to continue without running ssh-add, but ssh commands ' 317 'may fail.') 318 319 320def _get_board_from_host(remote): 321 """Get the board of the remote host. 322 323 @param remote: string representing the IP of the remote host. 324 325 @return: A string representing the board of the remote host. 326 """ 327 logging.info('Board unspecified, attempting to determine board from host.') 328 host = factory.create_host(remote) 329 try: 330 board = host.get_board().replace(constants.BOARD_PREFIX, '') 331 except error.AutoservRunError: 332 raise TestThatRunError('Cannot determine board, please specify ' 333 'a --board option.') 334 logging.info('Detected host board: %s', board) 335 return board 336 337 338def _auto_detect_labels(afe, remote): 339 """Automatically detect host labels and add them to the host in afe. 340 341 Note that the label of board will not be auto-detected. 342 This method assumes the host |remote| has already been added to afe. 343 344 @param afe: A direct_afe object used to interact with local afe database. 345 @param remote: The hostname of the remote device. 346 347 """ 348 cros_host = factory.create_host(remote) 349 labels_to_create = [label for label in cros_host.get_labels() 350 if not label.startswith(constants.BOARD_PREFIX)] 351 labels_to_add_to_afe_host = [] 352 for label in labels_to_create: 353 new_label = afe.create_label(label) 354 labels_to_add_to_afe_host.append(new_label.name) 355 hosts = afe.get_hosts(hostname=remote) 356 if not hosts: 357 raise TestThatRunError('Unexpected error: %s has not ' 358 'been added to afe.' % remote) 359 afe_host = hosts[0] 360 afe_host.add_labels(labels_to_add_to_afe_host) 361 362 363def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 364 build=_NO_BUILD, board=_NO_BOARD, args=None, 365 pretend=False, no_experimental=False, 366 ignore_deps=True, 367 results_directory=None, ssh_verbosity=0, 368 ssh_options=None, 369 autoserv_verbose=False): 370 """Perform local run of tests. 371 372 This method enforces satisfaction of test dependencies for tests that are 373 run as a part of a suite. 374 375 @param afe: A direct_afe object used to interact with local afe database. 376 @param autotest_path: Absolute path of autotest installed in sysroot or 377 custom autotest path set by --autotest_dir. 378 @param tests: List of strings naming tests and suites to run. Suite strings 379 should be formed like "suite:smoke". 380 @param remote: Remote hostname. 381 @param fast_mode: bool to use fast mode (disables slow autotest features). 382 @param build: String specifying build for local run. 383 @param board: String specifyinb board for local run. 384 @param args: String that should be passed as args parameter to autoserv, 385 and then ultimitely to test itself. 386 @param pretend: If True, will print out autoserv commands rather than 387 running them. 388 @param no_experimental: Skip experimental tests when scheduling a suite. 389 @param ignore_deps: If True, test dependencies will be ignored. 390 @param results_directory: Directory to store results in. Defaults to None, 391 in which case results will be stored in a new 392 subdirectory of /tmp 393 @param ssh_verbosity: SSH verbosity level, passed through to 394 autoserv_utils. 395 @param ssh_options: Additional ssh options to be passed to autoserv_utils 396 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 397 """ 398 399 # Create host in afe, add board and build labels. 400 cros_version_label = provision.cros_version_to_label(build) 401 build_label = afe.create_label(cros_version_label) 402 board_label = afe.create_label(constants.BOARD_PREFIX + board) 403 new_host = afe.create_host(remote) 404 new_host.add_labels([build_label.name, board_label.name]) 405 if not ignore_deps: 406 logging.info('Auto-detecting labels for %s', remote) 407 _auto_detect_labels(afe, remote) 408 # Provision the host to |build|. 409 if build != _NO_BUILD: 410 logging.info('Provisioning %s...', cros_version_label) 411 try: 412 run_provisioning_job(cros_version_label, remote, autotest_path, 413 results_directory, fast_mode, 414 ssh_verbosity, ssh_options, 415 pretend, autoserv_verbose) 416 except TestThatProvisioningError as e: 417 logging.error('Provisioning %s to %s failed, tests are aborted, ' 418 'failure reason: %s', 419 remote, cros_version_label, e) 420 return 421 422 # Schedule tests / suites in local afe 423 for test in tests: 424 (predicate, description) = get_predicate_for_test_arg(test) 425 logging.info('Scheduling %s...', description) 426 ntests = schedule_local_suite(autotest_path, predicate, afe, 427 remote=remote, 428 build=build, board=board, 429 results_directory=results_directory, 430 no_experimental=no_experimental, 431 ignore_deps=ignore_deps) 432 logging.info('... scheduled %s job(s).', ntests) 433 434 if not afe.get_jobs(): 435 logging.info('No jobs scheduled. End of local run.') 436 return 437 438 last_job_id = afe.get_jobs()[-1].id 439 job_id_digits = len(str(last_job_id)) 440 for job in afe.get_jobs(): 441 run_job(job, remote, autotest_path, results_directory, fast_mode, 442 job_id_digits, ssh_verbosity, ssh_options, args, pretend, 443 autoserv_verbose) 444 445 446def validate_arguments(arguments): 447 """ 448 Validates parsed arguments. 449 450 @param arguments: arguments object, as parsed by ParseArguments 451 @raises: ValueError if arguments were invalid. 452 """ 453 if arguments.remote == ':lab:': 454 if arguments.args: 455 raise ValueError('--args flag not supported when running against ' 456 ':lab:') 457 if arguments.pretend: 458 raise ValueError('--pretend flag not supported when running ' 459 'against :lab:') 460 if arguments.ssh_verbosity: 461 raise ValueError('--ssh_verbosity flag not supported when running ' 462 'against :lab:') 463 464 465def parse_arguments(argv): 466 """ 467 Parse command line arguments 468 469 @param argv: argument list to parse 470 @returns: parsed arguments. 471 @raises SystemExit if arguments are malformed, or required arguments 472 are not present. 473 """ 474 parser = argparse.ArgumentParser(description='Run remote tests.') 475 476 parser.add_argument('remote', metavar='REMOTE', 477 help='hostname[:port] for remote device. Specify ' 478 ':lab: to run in test lab, or :vm:PORT_NUMBER to ' 479 'run in vm.') 480 parser.add_argument('tests', nargs='+', metavar='TEST', 481 help='Run given test(s). Use suite:SUITE to specify ' 482 'test suite. Use e:[NAME_PATTERN] to specify a ' 483 'NAME-matching regular expression. Use ' 484 'f:[FILE_PATTERN] to specify a filename matching ' 485 'regular expression. Specified regular ' 486 'expressiosn will be implicitly wrapped in ' 487 '^ and $.') 488 default_board = cros_build_lib.GetDefaultBoard() 489 parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, 490 action='store', 491 help='Board for which the test will run. Default: %s' % 492 (default_board or 'Not configured')) 493 parser.add_argument('-i', '--build', metavar='BUILD', default=_NO_BUILD, 494 help='Build to test. Device will be reimaged if ' 495 'necessary. Omit flag to skip reimage and test ' 496 'against already installed DUT image.') 497 parser.add_argument('--fast', action='store_true', dest='fast_mode', 498 default=False, 499 help='Enable fast mode. This will cause test_that to ' 500 'skip time consuming steps like sysinfo and ' 501 'collecting crash information.') 502 parser.add_argument('--args', metavar='ARGS', 503 help='Argument string to pass through to test. Only ' 504 'supported for runs against a local DUT.') 505 parser.add_argument('--autotest_dir', metavar='AUTOTEST_DIR', 506 help='Use AUTOTEST_DIR instead of normal board sysroot ' 507 'copy of autotest, and skip the quickmerge step.') 508 parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, 509 help='Instead of storing results in a new subdirectory' 510 ' of /tmp , store results in RESULTS_DIR. If ' 511 'RESULTS_DIR already exists, it will be deleted.') 512 parser.add_argument('--pretend', action='store_true', default=False, 513 help='Print autoserv commands that would be run, ' 514 'rather than running them.') 515 parser.add_argument('--no-quickmerge', action='store_true', default=False, 516 dest='no_quickmerge', 517 help='Skip the quickmerge step and use the sysroot ' 518 'as it currently is. May result in un-merged ' 519 'source tree changes not being reflected in run.' 520 'If using --autotest_dir, this flag is ' 521 'automatically applied.') 522 parser.add_argument('--no-experimental', action='store_true', 523 default=False, dest='no_experimental', 524 help='When scheduling a suite, skip any tests marked ' 525 'as experimental. Applies only to tests scheduled' 526 ' via suite:[SUITE].') 527 parser.add_argument('--whitelist-chrome-crashes', action='store_true', 528 default=False, dest='whitelist_chrome_crashes', 529 help='Ignore chrome crashes when producing test ' 530 'report. This flag gets passed along to the ' 531 'report generation tool.') 532 parser.add_argument('--enforce-deps', action='store_true', 533 default=False, dest='enforce_deps', 534 help='Skip tests whose DEPENDENCIES can not ' 535 'be satisfied.') 536 parser.add_argument('--ssh_verbosity', action='store', type=int, 537 choices=[0, 1, 2, 3], default=0, 538 help='Verbosity level for ssh, between 0 and 3 ' 539 'inclusive.') 540 parser.add_argument('--ssh_options', action='store', default=None, 541 help='A string giving additional options to be ' 542 'added to ssh commands.') 543 parser.add_argument('--debug', action='store_true', 544 help='Include DEBUG level messages in stdout. Note: ' 545 'these messages will be included in output log ' 546 'file regardless. In addition, turn on autoserv ' 547 'verbosity.') 548 return parser.parse_args(argv) 549 550 551def sigint_handler(signum, stack_frame): 552 #pylint: disable-msg=C0111 553 """Handle SIGINT or SIGTERM to a local test_that run. 554 555 This handler sends a SIGINT to the running autoserv process, 556 if one is running, giving it up to 5 seconds to clean up and exit. After 557 the timeout elapses, autoserv is killed. In either case, after autoserv 558 exits then this process exits with status 1. 559 """ 560 # If multiple signals arrive before handler is unset, ignore duplicates 561 if not _sigint_handler_lock.acquire(False): 562 return 563 try: 564 # Ignore future signals by unsetting handler. 565 signal.signal(signal.SIGINT, signal.SIG_IGN) 566 signal.signal(signal.SIGTERM, signal.SIG_IGN) 567 568 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 569 if _autoserv_proc: 570 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 571 'to %s seconds for cleanup.', 572 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 573 _autoserv_proc.send_signal(signal.SIGINT) 574 timed_out, _ = retry.timeout(_autoserv_proc.wait, 575 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 576 if timed_out: 577 _autoserv_proc.kill() 578 logging.warning('Timed out waiting for autoserv to handle ' 579 'SIGINT. Killed autoserv.') 580 finally: 581 _sigint_handler_lock.release() # this is not really necessary? 582 sys.exit(1) 583 584 585def _create_results_directory(results_directory=None): 586 """Create a results directory. 587 588 If no directory is specified this method will create and return a 589 temp directory to hold results. If a directory name is specified this 590 method will create a directory at the given path, provided it doesn't 591 already exist. 592 593 @param results_directory: The path to the results_directory to create. 594 595 @return results_directory: A path to the results_directory, ready for use. 596 """ 597 if results_directory is None: 598 # Create a results_directory as subdir of /tmp 599 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 600 else: 601 # Delete results_directory if it already exists. 602 try: 603 shutil.rmtree(results_directory) 604 except OSError as e: 605 if e.errno != errno.ENOENT: 606 raise 607 608 # Create results_directory if it does not exist 609 try: 610 os.makedirs(results_directory) 611 except OSError as e: 612 if e.errno != errno.EEXIST: 613 raise 614 return results_directory 615 616 617def _perform_bootstrap_into_autotest_root(arguments, autotest_path, argv, 618 legacy_path=False): 619 """ 620 Perfoms a bootstrap to run test_that from the |autotest_path|. 621 622 This function is to be called from test_that's main() script, when 623 test_that is executed from the source tree location. It runs 624 autotest_quickmerge to update the sysroot unless arguments.no_quickmerge 625 is set. It then executes and waits on the version of test_that.py 626 in |autotest_path|. 627 628 @param arguments: A parsed arguments object, as returned from 629 parse_arguments(...). 630 @param autotest_path: Full absolute path to the autotest root directory. 631 @param argv: The arguments list, as passed to main(...) 632 @param legacy_path: Flag for backwards compatibility with builds 633 that have autotest in old usr/local/autotest location 634 635 @returns: The return code of the test_that script that was executed in 636 |autotest_path|. 637 """ 638 logging_manager.configure_logging( 639 server_logging_config.ServerLoggingConfig(), 640 use_console=True, 641 verbose=arguments.debug) 642 if arguments.no_quickmerge: 643 logging.info('Skipping quickmerge step.') 644 else: 645 logging.info('Running autotest_quickmerge step.') 646 command = [_QUICKMERGE_SCRIPTNAME, '--board='+arguments.board] 647 if legacy_path: 648 command.append('--legacy_path') 649 s = subprocess.Popen(command, 650 stdout=subprocess.PIPE, 651 stderr=subprocess.STDOUT) 652 for message in iter(s.stdout.readline, b''): 653 logging.info('quickmerge| %s', message.strip()) 654 s.wait() 655 656 logging.info('Re-running test_that script in %s copy of autotest.', 657 autotest_path) 658 script_command = os.path.join(autotest_path, 'site_utils', 659 os.path.basename(__file__)) 660 if not os.path.exists(script_command): 661 raise TestThatRunError('Unable to bootstrap to autotest root, ' 662 '%s not found.' % script_command) 663 proc = None 664 def resend_sig(signum, stack_frame): 665 #pylint: disable-msg=C0111 666 if proc: 667 proc.send_signal(signum) 668 signal.signal(signal.SIGINT, resend_sig) 669 signal.signal(signal.SIGTERM, resend_sig) 670 671 proc = subprocess.Popen([script_command] + argv) 672 673 return proc.wait() 674 675 676def _perform_run_from_autotest_root(arguments, autotest_path, argv): 677 """ 678 Perform a test_that run, from the |autotest_path|. 679 680 This function is to be called from test_that's main() script, when 681 test_that is executed from the |autotest_path|. It handles all stages 682 of a test_that run that come after the bootstrap into |autotest_path|. 683 684 @param arguments: A parsed arguments object, as returned from 685 parse_arguments(...). 686 @param autotest_path: Full absolute path to the autotest root directory. 687 @param argv: The arguments list, as passed to main(...) 688 689 @returns: A return code that test_that should exit with. 690 """ 691 results_directory = arguments.results_dir 692 if results_directory is None or not os.path.exists(results_directory): 693 raise ValueError('Expected valid results directory, got %s' % 694 results_directory) 695 696 logging_manager.configure_logging( 697 server_logging_config.ServerLoggingConfig(), 698 results_dir=results_directory, 699 use_console=True, 700 verbose=arguments.debug, 701 debug_log_name='test_that') 702 logging.info('Began logging to %s', results_directory) 703 704 logging.debug('test_that command line was: %s', argv) 705 706 signal.signal(signal.SIGINT, sigint_handler) 707 signal.signal(signal.SIGTERM, sigint_handler) 708 709 afe = setup_local_afe() 710 perform_local_run(afe, autotest_path, arguments.tests, 711 arguments.remote, arguments.fast_mode, 712 arguments.build, arguments.board, 713 args=arguments.args, 714 pretend=arguments.pretend, 715 no_experimental=arguments.no_experimental, 716 ignore_deps=not arguments.enforce_deps, 717 results_directory=results_directory, 718 ssh_verbosity=arguments.ssh_verbosity, 719 ssh_options=arguments.ssh_options, 720 autoserv_verbose=arguments.debug) 721 if arguments.pretend: 722 logging.info('Finished pretend run. Exiting.') 723 return 0 724 725 test_report_command = [_TEST_REPORT_SCRIPTNAME] 726 # Experimental test results do not influence the exit code. 727 test_report_command.append('--ignore_experimental_tests') 728 if arguments.whitelist_chrome_crashes: 729 test_report_command.append('--whitelist_chrome_crashes') 730 test_report_command.append(results_directory) 731 final_result = subprocess.call(test_report_command) 732 with open(os.path.join(results_directory, 'test_report.log'), 733 'w') as report_log: 734 subprocess.call(test_report_command, stdout=report_log) 735 logging.info('Finished running tests. Results can be found in %s', 736 results_directory) 737 try: 738 os.unlink(_LATEST_RESULTS_DIRECTORY) 739 except OSError: 740 pass 741 os.symlink(results_directory, _LATEST_RESULTS_DIRECTORY) 742 return final_result 743 744 745def _main_for_local_run(argv, arguments): 746 """ 747 Effective entry point for local test_that runs. 748 749 @param argv: Script command line arguments. 750 @param arguments: Parsed command line arguments. 751 """ 752 if not cros_build_lib.IsInsideChroot(): 753 print >> sys.stderr, 'For local runs, script must be run inside chroot.' 754 return 1 755 756 results_directory = _create_results_directory(arguments.results_dir) 757 _add_ssh_identity(results_directory) 758 arguments.results_dir = results_directory 759 legacy_path = False 760 761 # If the board has not been specified through --board, and is not set in the 762 # default_board file, determine the board by ssh-ing into the host. Also 763 # prepend it to argv so we can re-use it when we run test_that from the 764 # sysroot. 765 if arguments.board is None: 766 arguments.board = _get_board_from_host(arguments.remote) 767 argv = ['--board', arguments.board] + argv 768 769 if arguments.autotest_dir: 770 autotest_path = arguments.autotest_dir 771 arguments.no_quickmerge = True 772 else: 773 sysroot_path = os.path.join('/build', arguments.board, '') 774 775 if not os.path.exists(sysroot_path): 776 print >> sys.stderr, ('%s does not exist. Have you run ' 777 'setup_board?' % sysroot_path) 778 return 1 779 780 # For backwards compatibility with builds that pre-date 781 # https://chromium-review.googlesource.com/#/c/62880/ 782 # This code can eventually be removed once those builds no longer need 783 # test_that support. 784 legacy_path = os.path.exists(os.path.join(sysroot_path, 'usr', 'local', 785 'autotest', 'site_utils')) 786 if legacy_path: 787 path_ending = 'usr/local/autotest' 788 else: 789 path_ending = 'usr/local/build/autotest' 790 autotest_path = os.path.join(sysroot_path, path_ending) 791 792 site_utils_path = os.path.join(autotest_path, 'site_utils') 793 794 if not os.path.exists(autotest_path): 795 print >> sys.stderr, ('%s does not exist. Have you run ' 796 'build_packages? Or if you are using ' 797 '--autotest-dir, make sure it points to' 798 'a valid autotest directory.' % autotest_path) 799 return 1 800 801 realpath = os.path.realpath(__file__) 802 803 # If we are not running the sysroot version of script, perform 804 # a quickmerge if necessary and then re-execute 805 # the sysroot version of script with the same arguments. 806 if os.path.dirname(realpath) != site_utils_path: 807 return _perform_bootstrap_into_autotest_root( 808 arguments, autotest_path, argv, legacy_path) 809 else: 810 return _perform_run_from_autotest_root( 811 arguments, autotest_path, argv) 812 813 814def _main_for_lab_run(argv, arguments): 815 """ 816 Effective entry point for lab test_that runs. 817 818 @param argv: Script command line arguments. 819 @param arguments: Parsed command line arguments. 820 """ 821 autotest_path = os.path.realpath(os.path.join(os.path.dirname(__file__), 822 '..')) 823 flattened_argv = ' '.join([pipes.quote(item) for item in argv]) 824 command = [os.path.join(autotest_path, 'site_utils', 825 'run_suite.py'), 826 '--board', arguments.board, 827 '--build', arguments.build, 828 '--suite_name', 'test_that_wrapper', 829 '--pool', 'try-bot', 830 '--suite_args', flattened_argv] 831 logging.info('About to start lab suite with command %s.', command) 832 return subprocess.call(command) 833 834 835def main(argv): 836 """ 837 Entry point for test_that script. 838 839 @param argv: arguments list 840 """ 841 arguments = parse_arguments(argv) 842 try: 843 validate_arguments(arguments) 844 except ValueError as err: 845 print >> sys.stderr, ('Invalid arguments. %s' % err.message) 846 return 1 847 848 if arguments.remote == ':lab:': 849 return _main_for_lab_run(argv, arguments) 850 else: 851 return _main_for_local_run(argv, arguments) 852 853 854if __name__ == '__main__': 855 sys.exit(main(sys.argv[1:])) 856