test_that.py revision 2252db34500a41ecbc8db0972ad91adce3030ab6
1#!/usr/bin/python 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import errno 8import os 9import pipes 10import re 11import shutil 12import signal 13import stat 14import subprocess 15import sys 16import tempfile 17import threading 18 19import logging 20# Turn the logging level to INFO before importing other autotest 21# code, to avoid having failed import logging messages confuse the 22# test_that user. 23logging.basicConfig(level=logging.INFO) 24 25import common 26from autotest_lib.client.common_lib.cros import dev_server, retry 27from autotest_lib.client.common_lib import error, logging_manager 28from autotest_lib.server.cros.dynamic_suite import suite, constants 29from autotest_lib.server.cros import provision 30from autotest_lib.server.hosts import factory 31from autotest_lib.server import autoserv_utils 32from autotest_lib.server import server_logging_config 33from autotest_lib.server import utils 34 35 36try: 37 from chromite.lib import cros_build_lib 38except ImportError: 39 print 'Unable to import chromite.' 40 print 'This script must be either:' 41 print ' - Be run in the chroot.' 42 print ' - (not yet supported) be run after running ' 43 print ' ../utils/build_externals.py' 44 45_autoserv_proc = None 46_sigint_handler_lock = threading.Lock() 47 48_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 49_NO_BOARD = 'ad_hoc_board' 50_NO_BUILD = 'ad_hoc_build' 51_SUITE_REGEX = r'suite:(.*)' 52 53_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' 54_TEST_KEY_FILENAME = 'testing_rsa' 55_TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 56 'ssh_keys/%s' % _TEST_KEY_FILENAME) 57 58_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' 59 60_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 61 62 63class TestThatRunError(Exception): 64 """Raised if test_that encounters something unexpected while running.""" 65 66 67class TestThatProvisioningError(Exception): 68 """Raised when it fails to provision the DUT to the requested build.""" 69 70 71def fetch_local_suite(autotest_path, suite_predicate, afe, test_arg, remote, 72 build=_NO_BUILD, board=_NO_BOARD, 73 results_directory=None, no_experimental=False, 74 ignore_deps=True): 75 """Create a suite from the given suite predicate. 76 77 Satisfaction of dependencies is enforced by Suite.schedule() if 78 ignore_deps is False. Note that this method assumes only one host, 79 i.e. |remote|, was added to afe. Suite.schedule() will not 80 schedule a job if none of the hosts in the afe (in our case, 81 just one host |remote|) has a label that matches a requested 82 test dependency. 83 84 @param autotest_path: Absolute path to autotest (in sysroot or 85 custom autotest directory set by --autotest_dir). 86 @param suite_predicate: callable that takes ControlData objects, and 87 returns True on those that should be in suite 88 @param afe: afe object to schedule against (typically a directAFE) 89 @param test_arg: String. An individual TEST command line argument, e.g. 90 'login_CryptohomeMounted' or 'suite:smoke'. 91 @param remote: String representing the IP of the remote host. 92 @param build: Build to schedule suite for. 93 @param board: Board to schedule suite for. 94 @param results_directory: Absolute path of directory to store results in. 95 (results will be stored in subdirectory of this). 96 @param no_experimental: Skip experimental tests when scheduling a suite. 97 @param ignore_deps: If True, test dependencies will be ignored. 98 99 @returns: A suite.Suite object. 100 101 """ 102 fs_getter = suite.Suite.create_fs_getter(autotest_path) 103 devserver = dev_server.ImageServer('') 104 my_suite = suite.Suite.create_from_predicates([suite_predicate], 105 build, constants.BOARD_PREFIX + board, 106 devserver, fs_getter, afe=afe, 107 ignore_deps=ignore_deps, 108 results_dir=results_directory, forgiving_parser=False) 109 if len(my_suite.tests) == 0: 110 (similarity_predicate, similarity_description) = ( 111 get_predicate_for_possible_test_arg(test_arg)) 112 logging.error('No test found, searching for possible tests with %s', 113 similarity_description) 114 possible_tests = suite.Suite.find_possible_tests(fs_getter, 115 similarity_predicate) 116 raise ValueError('Found no tests. Check your suite name, test name, ' 117 'or test matching wildcard.\nDid you mean any of ' 118 'following tests?\n %s' % '\n '.join(possible_tests)) 119 120 if not ignore_deps: 121 # Log tests whose dependencies can't be satisfied. 122 labels = [label.name for label in 123 afe.get_labels(host__hostname=remote)] 124 for test in my_suite.tests: 125 if test.experimental and no_experimental: 126 continue 127 unsatisfiable_deps = set(test.dependencies).difference(labels) 128 if unsatisfiable_deps: 129 logging.warning('%s will be skipped, unsatisfiable ' 130 'test dependencies: %s', test.name, 131 unsatisfiable_deps) 132 return my_suite 133 134 135def _run_autoserv(command, pretend=False): 136 """Run autoserv command. 137 138 Run the autoserv command and wait on it. Log the stdout. 139 Ensure that SIGINT signals are passed along to autoserv. 140 141 @param command: the autoserv command to run. 142 @returns: exit code of the command. 143 144 """ 145 if not pretend: 146 logging.debug('Running autoserv command: %s', command) 147 global _autoserv_proc 148 _autoserv_proc = subprocess.Popen(command, 149 stdout=subprocess.PIPE, 150 stderr=subprocess.STDOUT) 151 # This incantation forces unbuffered reading from stdout, 152 # so that autoserv output can be displayed to the user 153 # immediately. 154 for message in iter(_autoserv_proc.stdout.readline, b''): 155 logging.info('autoserv| %s', message.strip()) 156 157 _autoserv_proc.wait() 158 returncode = _autoserv_proc.returncode 159 _autoserv_proc = None 160 else: 161 logging.info('Pretend mode. Would run autoserv command: %s', 162 command) 163 returncode = 0 164 return returncode 165 166 167def run_provisioning_job(provision_label, host, autotest_path, 168 results_directory, fast_mode, 169 ssh_verbosity=0, ssh_options=None, 170 pretend=False, autoserv_verbose=False): 171 """Shell out to autoserv to run provisioning job. 172 173 @param provision_label: Label to provision the machine to. 174 @param host: Hostname of DUT. 175 @param autotest_path: Absolute path of autotest directory. 176 @param results_directory: Absolute path of directory to store results in. 177 (results will be stored in subdirectory of this). 178 @param fast_mode: bool to use fast mode (disables slow autotest features). 179 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 180 @param ssh_options: Additional ssh options to be passed to autoserv_utils 181 @param pretend: If True, will print out autoserv commands rather than 182 running them. 183 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 184 185 @returns: Absolute path of directory where results were stored. 186 187 """ 188 # TODO(fdeng): When running against a local DUT, autoserv 189 # is still hitting the AFE in the lab. 190 # provision_AutoUpdate checks the current build of DUT by 191 # retrieving build info from AFE. crosbug.com/295178 192 results_directory = os.path.join(results_directory, 'results-provision') 193 provision_arg = '='.join(['--provision', provision_label]) 194 command = autoserv_utils.autoserv_run_job_command( 195 os.path.join(autotest_path, 'server'), 196 machines=host, job=None, verbose=autoserv_verbose, 197 results_directory=results_directory, 198 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 199 ssh_options=ssh_options, extra_args=[provision_arg], 200 no_console_prefix=True) 201 if _run_autoserv(command, pretend) != 0: 202 raise TestThatProvisioningError('Command returns non-zero code: %s ' % 203 command) 204 return results_directory 205 206 207def run_job(job, host, autotest_path, results_directory, fast_mode, 208 id_digits=1, ssh_verbosity=0, ssh_options=None, 209 args=None, pretend=False, 210 autoserv_verbose=False): 211 """ 212 Shell out to autoserv to run an individual test job. 213 214 @param job: A Job object containing the control file contents and other 215 relevent metadata for this test. 216 @param host: Hostname of DUT to run test against. 217 @param autotest_path: Absolute path of autotest directory. 218 @param results_directory: Absolute path of directory to store results in. 219 (results will be stored in subdirectory of this). 220 @param fast_mode: bool to use fast mode (disables slow autotest features). 221 @param id_digits: The minimum number of digits that job ids should be 222 0-padded to when formatting as a string for results 223 directory. 224 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 225 @param ssh_options: Additional ssh options to be passed to autoserv_utils 226 @param args: String that should be passed as args parameter to autoserv, 227 and then ultimitely to test itself. 228 @param pretend: If True, will print out autoserv commands rather than 229 running them. 230 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 231 @returns: Absolute path of directory where results were stored. 232 """ 233 with tempfile.NamedTemporaryFile() as temp_file: 234 temp_file.write(job.control_file) 235 temp_file.flush() 236 name_tail = job.name.split('/')[-1] 237 results_directory = os.path.join(results_directory, 238 'results-%0*d-%s' % (id_digits, job.id, 239 name_tail)) 240 # Drop experimental keyval in the keval file in the job result folder. 241 os.makedirs(results_directory) 242 utils.write_keyval(results_directory, 243 {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ 244 constants.JOB_EXPERIMENTAL_KEY]}) 245 extra_args = [temp_file.name] 246 if args: 247 extra_args.extend(['--args', args]) 248 249 command = autoserv_utils.autoserv_run_job_command( 250 os.path.join(autotest_path, 'server'), 251 machines=host, job=job, verbose=autoserv_verbose, 252 results_directory=results_directory, 253 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 254 ssh_options=ssh_options, 255 extra_args=extra_args, 256 no_console_prefix=True, 257 use_packaging=False) 258 259 _run_autoserv(command, pretend) 260 return results_directory 261 262 263def setup_local_afe(): 264 """ 265 Setup a local afe database and return a direct_afe object to access it. 266 267 @returns: A autotest_lib.frontend.afe.direct_afe instance. 268 """ 269 # This import statement is delayed until now rather than running at 270 # module load time, because it kicks off a local sqlite :memory: backed 271 # database, and we don't need that unless we are doing a local run. 272 from autotest_lib.frontend import setup_django_lite_environment 273 from autotest_lib.frontend.afe import direct_afe 274 return direct_afe.directAFE() 275 276 277def get_predicate_for_test_arg(test): 278 """ 279 Gets a suite predicte function for a given command-line argument. 280 281 @param test: String. An individual TEST command line argument, e.g. 282 'login_CryptohomeMounted' or 'suite:smoke' 283 @returns: A (predicate, string) tuple with the necessary suite 284 predicate, and a description string of the suite that 285 this predicate will produce. 286 """ 287 suitematch = re.match(_SUITE_REGEX, test) 288 name_pattern_match = re.match(r'e:(.*)', test) 289 file_pattern_match = re.match(r'f:(.*)', test) 290 if suitematch: 291 suitename = suitematch.group(1) 292 return (suite.Suite.name_in_tag_predicate(suitename), 293 'suite named %s' % suitename) 294 if name_pattern_match: 295 pattern = '^%s$' % name_pattern_match.group(1) 296 return (suite.Suite.test_name_matches_pattern_predicate(pattern), 297 'suite to match name pattern %s' % pattern) 298 if file_pattern_match: 299 pattern = '^%s$' % file_pattern_match.group(1) 300 return (suite.Suite.test_file_matches_pattern_predicate(pattern), 301 'suite to match file name pattern %s' % pattern) 302 return (suite.Suite.test_name_equals_predicate(test), 303 'job named %s' % test) 304 305 306def get_predicate_for_possible_test_arg(test): 307 """ 308 Gets a suite predicte function to calculate the similarity of given test 309 and possible tests. 310 311 @param test: String. An individual TEST command line argument, e.g. 312 'login_CryptohomeMounted' or 'suite:smoke' 313 @returns: A (predicate, string) tuple with the necessary suite 314 predicate, and a description string of the suite that 315 this predicate will produce. 316 """ 317 suitematch = re.match(_SUITE_REGEX, test) 318 name_pattern_match = re.match(r'e:(.*)', test) 319 file_pattern_match = re.match(r'f:(.*)', test) 320 if suitematch: 321 suitename = suitematch.group(1) 322 return (suite.Suite.name_in_tag_similarity_predicate(suitename), 323 'suite name similar to %s' % suitename) 324 if name_pattern_match: 325 pattern = '^%s$' % name_pattern_match.group(1) 326 return (suite.Suite.test_name_similarity_predicate(pattern), 327 'job name similar to %s' % pattern) 328 if file_pattern_match: 329 pattern = '^%s$' % file_pattern_match.group(1) 330 return (suite.Suite.test_file_similarity_predicate(pattern), 331 'suite to match file name similar to %s' % pattern) 332 return (suite.Suite.test_name_similarity_predicate(test), 333 'job name similar to %s' % test) 334 335 336def _add_ssh_identity(temp_directory): 337 """Add an ssh identity to the agent. 338 339 @param temp_directory: A directory to copy the testing_rsa into. 340 """ 341 # Add the testing key to the current ssh agent. 342 if os.environ.has_key('SSH_AGENT_PID'): 343 # Copy the testing key to the temp directory and make it NOT 344 # world-readable. Otherwise, ssh-add complains. 345 shutil.copy(_TEST_KEY_PATH, temp_directory) 346 key_copy_path = os.path.join(temp_directory, _TEST_KEY_FILENAME) 347 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 348 p = subprocess.Popen(['ssh-add', key_copy_path], 349 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 350 p_out, _ = p.communicate() 351 for line in p_out.splitlines(): 352 logging.info(line) 353 else: 354 logging.warning('There appears to be no running ssh-agent. Attempting ' 355 'to continue without running ssh-add, but ssh commands ' 356 'may fail.') 357 358 359def _get_board_from_host(remote): 360 """Get the board of the remote host. 361 362 @param remote: string representing the IP of the remote host. 363 364 @return: A string representing the board of the remote host. 365 """ 366 logging.info('Board unspecified, attempting to determine board from host.') 367 host = factory.create_host(remote) 368 try: 369 board = host.get_board().replace(constants.BOARD_PREFIX, '') 370 except error.AutoservRunError: 371 raise TestThatRunError('Cannot determine board, please specify ' 372 'a --board option.') 373 logging.info('Detected host board: %s', board) 374 return board 375 376 377def _auto_detect_labels(afe, remote): 378 """Automatically detect host labels and add them to the host in afe. 379 380 Note that the label of board will not be auto-detected. 381 This method assumes the host |remote| has already been added to afe. 382 383 @param afe: A direct_afe object used to interact with local afe database. 384 @param remote: The hostname of the remote device. 385 386 """ 387 cros_host = factory.create_host(remote) 388 labels_to_create = [label for label in cros_host.get_labels() 389 if not label.startswith(constants.BOARD_PREFIX)] 390 labels_to_add_to_afe_host = [] 391 for label in labels_to_create: 392 new_label = afe.create_label(label) 393 labels_to_add_to_afe_host.append(new_label.name) 394 hosts = afe.get_hosts(hostname=remote) 395 if not hosts: 396 raise TestThatRunError('Unexpected error: %s has not ' 397 'been added to afe.' % remote) 398 afe_host = hosts[0] 399 afe_host.add_labels(labels_to_add_to_afe_host) 400 401 402def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 403 build=_NO_BUILD, board=_NO_BOARD, args=None, 404 pretend=False, no_experimental=False, 405 ignore_deps=True, 406 results_directory=None, ssh_verbosity=0, 407 ssh_options=None, 408 autoserv_verbose=False, 409 iterations=1): 410 """Perform local run of tests. 411 412 This method enforces satisfaction of test dependencies for tests that are 413 run as a part of a suite. 414 415 @param afe: A direct_afe object used to interact with local afe database. 416 @param autotest_path: Absolute path of autotest installed in sysroot or 417 custom autotest path set by --autotest_dir. 418 @param tests: List of strings naming tests and suites to run. Suite strings 419 should be formed like "suite:smoke". 420 @param remote: Remote hostname. 421 @param fast_mode: bool to use fast mode (disables slow autotest features). 422 @param build: String specifying build for local run. 423 @param board: String specifyinb board for local run. 424 @param args: String that should be passed as args parameter to autoserv, 425 and then ultimitely to test itself. 426 @param pretend: If True, will print out autoserv commands rather than 427 running them. 428 @param no_experimental: Skip experimental tests when scheduling a suite. 429 @param ignore_deps: If True, test dependencies will be ignored. 430 @param results_directory: Directory to store results in. Defaults to None, 431 in which case results will be stored in a new 432 subdirectory of /tmp 433 @param ssh_verbosity: SSH verbosity level, passed through to 434 autoserv_utils. 435 @param ssh_options: Additional ssh options to be passed to autoserv_utils 436 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 437 @param iterations: int number of times to schedule tests. 438 """ 439 # Create host in afe, add board and build labels. 440 cros_version_label = provision.cros_version_to_label(build) 441 build_label = afe.create_label(cros_version_label) 442 board_label = afe.create_label(constants.BOARD_PREFIX + board) 443 new_host = afe.create_host(remote) 444 new_host.add_labels([build_label.name, board_label.name]) 445 if not ignore_deps: 446 logging.info('Auto-detecting labels for %s', remote) 447 _auto_detect_labels(afe, remote) 448 # Provision the host to |build|. 449 if build != _NO_BUILD: 450 logging.info('Provisioning %s...', cros_version_label) 451 try: 452 run_provisioning_job(cros_version_label, remote, autotest_path, 453 results_directory, fast_mode, 454 ssh_verbosity, ssh_options, 455 pretend, autoserv_verbose) 456 except TestThatProvisioningError as e: 457 logging.error('Provisioning %s to %s failed, tests are aborted, ' 458 'failure reason: %s', 459 remote, cros_version_label, e) 460 return 461 462 # Create suites that will be scheduled. 463 suites_and_descriptions = [] 464 for test in tests: 465 (predicate, description) = get_predicate_for_test_arg(test) 466 logging.info('Fetching suite for %s...', description) 467 suite = fetch_local_suite(autotest_path, predicate, afe, test_arg=test, 468 remote=remote, 469 build=build, board=board, 470 results_directory=results_directory, 471 no_experimental=no_experimental, 472 ignore_deps=ignore_deps) 473 suites_and_descriptions.append((suite, description)) 474 475 # Schedule the suites, looping over iterations if necessary. 476 for iteration in range(iterations): 477 if iteration > 0: 478 logging.info('Repeating scheduling for iteration %d:', iteration) 479 480 for suite, description in suites_and_descriptions: 481 logging.info('Scheduling suite for %s...', description) 482 ntests = suite.schedule( 483 lambda log_entry, log_in_subdir=False: None, 484 add_experimental=not no_experimental) 485 logging.info('... scheduled %s job(s).', ntests) 486 487 if not afe.get_jobs(): 488 logging.info('No jobs scheduled. End of local run.') 489 return 490 491 last_job_id = afe.get_jobs()[-1].id 492 job_id_digits = len(str(last_job_id)) 493 for job in afe.get_jobs(): 494 run_job(job, remote, autotest_path, results_directory, fast_mode, 495 job_id_digits, ssh_verbosity, ssh_options, args, pretend, 496 autoserv_verbose) 497 498 499def validate_arguments(arguments): 500 """ 501 Validates parsed arguments. 502 503 @param arguments: arguments object, as parsed by ParseArguments 504 @raises: ValueError if arguments were invalid. 505 """ 506 if arguments.remote == ':lab:': 507 if arguments.args: 508 raise ValueError('--args flag not supported when running against ' 509 ':lab:') 510 if arguments.pretend: 511 raise ValueError('--pretend flag not supported when running ' 512 'against :lab:') 513 if arguments.ssh_verbosity: 514 raise ValueError('--ssh_verbosity flag not supported when running ' 515 'against :lab:') 516 517 518def parse_arguments(argv): 519 """ 520 Parse command line arguments 521 522 @param argv: argument list to parse 523 @returns: parsed arguments. 524 @raises SystemExit if arguments are malformed, or required arguments 525 are not present. 526 """ 527 parser = argparse.ArgumentParser(description='Run remote tests.') 528 529 parser.add_argument('remote', metavar='REMOTE', 530 help='hostname[:port] for remote device. Specify ' 531 ':lab: to run in test lab, or :vm:PORT_NUMBER to ' 532 'run in vm. When tests are run in the lab, ' 533 'test_that will use the client autotest package ' 534 'for the build specified with --build, and the ' 535 'lab server code rather than local changes.') 536 parser.add_argument('tests', nargs='+', metavar='TEST', 537 help='Run given test(s). Use suite:SUITE to specify ' 538 'test suite. Use e:[NAME_PATTERN] to specify a ' 539 'NAME-matching regular expression. Use ' 540 'f:[FILE_PATTERN] to specify a filename matching ' 541 'regular expression. Specified regular ' 542 'expressions will be implicitly wrapped in ' 543 '^ and $.') 544 default_board = cros_build_lib.GetDefaultBoard() 545 parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, 546 action='store', 547 help='Board for which the test will run. Default: %s' % 548 (default_board or 'Not configured')) 549 parser.add_argument('-i', '--build', metavar='BUILD', default=_NO_BUILD, 550 help='Build to test. Device will be reimaged if ' 551 'necessary. Omit flag to skip reimage and test ' 552 'against already installed DUT image. Examples: ' 553 'link-paladin/R34-5222.0.0-rc2, ' 554 'lumpy-release/R34-5205.0.0') 555 parser.add_argument('-p', '--pool', metavar='POOL', default='suites', 556 help='Pool to use when running tests in the lab. ' 557 'Default is "suites"') 558 parser.add_argument('--fast', action='store_true', dest='fast_mode', 559 default=False, 560 help='Enable fast mode. This will cause test_that to ' 561 'skip time consuming steps like sysinfo and ' 562 'collecting crash information.') 563 parser.add_argument('--args', metavar='ARGS', 564 help='Whitespace separated argument string to pass ' 565 'through to test. Only supported for runs ' 566 'against a local DUT.') 567 parser.add_argument('--autotest_dir', metavar='AUTOTEST_DIR', 568 help='Use AUTOTEST_DIR instead of normal board sysroot ' 569 'copy of autotest, and skip the quickmerge step.') 570 parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, 571 help='Instead of storing results in a new subdirectory' 572 ' of /tmp , store results in RESULTS_DIR. If ' 573 'RESULTS_DIR already exists, it will be deleted.') 574 parser.add_argument('--pretend', action='store_true', default=False, 575 help='Print autoserv commands that would be run, ' 576 'rather than running them.') 577 parser.add_argument('--no-quickmerge', action='store_true', default=False, 578 dest='no_quickmerge', 579 help='Skip the quickmerge step and use the sysroot ' 580 'as it currently is. May result in un-merged ' 581 'source tree changes not being reflected in the ' 582 'run. If using --autotest_dir, this flag is ' 583 'automatically applied.') 584 parser.add_argument('--no-experimental', action='store_true', 585 default=False, dest='no_experimental', 586 help='When scheduling a suite, skip any tests marked ' 587 'as experimental. Applies only to tests scheduled' 588 ' via suite:[SUITE].') 589 parser.add_argument('--whitelist-chrome-crashes', action='store_true', 590 default=False, dest='whitelist_chrome_crashes', 591 help='Ignore chrome crashes when producing test ' 592 'report. This flag gets passed along to the ' 593 'report generation tool.') 594 parser.add_argument('--enforce-deps', action='store_true', 595 default=False, dest='enforce_deps', 596 help='Skip tests whose DEPENDENCIES can not ' 597 'be satisfied.') 598 parser.add_argument('--ssh_verbosity', action='store', type=int, 599 choices=[0, 1, 2, 3], default=0, 600 help='Verbosity level for ssh, between 0 and 3 ' 601 'inclusive.') 602 parser.add_argument('--ssh_options', action='store', default=None, 603 help='A string giving additional options to be ' 604 'added to ssh commands.') 605 parser.add_argument('--debug', action='store_true', 606 help='Include DEBUG level messages in stdout. Note: ' 607 'these messages will be included in output log ' 608 'file regardless. In addition, turn on autoserv ' 609 'verbosity.') 610 parser.add_argument('--iterations', action='store', type=int, default=1, 611 help='Number of times to run the tests specified.') 612 return parser.parse_args(argv) 613 614 615def sigint_handler(signum, stack_frame): 616 #pylint: disable-msg=C0111 617 """Handle SIGINT or SIGTERM to a local test_that run. 618 619 This handler sends a SIGINT to the running autoserv process, 620 if one is running, giving it up to 5 seconds to clean up and exit. After 621 the timeout elapses, autoserv is killed. In either case, after autoserv 622 exits then this process exits with status 1. 623 """ 624 # If multiple signals arrive before handler is unset, ignore duplicates 625 if not _sigint_handler_lock.acquire(False): 626 return 627 try: 628 # Ignore future signals by unsetting handler. 629 signal.signal(signal.SIGINT, signal.SIG_IGN) 630 signal.signal(signal.SIGTERM, signal.SIG_IGN) 631 632 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 633 if _autoserv_proc: 634 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 635 'to %s seconds for cleanup.', 636 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 637 _autoserv_proc.send_signal(signal.SIGINT) 638 timed_out, _ = retry.timeout(_autoserv_proc.wait, 639 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 640 if timed_out: 641 _autoserv_proc.kill() 642 logging.warning('Timed out waiting for autoserv to handle ' 643 'SIGINT. Killed autoserv.') 644 finally: 645 _sigint_handler_lock.release() # this is not really necessary? 646 sys.exit(1) 647 648 649def _create_results_directory(results_directory=None): 650 """Create a results directory. 651 652 If no directory is specified this method will create and return a 653 temp directory to hold results. If a directory name is specified this 654 method will create a directory at the given path, provided it doesn't 655 already exist. 656 657 @param results_directory: The path to the results_directory to create. 658 659 @return results_directory: A path to the results_directory, ready for use. 660 """ 661 if results_directory is None: 662 # Create a results_directory as subdir of /tmp 663 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 664 else: 665 # Delete results_directory if it already exists. 666 try: 667 shutil.rmtree(results_directory) 668 except OSError as e: 669 if e.errno != errno.ENOENT: 670 raise 671 672 # Create results_directory if it does not exist 673 try: 674 os.makedirs(results_directory) 675 except OSError as e: 676 if e.errno != errno.EEXIST: 677 raise 678 return results_directory 679 680 681def _perform_bootstrap_into_autotest_root(arguments, autotest_path, argv): 682 """ 683 Perfoms a bootstrap to run test_that from the |autotest_path|. 684 685 This function is to be called from test_that's main() script, when 686 test_that is executed from the source tree location. It runs 687 autotest_quickmerge to update the sysroot unless arguments.no_quickmerge 688 is set. It then executes and waits on the version of test_that.py 689 in |autotest_path|. 690 691 @param arguments: A parsed arguments object, as returned from 692 parse_arguments(...). 693 @param autotest_path: Full absolute path to the autotest root directory. 694 @param argv: The arguments list, as passed to main(...) 695 696 @returns: The return code of the test_that script that was executed in 697 |autotest_path|. 698 """ 699 logging_manager.configure_logging( 700 server_logging_config.ServerLoggingConfig(), 701 use_console=True, 702 verbose=arguments.debug) 703 if arguments.no_quickmerge: 704 logging.info('Skipping quickmerge step.') 705 else: 706 logging.info('Running autotest_quickmerge step.') 707 command = [_QUICKMERGE_SCRIPTNAME, '--board='+arguments.board, '--force'] 708 s = subprocess.Popen(command, 709 stdout=subprocess.PIPE, 710 stderr=subprocess.STDOUT) 711 for message in iter(s.stdout.readline, b''): 712 logging.info('quickmerge| %s', message.strip()) 713 return_code = s.wait() 714 if return_code: 715 raise TestThatRunError('autotest_quickmerge failed with error code' 716 ' %s.' % return_code) 717 718 logging.info('Re-running test_that script in %s copy of autotest.', 719 autotest_path) 720 script_command = os.path.join(autotest_path, 'site_utils', 721 os.path.basename(__file__)) 722 if not os.path.exists(script_command): 723 raise TestThatRunError('Unable to bootstrap to autotest root, ' 724 '%s not found.' % script_command) 725 proc = None 726 def resend_sig(signum, stack_frame): 727 #pylint: disable-msg=C0111 728 if proc: 729 proc.send_signal(signum) 730 signal.signal(signal.SIGINT, resend_sig) 731 signal.signal(signal.SIGTERM, resend_sig) 732 733 proc = subprocess.Popen([script_command] + argv) 734 735 return proc.wait() 736 737 738def _perform_run_from_autotest_root(arguments, autotest_path, argv): 739 """ 740 Perform a test_that run, from the |autotest_path|. 741 742 This function is to be called from test_that's main() script, when 743 test_that is executed from the |autotest_path|. It handles all stages 744 of a test_that run that come after the bootstrap into |autotest_path|. 745 746 @param arguments: A parsed arguments object, as returned from 747 parse_arguments(...). 748 @param autotest_path: Full absolute path to the autotest root directory. 749 @param argv: The arguments list, as passed to main(...) 750 751 @returns: A return code that test_that should exit with. 752 """ 753 results_directory = arguments.results_dir 754 if results_directory is None or not os.path.exists(results_directory): 755 raise ValueError('Expected valid results directory, got %s' % 756 results_directory) 757 758 logging_manager.configure_logging( 759 server_logging_config.ServerLoggingConfig(), 760 results_dir=results_directory, 761 use_console=True, 762 verbose=arguments.debug, 763 debug_log_name='test_that') 764 logging.info('Began logging to %s', results_directory) 765 766 logging.debug('test_that command line was: %s', argv) 767 768 signal.signal(signal.SIGINT, sigint_handler) 769 signal.signal(signal.SIGTERM, sigint_handler) 770 771 afe = setup_local_afe() 772 perform_local_run(afe, autotest_path, arguments.tests, 773 arguments.remote, arguments.fast_mode, 774 arguments.build, arguments.board, 775 args=arguments.args, 776 pretend=arguments.pretend, 777 no_experimental=arguments.no_experimental, 778 ignore_deps=not arguments.enforce_deps, 779 results_directory=results_directory, 780 ssh_verbosity=arguments.ssh_verbosity, 781 ssh_options=arguments.ssh_options, 782 autoserv_verbose=arguments.debug, 783 iterations=arguments.iterations) 784 if arguments.pretend: 785 logging.info('Finished pretend run. Exiting.') 786 return 0 787 788 test_report_command = [_TEST_REPORT_SCRIPTNAME] 789 # Experimental test results do not influence the exit code. 790 test_report_command.append('--ignore_experimental_tests') 791 if arguments.whitelist_chrome_crashes: 792 test_report_command.append('--whitelist_chrome_crashes') 793 test_report_command.append(results_directory) 794 final_result = subprocess.call(test_report_command) 795 with open(os.path.join(results_directory, 'test_report.log'), 796 'w') as report_log: 797 subprocess.call(test_report_command, stdout=report_log) 798 try: 799 os.unlink(_LATEST_RESULTS_DIRECTORY) 800 except OSError: 801 pass 802 link_target = os.path.relpath(results_directory, 803 os.path.dirname(_LATEST_RESULTS_DIRECTORY)) 804 os.symlink(link_target, _LATEST_RESULTS_DIRECTORY) 805 logging.info('Finished running tests. Results can be found in %s or %s', 806 results_directory, _LATEST_RESULTS_DIRECTORY) 807 return final_result 808 809 810def _main_for_local_run(argv, arguments): 811 """ 812 Effective entry point for local test_that runs. 813 814 @param argv: Script command line arguments. 815 @param arguments: Parsed command line arguments. 816 """ 817 if not cros_build_lib.IsInsideChroot(): 818 print >> sys.stderr, 'For local runs, script must be run inside chroot.' 819 return 1 820 821 results_directory = _create_results_directory(arguments.results_dir) 822 _add_ssh_identity(results_directory) 823 arguments.results_dir = results_directory 824 825 # If the board has not been specified through --board, and is not set in the 826 # default_board file, determine the board by ssh-ing into the host. Also 827 # prepend it to argv so we can re-use it when we run test_that from the 828 # sysroot. 829 if arguments.board is None: 830 arguments.board = _get_board_from_host(arguments.remote) 831 argv = ['--board', arguments.board] + argv 832 833 if arguments.autotest_dir: 834 autotest_path = arguments.autotest_dir 835 arguments.no_quickmerge = True 836 else: 837 sysroot_path = os.path.join('/build', arguments.board, '') 838 839 if not os.path.exists(sysroot_path): 840 print >> sys.stderr, ('%s does not exist. Have you run ' 841 'setup_board?' % sysroot_path) 842 return 1 843 844 path_ending = 'usr/local/build/autotest' 845 autotest_path = os.path.join(sysroot_path, path_ending) 846 847 site_utils_path = os.path.join(autotest_path, 'site_utils') 848 849 if not os.path.exists(autotest_path): 850 print >> sys.stderr, ('%s does not exist. Have you run ' 851 'build_packages? Or if you are using ' 852 '--autotest-dir, make sure it points to ' 853 'a valid autotest directory.' % autotest_path) 854 return 1 855 856 realpath = os.path.realpath(__file__) 857 site_utils_path = os.path.realpath(site_utils_path) 858 859 # If we are not running the sysroot version of script, perform 860 # a quickmerge if necessary and then re-execute 861 # the sysroot version of script with the same arguments. 862 if os.path.dirname(realpath) != site_utils_path: 863 return _perform_bootstrap_into_autotest_root( 864 arguments, autotest_path, argv) 865 else: 866 return _perform_run_from_autotest_root( 867 arguments, autotest_path, argv) 868 869 870def _main_for_lab_run(argv, arguments): 871 """ 872 Effective entry point for lab test_that runs. 873 874 @param argv: Script command line arguments. 875 @param arguments: Parsed command line arguments. 876 """ 877 autotest_path = os.path.realpath(os.path.join(os.path.dirname(__file__), 878 '..')) 879 flattened_argv = ' '.join([pipes.quote(item) for item in argv]) 880 command = [os.path.join(autotest_path, 'site_utils', 881 'run_suite.py'), 882 '--board', arguments.board, 883 '--build', arguments.build, 884 '--suite_name', 'test_that_wrapper', 885 '--pool', arguments.pool, 886 '--suite_args', flattened_argv] 887 logging.info('About to start lab suite with command %s.', command) 888 return subprocess.call(command) 889 890 891def main(argv): 892 """ 893 Entry point for test_that script. 894 895 @param argv: arguments list 896 """ 897 arguments = parse_arguments(argv) 898 try: 899 validate_arguments(arguments) 900 except ValueError as err: 901 print >> sys.stderr, ('Invalid arguments. %s' % err.message) 902 return 1 903 904 if arguments.remote == ':lab:': 905 return _main_for_lab_run(argv, arguments) 906 else: 907 return _main_for_local_run(argv, arguments) 908 909 910if __name__ == '__main__': 911 sys.exit(main(sys.argv[1:])) 912