test_that.py revision f81917c3692293e7b72a4fa467813cd1a43f3126
1#!/usr/bin/python 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import errno 8import os 9import pipes 10import re 11import shutil 12import signal 13import stat 14import subprocess 15import sys 16import tempfile 17import threading 18 19import logging 20# Turn the logging level to INFO before importing other autotest 21# code, to avoid having failed import logging messages confuse the 22# test_that user. 23logging.basicConfig(level=logging.INFO) 24 25import common 26from autotest_lib.client.common_lib.cros import dev_server, retry 27from autotest_lib.client.common_lib import error, logging_manager 28from autotest_lib.server.cros.dynamic_suite import suite, constants 29from autotest_lib.server.cros import provision 30from autotest_lib.server.hosts import factory 31from autotest_lib.server import autoserv_utils 32from autotest_lib.server import server_logging_config 33from autotest_lib.server import utils 34 35 36try: 37 from chromite.lib import cros_build_lib 38except ImportError: 39 print 'Unable to import chromite.' 40 print 'This script must be either:' 41 print ' - Be run in the chroot.' 42 print ' - (not yet supported) be run after running ' 43 print ' ../utils/build_externals.py' 44 45_autoserv_proc = None 46_sigint_handler_lock = threading.Lock() 47 48_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 49_NO_BOARD = 'ad_hoc_board' 50_NO_BUILD = 'ad_hoc_build' 51_SUITE_REGEX = r'suite:(.*)' 52 53_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' 54_TEST_KEY_FILENAME = 'testing_rsa' 55_TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 56 'ssh_keys/%s' % _TEST_KEY_FILENAME) 57 58_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' 59 60_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 61 62 63class TestThatRunError(Exception): 64 """Raised if test_that encounters something unexpected while running.""" 65 66 67class TestThatProvisioningError(Exception): 68 """Raised when it fails to provision the DUT to the requested build.""" 69 70 71def fetch_local_suite(autotest_path, suite_predicate, afe, remote, 72 build=_NO_BUILD, board=_NO_BOARD, 73 results_directory=None, no_experimental=False, 74 ignore_deps=True): 75 """Create a suite from the given suite predicate. 76 77 Satisfaction of dependencies is enforced by Suite.schedule() if 78 ignore_deps is False. Note that this method assumes only one host, 79 i.e. |remote|, was added to afe. Suite.schedule() will not 80 schedule a job if none of the hosts in the afe (in our case, 81 just one host |remote|) has a label that matches a requested 82 test dependency. 83 84 @param autotest_path: Absolute path to autotest (in sysroot or 85 custom autotest directory set by --autotest_dir). 86 @param suite_predicate: callable that takes ControlData objects, and 87 returns True on those that should be in suite 88 @param afe: afe object to schedule against (typically a directAFE) 89 @param remote: String representing the IP of the remote host. 90 @param build: Build to schedule suite for. 91 @param board: Board to schedule suite for. 92 @param results_directory: Absolute path of directory to store results in. 93 (results will be stored in subdirectory of this). 94 @param no_experimental: Skip experimental tests when scheduling a suite. 95 @param ignore_deps: If True, test dependencies will be ignored. 96 97 @returns: A suite.Suite object. 98 99 """ 100 fs_getter = suite.Suite.create_fs_getter(autotest_path) 101 devserver = dev_server.ImageServer('') 102 my_suite = suite.Suite.create_from_predicates([suite_predicate], 103 build, constants.BOARD_PREFIX + board, 104 devserver, fs_getter, afe=afe, 105 ignore_deps=ignore_deps, 106 results_dir=results_directory, forgiving_parser=False) 107 if len(my_suite.tests) == 0: 108 raise ValueError('Suite contained no tests.') 109 110 if not ignore_deps: 111 # Log tests whose dependencies can't be satisfied. 112 labels = [label.name for label in 113 afe.get_labels(host__hostname=remote)] 114 for test in my_suite.tests: 115 if test.experimental and no_experimental: 116 continue 117 unsatisfiable_deps = set(test.dependencies).difference(labels) 118 if unsatisfiable_deps: 119 logging.warn('%s will be skipped, unsatisfiable ' 120 'test dependencies: %s', test.name, 121 unsatisfiable_deps) 122 return my_suite 123 124def _run_autoserv(command, pretend=False): 125 """Run autoserv command. 126 127 Run the autoserv command and wait on it. Log the stdout. 128 Ensure that SIGINT signals are passed along to autoserv. 129 130 @param command: the autoserv command to run. 131 @returns: exit code of the command. 132 133 """ 134 if not pretend: 135 logging.debug('Running autoserv command: %s', command) 136 global _autoserv_proc 137 _autoserv_proc = subprocess.Popen(command, 138 stdout=subprocess.PIPE, 139 stderr=subprocess.STDOUT) 140 # This incantation forces unbuffered reading from stdout, 141 # so that autoserv output can be displayed to the user 142 # immediately. 143 for message in iter(_autoserv_proc.stdout.readline, b''): 144 logging.info('autoserv| %s', message.strip()) 145 146 _autoserv_proc.wait() 147 returncode = _autoserv_proc.returncode 148 _autoserv_proc = None 149 else: 150 logging.info('Pretend mode. Would run autoserv command: %s', 151 command) 152 returncode = 0 153 return returncode 154 155 156def run_provisioning_job(provision_label, host, autotest_path, 157 results_directory, fast_mode, 158 ssh_verbosity=0, ssh_options=None, 159 pretend=False, autoserv_verbose=False): 160 """Shell out to autoserv to run provisioning job. 161 162 @param provision_label: Label to provision the machine to. 163 @param host: Hostname of DUT. 164 @param autotest_path: Absolute path of autotest directory. 165 @param results_directory: Absolute path of directory to store results in. 166 (results will be stored in subdirectory of this). 167 @param fast_mode: bool to use fast mode (disables slow autotest features). 168 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 169 @param ssh_options: Additional ssh options to be passed to autoserv_utils 170 @param pretend: If True, will print out autoserv commands rather than 171 running them. 172 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 173 174 @returns: Absolute path of directory where results were stored. 175 176 """ 177 # TODO(fdeng): When running against a local DUT, autoserv 178 # is still hitting the AFE in the lab. 179 # provision_AutoUpdate checks the current build of DUT by 180 # retrieving build info from AFE. crosbug.com/295178 181 results_directory = os.path.join(results_directory, 'results-provision') 182 provision_arg = '='.join(['--provision', provision_label]) 183 command = autoserv_utils.autoserv_run_job_command( 184 os.path.join(autotest_path, 'server'), 185 machines=host, job=None, verbose=autoserv_verbose, 186 results_directory=results_directory, 187 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 188 ssh_options=ssh_options, extra_args=[provision_arg], 189 no_console_prefix=True) 190 if _run_autoserv(command, pretend) != 0: 191 raise TestThatProvisioningError('Command returns non-zero code: %s ' % 192 command) 193 return results_directory 194 195 196def run_job(job, host, autotest_path, results_directory, fast_mode, 197 id_digits=1, ssh_verbosity=0, ssh_options=None, 198 args=None, pretend=False, 199 autoserv_verbose=False): 200 """ 201 Shell out to autoserv to run an individual test job. 202 203 @param job: A Job object containing the control file contents and other 204 relevent metadata for this test. 205 @param host: Hostname of DUT to run test against. 206 @param autotest_path: Absolute path of autotest directory. 207 @param results_directory: Absolute path of directory to store results in. 208 (results will be stored in subdirectory of this). 209 @param fast_mode: bool to use fast mode (disables slow autotest features). 210 @param id_digits: The minimum number of digits that job ids should be 211 0-padded to when formatting as a string for results 212 directory. 213 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 214 @param ssh_options: Additional ssh options to be passed to autoserv_utils 215 @param args: String that should be passed as args parameter to autoserv, 216 and then ultimitely to test itself. 217 @param pretend: If True, will print out autoserv commands rather than 218 running them. 219 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 220 @returns: Absolute path of directory where results were stored. 221 """ 222 with tempfile.NamedTemporaryFile() as temp_file: 223 temp_file.write(job.control_file) 224 temp_file.flush() 225 name_tail = job.name.split('/')[-1] 226 results_directory = os.path.join(results_directory, 227 'results-%0*d-%s' % (id_digits, job.id, 228 name_tail)) 229 # Drop experimental keyval in the keval file in the job result folder. 230 os.makedirs(results_directory) 231 utils.write_keyval(results_directory, 232 {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ 233 constants.JOB_EXPERIMENTAL_KEY]}) 234 extra_args = [temp_file.name] 235 if args: 236 extra_args.extend(['--args', args]) 237 238 command = autoserv_utils.autoserv_run_job_command( 239 os.path.join(autotest_path, 'server'), 240 machines=host, job=job, verbose=autoserv_verbose, 241 results_directory=results_directory, 242 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 243 ssh_options=ssh_options, 244 extra_args=extra_args, 245 no_console_prefix=True) 246 247 _run_autoserv(command, pretend) 248 return results_directory 249 250 251def setup_local_afe(): 252 """ 253 Setup a local afe database and return a direct_afe object to access it. 254 255 @returns: A autotest_lib.frontend.afe.direct_afe instance. 256 """ 257 # This import statement is delayed until now rather than running at 258 # module load time, because it kicks off a local sqlite :memory: backed 259 # database, and we don't need that unless we are doing a local run. 260 from autotest_lib.frontend import setup_django_lite_environment 261 from autotest_lib.frontend.afe import direct_afe 262 return direct_afe.directAFE() 263 264 265def get_predicate_for_test_arg(test): 266 """ 267 Gets a suite predicte function for a given command-line argument. 268 269 @param test: String. An individual TEST command line argument, e.g. 270 'login_CryptohomeMounted' or 'suite:smoke' 271 @returns: A (predicate, string) tuple with the necessary suite 272 predicate, and a description string of the suite that 273 this predicate will produce. 274 """ 275 suitematch = re.match(_SUITE_REGEX, test) 276 name_pattern_match = re.match(r'e:(.*)', test) 277 file_pattern_match = re.match(r'f:(.*)', test) 278 if suitematch: 279 suitename = suitematch.group(1) 280 return (suite.Suite.name_in_tag_predicate(suitename), 281 'suite named %s' % suitename) 282 if name_pattern_match: 283 pattern = '^%s$' % name_pattern_match.group(1) 284 return (suite.Suite.test_name_matches_pattern_predicate(pattern), 285 'suite to match name pattern %s' % pattern) 286 if file_pattern_match: 287 pattern = '^%s$' % file_pattern_match.group(1) 288 return (suite.Suite.test_file_matches_pattern_predicate(pattern), 289 'suite to match file name pattern %s' % pattern) 290 return (suite.Suite.test_name_equals_predicate(test), 291 'job named %s' % test) 292 293 294def _add_ssh_identity(temp_directory): 295 """Add an ssh identity to the agent. 296 297 @param temp_directory: A directory to copy the testing_rsa into. 298 """ 299 # Add the testing key to the current ssh agent. 300 if os.environ.has_key('SSH_AGENT_PID'): 301 # Copy the testing key to the temp directory and make it NOT 302 # world-readable. Otherwise, ssh-add complains. 303 shutil.copy(_TEST_KEY_PATH, temp_directory) 304 key_copy_path = os.path.join(temp_directory, _TEST_KEY_FILENAME) 305 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 306 p = subprocess.Popen(['ssh-add', key_copy_path], 307 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 308 p_out, _ = p.communicate() 309 for line in p_out.splitlines(): 310 logging.info(line) 311 else: 312 logging.warning('There appears to be no running ssh-agent. Attempting ' 313 'to continue without running ssh-add, but ssh commands ' 314 'may fail.') 315 316 317def _get_board_from_host(remote): 318 """Get the board of the remote host. 319 320 @param remote: string representing the IP of the remote host. 321 322 @return: A string representing the board of the remote host. 323 """ 324 logging.info('Board unspecified, attempting to determine board from host.') 325 host = factory.create_host(remote) 326 try: 327 board = host.get_board().replace(constants.BOARD_PREFIX, '') 328 except error.AutoservRunError: 329 raise TestThatRunError('Cannot determine board, please specify ' 330 'a --board option.') 331 logging.info('Detected host board: %s', board) 332 return board 333 334 335def _auto_detect_labels(afe, remote): 336 """Automatically detect host labels and add them to the host in afe. 337 338 Note that the label of board will not be auto-detected. 339 This method assumes the host |remote| has already been added to afe. 340 341 @param afe: A direct_afe object used to interact with local afe database. 342 @param remote: The hostname of the remote device. 343 344 """ 345 cros_host = factory.create_host(remote) 346 labels_to_create = [label for label in cros_host.get_labels() 347 if not label.startswith(constants.BOARD_PREFIX)] 348 labels_to_add_to_afe_host = [] 349 for label in labels_to_create: 350 new_label = afe.create_label(label) 351 labels_to_add_to_afe_host.append(new_label.name) 352 hosts = afe.get_hosts(hostname=remote) 353 if not hosts: 354 raise TestThatRunError('Unexpected error: %s has not ' 355 'been added to afe.' % remote) 356 afe_host = hosts[0] 357 afe_host.add_labels(labels_to_add_to_afe_host) 358 359 360def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 361 build=_NO_BUILD, board=_NO_BOARD, args=None, 362 pretend=False, no_experimental=False, 363 ignore_deps=True, 364 results_directory=None, ssh_verbosity=0, 365 ssh_options=None, 366 autoserv_verbose=False, 367 iterations=1): 368 """Perform local run of tests. 369 370 This method enforces satisfaction of test dependencies for tests that are 371 run as a part of a suite. 372 373 @param afe: A direct_afe object used to interact with local afe database. 374 @param autotest_path: Absolute path of autotest installed in sysroot or 375 custom autotest path set by --autotest_dir. 376 @param tests: List of strings naming tests and suites to run. Suite strings 377 should be formed like "suite:smoke". 378 @param remote: Remote hostname. 379 @param fast_mode: bool to use fast mode (disables slow autotest features). 380 @param build: String specifying build for local run. 381 @param board: String specifyinb board for local run. 382 @param args: String that should be passed as args parameter to autoserv, 383 and then ultimitely to test itself. 384 @param pretend: If True, will print out autoserv commands rather than 385 running them. 386 @param no_experimental: Skip experimental tests when scheduling a suite. 387 @param ignore_deps: If True, test dependencies will be ignored. 388 @param results_directory: Directory to store results in. Defaults to None, 389 in which case results will be stored in a new 390 subdirectory of /tmp 391 @param ssh_verbosity: SSH verbosity level, passed through to 392 autoserv_utils. 393 @param ssh_options: Additional ssh options to be passed to autoserv_utils 394 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 395 @param iterations: int number of times to schedule tests. 396 """ 397 398 # Create host in afe, add board and build labels. 399 cros_version_label = provision.cros_version_to_label(build) 400 build_label = afe.create_label(cros_version_label) 401 board_label = afe.create_label(constants.BOARD_PREFIX + board) 402 new_host = afe.create_host(remote) 403 new_host.add_labels([build_label.name, board_label.name]) 404 if not ignore_deps: 405 logging.info('Auto-detecting labels for %s', remote) 406 _auto_detect_labels(afe, remote) 407 # Provision the host to |build|. 408 if build != _NO_BUILD: 409 logging.info('Provisioning %s...', cros_version_label) 410 try: 411 run_provisioning_job(cros_version_label, remote, autotest_path, 412 results_directory, fast_mode, 413 ssh_verbosity, ssh_options, 414 pretend, autoserv_verbose) 415 except TestThatProvisioningError as e: 416 logging.error('Provisioning %s to %s failed, tests are aborted, ' 417 'failure reason: %s', 418 remote, cros_version_label, e) 419 return 420 421 # Create suites that will be scheduled. 422 suites_and_descriptions = [] 423 for test in tests: 424 (predicate, description) = get_predicate_for_test_arg(test) 425 logging.info('Fetching suite for %s...', description) 426 suite = fetch_local_suite(autotest_path, predicate, afe, 427 remote=remote, 428 build=build, board=board, 429 results_directory=results_directory, 430 no_experimental=no_experimental, 431 ignore_deps=ignore_deps) 432 suites_and_descriptions.append((suite, description)) 433 434 # Schedule the suites, looping over iterations if necessary. 435 for iteration in range(iterations): 436 if iteration > 0: 437 logging.info('Repeating scheduling for iteration %d:', iteration) 438 439 for suite, description in suites_and_descriptions: 440 logging.info('Scheduling suite for %s...', description) 441 ntests = suite.schedule( 442 lambda log_entry, log_in_subdir=False: None, 443 add_experimental=not no_experimental) 444 logging.info('... scheduled %s job(s).', ntests) 445 446 if not afe.get_jobs(): 447 logging.info('No jobs scheduled. End of local run.') 448 return 449 450 last_job_id = afe.get_jobs()[-1].id 451 job_id_digits = len(str(last_job_id)) 452 for job in afe.get_jobs(): 453 run_job(job, remote, autotest_path, results_directory, fast_mode, 454 job_id_digits, ssh_verbosity, ssh_options, args, pretend, 455 autoserv_verbose) 456 457 458def validate_arguments(arguments): 459 """ 460 Validates parsed arguments. 461 462 @param arguments: arguments object, as parsed by ParseArguments 463 @raises: ValueError if arguments were invalid. 464 """ 465 if arguments.remote == ':lab:': 466 if arguments.args: 467 raise ValueError('--args flag not supported when running against ' 468 ':lab:') 469 if arguments.pretend: 470 raise ValueError('--pretend flag not supported when running ' 471 'against :lab:') 472 if arguments.ssh_verbosity: 473 raise ValueError('--ssh_verbosity flag not supported when running ' 474 'against :lab:') 475 476 477def parse_arguments(argv): 478 """ 479 Parse command line arguments 480 481 @param argv: argument list to parse 482 @returns: parsed arguments. 483 @raises SystemExit if arguments are malformed, or required arguments 484 are not present. 485 """ 486 parser = argparse.ArgumentParser(description='Run remote tests.') 487 488 parser.add_argument('remote', metavar='REMOTE', 489 help='hostname[:port] for remote device. Specify ' 490 ':lab: to run in test lab, or :vm:PORT_NUMBER to ' 491 'run in vm.') 492 parser.add_argument('tests', nargs='+', metavar='TEST', 493 help='Run given test(s). Use suite:SUITE to specify ' 494 'test suite. Use e:[NAME_PATTERN] to specify a ' 495 'NAME-matching regular expression. Use ' 496 'f:[FILE_PATTERN] to specify a filename matching ' 497 'regular expression. Specified regular ' 498 'expressions will be implicitly wrapped in ' 499 '^ and $.') 500 default_board = cros_build_lib.GetDefaultBoard() 501 parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, 502 action='store', 503 help='Board for which the test will run. Default: %s' % 504 (default_board or 'Not configured')) 505 parser.add_argument('-i', '--build', metavar='BUILD', default=_NO_BUILD, 506 help='Build to test. Device will be reimaged if ' 507 'necessary. Omit flag to skip reimage and test ' 508 'against already installed DUT image. Examples: ' 509 'link-paladin/R34-5222.0.0-rc2, ' 510 'lumpy-release/R34-5205.0.0') 511 parser.add_argument('--fast', action='store_true', dest='fast_mode', 512 default=False, 513 help='Enable fast mode. This will cause test_that to ' 514 'skip time consuming steps like sysinfo and ' 515 'collecting crash information.') 516 parser.add_argument('--args', metavar='ARGS', 517 help='Whitespace separated argument string to pass ' 518 'through to test. Only supported for runs ' 519 'against a local DUT.') 520 parser.add_argument('--autotest_dir', metavar='AUTOTEST_DIR', 521 help='Use AUTOTEST_DIR instead of normal board sysroot ' 522 'copy of autotest, and skip the quickmerge step.') 523 parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, 524 help='Instead of storing results in a new subdirectory' 525 ' of /tmp , store results in RESULTS_DIR. If ' 526 'RESULTS_DIR already exists, it will be deleted.') 527 parser.add_argument('--pretend', action='store_true', default=False, 528 help='Print autoserv commands that would be run, ' 529 'rather than running them.') 530 parser.add_argument('--no-quickmerge', action='store_true', default=False, 531 dest='no_quickmerge', 532 help='Skip the quickmerge step and use the sysroot ' 533 'as it currently is. May result in un-merged ' 534 'source tree changes not being reflected in the ' 535 'run. If using --autotest_dir, this flag is ' 536 'automatically applied.') 537 parser.add_argument('--no-experimental', action='store_true', 538 default=False, dest='no_experimental', 539 help='When scheduling a suite, skip any tests marked ' 540 'as experimental. Applies only to tests scheduled' 541 ' via suite:[SUITE].') 542 parser.add_argument('--whitelist-chrome-crashes', action='store_true', 543 default=False, dest='whitelist_chrome_crashes', 544 help='Ignore chrome crashes when producing test ' 545 'report. This flag gets passed along to the ' 546 'report generation tool.') 547 parser.add_argument('--enforce-deps', action='store_true', 548 default=False, dest='enforce_deps', 549 help='Skip tests whose DEPENDENCIES can not ' 550 'be satisfied.') 551 parser.add_argument('--ssh_verbosity', action='store', type=int, 552 choices=[0, 1, 2, 3], default=0, 553 help='Verbosity level for ssh, between 0 and 3 ' 554 'inclusive.') 555 parser.add_argument('--ssh_options', action='store', default=None, 556 help='A string giving additional options to be ' 557 'added to ssh commands.') 558 parser.add_argument('--debug', action='store_true', 559 help='Include DEBUG level messages in stdout. Note: ' 560 'these messages will be included in output log ' 561 'file regardless. In addition, turn on autoserv ' 562 'verbosity.') 563 parser.add_argument('--iterations', action='store', type=int, default=1, 564 help='Number of times to run the tests specified.') 565 return parser.parse_args(argv) 566 567 568def sigint_handler(signum, stack_frame): 569 #pylint: disable-msg=C0111 570 """Handle SIGINT or SIGTERM to a local test_that run. 571 572 This handler sends a SIGINT to the running autoserv process, 573 if one is running, giving it up to 5 seconds to clean up and exit. After 574 the timeout elapses, autoserv is killed. In either case, after autoserv 575 exits then this process exits with status 1. 576 """ 577 # If multiple signals arrive before handler is unset, ignore duplicates 578 if not _sigint_handler_lock.acquire(False): 579 return 580 try: 581 # Ignore future signals by unsetting handler. 582 signal.signal(signal.SIGINT, signal.SIG_IGN) 583 signal.signal(signal.SIGTERM, signal.SIG_IGN) 584 585 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 586 if _autoserv_proc: 587 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 588 'to %s seconds for cleanup.', 589 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 590 _autoserv_proc.send_signal(signal.SIGINT) 591 timed_out, _ = retry.timeout(_autoserv_proc.wait, 592 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 593 if timed_out: 594 _autoserv_proc.kill() 595 logging.warning('Timed out waiting for autoserv to handle ' 596 'SIGINT. Killed autoserv.') 597 finally: 598 _sigint_handler_lock.release() # this is not really necessary? 599 sys.exit(1) 600 601 602def _create_results_directory(results_directory=None): 603 """Create a results directory. 604 605 If no directory is specified this method will create and return a 606 temp directory to hold results. If a directory name is specified this 607 method will create a directory at the given path, provided it doesn't 608 already exist. 609 610 @param results_directory: The path to the results_directory to create. 611 612 @return results_directory: A path to the results_directory, ready for use. 613 """ 614 if results_directory is None: 615 # Create a results_directory as subdir of /tmp 616 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 617 else: 618 # Delete results_directory if it already exists. 619 try: 620 shutil.rmtree(results_directory) 621 except OSError as e: 622 if e.errno != errno.ENOENT: 623 raise 624 625 # Create results_directory if it does not exist 626 try: 627 os.makedirs(results_directory) 628 except OSError as e: 629 if e.errno != errno.EEXIST: 630 raise 631 return results_directory 632 633 634def _perform_bootstrap_into_autotest_root(arguments, autotest_path, argv, 635 legacy_path=False): 636 """ 637 Perfoms a bootstrap to run test_that from the |autotest_path|. 638 639 This function is to be called from test_that's main() script, when 640 test_that is executed from the source tree location. It runs 641 autotest_quickmerge to update the sysroot unless arguments.no_quickmerge 642 is set. It then executes and waits on the version of test_that.py 643 in |autotest_path|. 644 645 @param arguments: A parsed arguments object, as returned from 646 parse_arguments(...). 647 @param autotest_path: Full absolute path to the autotest root directory. 648 @param argv: The arguments list, as passed to main(...) 649 @param legacy_path: Flag for backwards compatibility with builds 650 that have autotest in old usr/local/autotest location 651 652 @returns: The return code of the test_that script that was executed in 653 |autotest_path|. 654 """ 655 logging_manager.configure_logging( 656 server_logging_config.ServerLoggingConfig(), 657 use_console=True, 658 verbose=arguments.debug) 659 if arguments.no_quickmerge: 660 logging.info('Skipping quickmerge step.') 661 else: 662 logging.info('Running autotest_quickmerge step.') 663 command = [_QUICKMERGE_SCRIPTNAME, '--board='+arguments.board] 664 if legacy_path: 665 command.append('--legacy_path') 666 s = subprocess.Popen(command, 667 stdout=subprocess.PIPE, 668 stderr=subprocess.STDOUT) 669 for message in iter(s.stdout.readline, b''): 670 logging.info('quickmerge| %s', message.strip()) 671 return_code = s.wait() 672 if return_code: 673 raise TestThatRunError('autotest_quickmerge failed with error code' 674 ' %s.' % return_code) 675 676 logging.info('Re-running test_that script in %s copy of autotest.', 677 autotest_path) 678 script_command = os.path.join(autotest_path, 'site_utils', 679 os.path.basename(__file__)) 680 if not os.path.exists(script_command): 681 raise TestThatRunError('Unable to bootstrap to autotest root, ' 682 '%s not found.' % script_command) 683 proc = None 684 def resend_sig(signum, stack_frame): 685 #pylint: disable-msg=C0111 686 if proc: 687 proc.send_signal(signum) 688 signal.signal(signal.SIGINT, resend_sig) 689 signal.signal(signal.SIGTERM, resend_sig) 690 691 proc = subprocess.Popen([script_command] + argv) 692 693 return proc.wait() 694 695 696def _perform_run_from_autotest_root(arguments, autotest_path, argv): 697 """ 698 Perform a test_that run, from the |autotest_path|. 699 700 This function is to be called from test_that's main() script, when 701 test_that is executed from the |autotest_path|. It handles all stages 702 of a test_that run that come after the bootstrap into |autotest_path|. 703 704 @param arguments: A parsed arguments object, as returned from 705 parse_arguments(...). 706 @param autotest_path: Full absolute path to the autotest root directory. 707 @param argv: The arguments list, as passed to main(...) 708 709 @returns: A return code that test_that should exit with. 710 """ 711 results_directory = arguments.results_dir 712 if results_directory is None or not os.path.exists(results_directory): 713 raise ValueError('Expected valid results directory, got %s' % 714 results_directory) 715 716 logging_manager.configure_logging( 717 server_logging_config.ServerLoggingConfig(), 718 results_dir=results_directory, 719 use_console=True, 720 verbose=arguments.debug, 721 debug_log_name='test_that') 722 logging.info('Began logging to %s', results_directory) 723 724 logging.debug('test_that command line was: %s', argv) 725 726 signal.signal(signal.SIGINT, sigint_handler) 727 signal.signal(signal.SIGTERM, sigint_handler) 728 729 afe = setup_local_afe() 730 perform_local_run(afe, autotest_path, arguments.tests, 731 arguments.remote, arguments.fast_mode, 732 arguments.build, arguments.board, 733 args=arguments.args, 734 pretend=arguments.pretend, 735 no_experimental=arguments.no_experimental, 736 ignore_deps=not arguments.enforce_deps, 737 results_directory=results_directory, 738 ssh_verbosity=arguments.ssh_verbosity, 739 ssh_options=arguments.ssh_options, 740 autoserv_verbose=arguments.debug, 741 iterations=arguments.iterations) 742 if arguments.pretend: 743 logging.info('Finished pretend run. Exiting.') 744 return 0 745 746 test_report_command = [_TEST_REPORT_SCRIPTNAME] 747 # Experimental test results do not influence the exit code. 748 test_report_command.append('--ignore_experimental_tests') 749 if arguments.whitelist_chrome_crashes: 750 test_report_command.append('--whitelist_chrome_crashes') 751 test_report_command.append(results_directory) 752 final_result = subprocess.call(test_report_command) 753 with open(os.path.join(results_directory, 'test_report.log'), 754 'w') as report_log: 755 subprocess.call(test_report_command, stdout=report_log) 756 logging.info('Finished running tests. Results can be found in %s', 757 results_directory) 758 try: 759 os.unlink(_LATEST_RESULTS_DIRECTORY) 760 except OSError: 761 pass 762 os.symlink(results_directory, _LATEST_RESULTS_DIRECTORY) 763 return final_result 764 765 766def _main_for_local_run(argv, arguments): 767 """ 768 Effective entry point for local test_that runs. 769 770 @param argv: Script command line arguments. 771 @param arguments: Parsed command line arguments. 772 """ 773 if not cros_build_lib.IsInsideChroot(): 774 print >> sys.stderr, 'For local runs, script must be run inside chroot.' 775 return 1 776 777 results_directory = _create_results_directory(arguments.results_dir) 778 _add_ssh_identity(results_directory) 779 arguments.results_dir = results_directory 780 legacy_path = False 781 782 # If the board has not been specified through --board, and is not set in the 783 # default_board file, determine the board by ssh-ing into the host. Also 784 # prepend it to argv so we can re-use it when we run test_that from the 785 # sysroot. 786 if arguments.board is None: 787 arguments.board = _get_board_from_host(arguments.remote) 788 argv = ['--board', arguments.board] + argv 789 790 if arguments.autotest_dir: 791 autotest_path = arguments.autotest_dir 792 arguments.no_quickmerge = True 793 else: 794 sysroot_path = os.path.join('/build', arguments.board, '') 795 796 if not os.path.exists(sysroot_path): 797 print >> sys.stderr, ('%s does not exist. Have you run ' 798 'setup_board?' % sysroot_path) 799 return 1 800 801 # For backwards compatibility with builds that pre-date 802 # https://chromium-review.googlesource.com/#/c/62880/ 803 # This code can eventually be removed once those builds no longer need 804 # test_that support. 805 new_path = 'usr/local/build/autotest' 806 old_path = 'usr/local/autotest' 807 legacy_path = (os.path.exists(os.path.join(sysroot_path, old_path)) 808 and not 809 os.path.exists(os.path.join(sysroot_path, new_path))) 810 if legacy_path: 811 path_ending = old_path 812 else: 813 path_ending = new_path 814 autotest_path = os.path.join(sysroot_path, path_ending) 815 816 site_utils_path = os.path.join(autotest_path, 'site_utils') 817 818 if not os.path.exists(autotest_path): 819 print >> sys.stderr, ('%s does not exist. Have you run ' 820 'build_packages? Or if you are using ' 821 '--autotest-dir, make sure it points to' 822 'a valid autotest directory.' % autotest_path) 823 return 1 824 825 realpath = os.path.realpath(__file__) 826 827 # If we are not running the sysroot version of script, perform 828 # a quickmerge if necessary and then re-execute 829 # the sysroot version of script with the same arguments. 830 if os.path.dirname(realpath) != site_utils_path: 831 return _perform_bootstrap_into_autotest_root( 832 arguments, autotest_path, argv, legacy_path) 833 else: 834 return _perform_run_from_autotest_root( 835 arguments, autotest_path, argv) 836 837 838def _main_for_lab_run(argv, arguments): 839 """ 840 Effective entry point for lab test_that runs. 841 842 @param argv: Script command line arguments. 843 @param arguments: Parsed command line arguments. 844 """ 845 autotest_path = os.path.realpath(os.path.join(os.path.dirname(__file__), 846 '..')) 847 flattened_argv = ' '.join([pipes.quote(item) for item in argv]) 848 command = [os.path.join(autotest_path, 'site_utils', 849 'run_suite.py'), 850 '--board', arguments.board, 851 '--build', arguments.build, 852 '--suite_name', 'test_that_wrapper', 853 '--pool', 'try-bot', 854 '--suite_args', flattened_argv] 855 logging.info('About to start lab suite with command %s.', command) 856 return subprocess.call(command) 857 858 859def main(argv): 860 """ 861 Entry point for test_that script. 862 863 @param argv: arguments list 864 """ 865 arguments = parse_arguments(argv) 866 try: 867 validate_arguments(arguments) 868 except ValueError as err: 869 print >> sys.stderr, ('Invalid arguments. %s' % err.message) 870 return 1 871 872 if arguments.remote == ':lab:': 873 return _main_for_lab_run(argv, arguments) 874 else: 875 return _main_for_local_run(argv, arguments) 876 877 878if __name__ == '__main__': 879 sys.exit(main(sys.argv[1:])) 880