1# Copyright 2015 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import errno 6import os 7import re 8import shutil 9import signal 10import stat 11import subprocess 12import sys 13import tempfile 14import threading 15 16import logging 17# Turn the logging level to INFO before importing other autotest 18# code, to avoid having failed import logging messages confuse the 19# test_that user. 20logging.basicConfig(level=logging.INFO) 21 22import common 23from autotest_lib.client.common_lib.cros import dev_server, retry 24from autotest_lib.client.common_lib import logging_manager 25from autotest_lib.server.cros.dynamic_suite import suite, constants 26from autotest_lib.server.cros import provision 27from autotest_lib.server.hosts import factory 28from autotest_lib.server import autoserv_utils 29from autotest_lib.server import server_logging_config 30from autotest_lib.server import utils 31 32 33_autoserv_proc = None 34_sigint_handler_lock = threading.Lock() 35 36_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 37NO_BOARD = 'ad_hoc_board' 38NO_BUILD = 'ad_hoc_build' 39_SUITE_REGEX = r'suite:(.*)' 40 41_TEST_KEY_FILENAME = 'testing_rsa' 42TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 43 'ssh_keys/%s' % _TEST_KEY_FILENAME) 44 45_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 46 47 48class TestThatRunError(Exception): 49 """Raised if test_that encounters something unexpected while running.""" 50 51 52class TestThatProvisioningError(Exception): 53 """Raised when it fails to provision the DUT to the requested build.""" 54 55 56def add_common_args(parser): 57 """ 58 Add common arguments for both test_that and test_droid to their parser. 59 60 @param parser: argparse.ArgumentParser object to add arguments to. 61 """ 62 parser.add_argument('tests', nargs='+', metavar='TEST', 63 help='Run given test(s). Use suite:SUITE to specify ' 64 'test suite. Use e:[NAME_PATTERN] to specify a ' 65 'NAME-matching regular expression. Use ' 66 'f:[FILE_PATTERN] to specify a filename matching ' 67 'regular expression. Specified regular ' 68 'expressions will be implicitly wrapped in ' 69 '^ and $.') 70 parser.add_argument('--fast', action='store_true', dest='fast_mode', 71 default=False, 72 help='Enable fast mode. This will cause test_droid ' 73 'to skip time consuming steps like sysinfo and ' 74 'collecting crash information.') 75 parser.add_argument('--args', metavar='ARGS', 76 help='Whitespace separated argument string to pass ' 77 'through to test. Only supported for runs ' 78 'against a local DUT.') 79 parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, 80 help='Instead of storing results in a new subdirectory' 81 ' of /tmp , store results in RESULTS_DIR. If ' 82 'RESULTS_DIR already exists, it will be deleted.') 83 parser.add_argument('--pretend', action='store_true', default=False, 84 help='Print autoserv commands that would be run, ' 85 'rather than running them.') 86 parser.add_argument('--no-experimental', action='store_true', 87 default=False, dest='no_experimental', 88 help='When scheduling a suite, skip any tests marked ' 89 'as experimental. Applies only to tests scheduled' 90 ' via suite:[SUITE].') 91 parser.add_argument('--enforce-deps', action='store_true', 92 default=False, dest='enforce_deps', 93 help='Skip tests whose DEPENDENCIES can not ' 94 'be satisfied.') 95 parser.add_argument('--debug', action='store_true', 96 help='Include DEBUG level messages in stdout. Note: ' 97 'these messages will be included in output log ' 98 'file regardless. In addition, turn on autoserv ' 99 'verbosity.') 100 parser.add_argument('--iterations', action='store', type=int, default=1, 101 help='Number of times to run the tests specified.') 102 parser.add_argument('--ssh_verbosity', action='store', type=int, 103 choices=[0, 1, 2, 3], default=0, 104 help='Verbosity level for ssh, between 0 and 3 ' 105 'inclusive.') 106 parser.add_argument('--ssh_options', action='store', default=None, 107 help='A string giving additional options to be ' 108 'added to ssh commands.') 109 110 111 112def fetch_local_suite(autotest_path, suite_predicate, afe, test_arg, remote, 113 build=NO_BUILD, board=NO_BOARD, 114 results_directory=None, no_experimental=False, 115 ignore_deps=True): 116 """Create a suite from the given suite predicate. 117 118 Satisfaction of dependencies is enforced by Suite.schedule() if 119 ignore_deps is False. Note that this method assumes only one host, 120 i.e. |remote|, was added to afe. Suite.schedule() will not 121 schedule a job if none of the hosts in the afe (in our case, 122 just one host |remote|) has a label that matches a requested 123 test dependency. 124 125 @param autotest_path: Absolute path to autotest (in sysroot or 126 custom autotest directory set by --autotest_dir). 127 @param suite_predicate: callable that takes ControlData objects, and 128 returns True on those that should be in suite 129 @param afe: afe object to schedule against (typically a directAFE) 130 @param test_arg: String. An individual TEST command line argument, e.g. 131 'login_CryptohomeMounted' or 'suite:smoke'. 132 @param remote: String representing the IP of the remote host. 133 @param build: Build to schedule suite for. 134 @param board: Board to schedule suite for. 135 @param results_directory: Absolute path of directory to store results in. 136 (results will be stored in subdirectory of this). 137 @param no_experimental: Skip experimental tests when scheduling a suite. 138 @param ignore_deps: If True, test dependencies will be ignored. 139 140 @returns: A suite.Suite object. 141 142 """ 143 fs_getter = suite.Suite.create_fs_getter(autotest_path) 144 devserver = dev_server.ImageServer('') 145 my_suite = suite.Suite.create_from_predicates([suite_predicate], 146 {provision.CROS_VERSION_PREFIX: build}, 147 constants.BOARD_PREFIX + board, 148 devserver, fs_getter, afe=afe, 149 ignore_deps=ignore_deps, 150 results_dir=results_directory, forgiving_parser=False) 151 if len(my_suite.tests) == 0: 152 (similarity_predicate, similarity_description) = ( 153 get_predicate_for_possible_test_arg(test_arg)) 154 logging.error('No test found, searching for possible tests with %s', 155 similarity_description) 156 possible_tests = suite.Suite.find_possible_tests(fs_getter, 157 similarity_predicate) 158 raise ValueError('Found no tests. Check your suite name, test name, ' 159 'or test matching wildcard.\nDid you mean any of ' 160 'following tests?\n %s' % '\n '.join(possible_tests)) 161 162 if not ignore_deps: 163 # Log tests whose dependencies can't be satisfied. 164 labels = [label.name for label in 165 afe.get_labels(host__hostname=remote)] 166 for test in my_suite.tests: 167 if test.experimental and no_experimental: 168 continue 169 unsatisfiable_deps = set(test.dependencies).difference(labels) 170 if unsatisfiable_deps: 171 logging.warning('%s will be skipped, unsatisfiable ' 172 'test dependencies: %s', test.name, 173 unsatisfiable_deps) 174 return my_suite 175 176 177def _run_autoserv(command, pretend=False): 178 """Run autoserv command. 179 180 Run the autoserv command and wait on it. Log the stdout. 181 Ensure that SIGINT signals are passed along to autoserv. 182 183 @param command: the autoserv command to run. 184 @returns: exit code of the command. 185 186 """ 187 if not pretend: 188 logging.debug('Running autoserv command: %s', command) 189 global _autoserv_proc 190 _autoserv_proc = subprocess.Popen(command, 191 stdout=subprocess.PIPE, 192 stderr=subprocess.STDOUT) 193 # This incantation forces unbuffered reading from stdout, 194 # so that autoserv output can be displayed to the user 195 # immediately. 196 for message in iter(_autoserv_proc.stdout.readline, b''): 197 logging.info('autoserv| %s', message.strip()) 198 199 _autoserv_proc.wait() 200 returncode = _autoserv_proc.returncode 201 _autoserv_proc = None 202 else: 203 logging.info('Pretend mode. Would run autoserv command: %s', 204 command) 205 returncode = 0 206 return returncode 207 208 209def run_provisioning_job(provision_label, host, autotest_path, 210 results_directory, fast_mode, 211 ssh_verbosity=0, ssh_options=None, 212 pretend=False, autoserv_verbose=False): 213 """Shell out to autoserv to run provisioning job. 214 215 @param provision_label: Label to provision the machine to. 216 @param host: Hostname of DUT. 217 @param autotest_path: Absolute path of autotest directory. 218 @param results_directory: Absolute path of directory to store results in. 219 (results will be stored in subdirectory of this). 220 @param fast_mode: bool to use fast mode (disables slow autotest features). 221 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 222 @param ssh_options: Additional ssh options to be passed to autoserv_utils 223 @param pretend: If True, will print out autoserv commands rather than 224 running them. 225 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 226 227 @returns: Absolute path of directory where results were stored. 228 229 """ 230 # TODO(fdeng): When running against a local DUT, autoserv 231 # is still hitting the AFE in the lab. 232 # provision_AutoUpdate checks the current build of DUT by 233 # retrieving build info from AFE. crosbug.com/295178 234 results_directory = os.path.join(results_directory, 'results-provision') 235 command = autoserv_utils.autoserv_run_job_command( 236 os.path.join(autotest_path, 'server'), 237 machines=host, job=None, verbose=autoserv_verbose, 238 results_directory=results_directory, 239 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 240 ssh_options=ssh_options, 241 extra_args=['--provision', '--job-labels', provision_label], 242 no_console_prefix=True) 243 if _run_autoserv(command, pretend) != 0: 244 raise TestThatProvisioningError('Command returns non-zero code: %s ' % 245 command) 246 return results_directory 247 248 249def run_job(job, host, autotest_path, results_directory, fast_mode, 250 id_digits=1, ssh_verbosity=0, ssh_options=None, 251 args=None, pretend=False, 252 autoserv_verbose=False, host_attributes={}): 253 """ 254 Shell out to autoserv to run an individual test job. 255 256 @param job: A Job object containing the control file contents and other 257 relevent metadata for this test. 258 @param host: Hostname of DUT to run test against. 259 @param autotest_path: Absolute path of autotest directory. 260 @param results_directory: Absolute path of directory to store results in. 261 (results will be stored in subdirectory of this). 262 @param fast_mode: bool to use fast mode (disables slow autotest features). 263 @param id_digits: The minimum number of digits that job ids should be 264 0-padded to when formatting as a string for results 265 directory. 266 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 267 @param ssh_options: Additional ssh options to be passed to autoserv_utils 268 @param args: String that should be passed as args parameter to autoserv, 269 and then ultimitely to test itself. 270 @param pretend: If True, will print out autoserv commands rather than 271 running them. 272 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 273 @param host_attributes: Dict of host attributes to pass into autoserv. 274 275 @returns: a tuple, return code of the job and absolute path of directory 276 where results were stored. 277 """ 278 with tempfile.NamedTemporaryFile() as temp_file: 279 temp_file.write(job.control_file) 280 temp_file.flush() 281 name_tail = job.name.split('/')[-1] 282 results_directory = os.path.join(results_directory, 283 'results-%0*d-%s' % (id_digits, job.id, 284 name_tail)) 285 # Drop experimental keyval in the keval file in the job result folder. 286 os.makedirs(results_directory) 287 utils.write_keyval(results_directory, 288 {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ 289 constants.JOB_EXPERIMENTAL_KEY]}) 290 extra_args = [temp_file.name] 291 if args: 292 extra_args.extend(['--args', args]) 293 294 command = autoserv_utils.autoserv_run_job_command( 295 os.path.join(autotest_path, 'server'), 296 machines=host, job=job, verbose=autoserv_verbose, 297 results_directory=results_directory, 298 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 299 ssh_options=ssh_options, 300 extra_args=extra_args, 301 no_console_prefix=True, 302 use_packaging=False, 303 host_attributes=host_attributes) 304 305 code = _run_autoserv(command, pretend) 306 return code, results_directory 307 308 309def setup_local_afe(): 310 """ 311 Setup a local afe database and return a direct_afe object to access it. 312 313 @returns: A autotest_lib.frontend.afe.direct_afe instance. 314 """ 315 # This import statement is delayed until now rather than running at 316 # module load time, because it kicks off a local sqlite :memory: backed 317 # database, and we don't need that unless we are doing a local run. 318 from autotest_lib.frontend import setup_django_lite_environment 319 from autotest_lib.frontend.afe import direct_afe 320 return direct_afe.directAFE() 321 322 323def get_predicate_for_test_arg(test): 324 """ 325 Gets a suite predicte function for a given command-line argument. 326 327 @param test: String. An individual TEST command line argument, e.g. 328 'login_CryptohomeMounted' or 'suite:smoke' 329 @returns: A (predicate, string) tuple with the necessary suite 330 predicate, and a description string of the suite that 331 this predicate will produce. 332 """ 333 suitematch = re.match(_SUITE_REGEX, test) 334 name_pattern_match = re.match(r'e:(.*)', test) 335 file_pattern_match = re.match(r'f:(.*)', test) 336 if suitematch: 337 suitename = suitematch.group(1) 338 return (suite.Suite.name_in_tag_predicate(suitename), 339 'suite named %s' % suitename) 340 if name_pattern_match: 341 pattern = '^%s$' % name_pattern_match.group(1) 342 return (suite.Suite.test_name_matches_pattern_predicate(pattern), 343 'suite to match name pattern %s' % pattern) 344 if file_pattern_match: 345 pattern = '^%s$' % file_pattern_match.group(1) 346 return (suite.Suite.test_file_matches_pattern_predicate(pattern), 347 'suite to match file name pattern %s' % pattern) 348 return (suite.Suite.test_name_equals_predicate(test), 349 'job named %s' % test) 350 351 352def get_predicate_for_possible_test_arg(test): 353 """ 354 Gets a suite predicte function to calculate the similarity of given test 355 and possible tests. 356 357 @param test: String. An individual TEST command line argument, e.g. 358 'login_CryptohomeMounted' or 'suite:smoke' 359 @returns: A (predicate, string) tuple with the necessary suite 360 predicate, and a description string of the suite that 361 this predicate will produce. 362 """ 363 suitematch = re.match(_SUITE_REGEX, test) 364 name_pattern_match = re.match(r'e:(.*)', test) 365 file_pattern_match = re.match(r'f:(.*)', test) 366 if suitematch: 367 suitename = suitematch.group(1) 368 return (suite.Suite.name_in_tag_similarity_predicate(suitename), 369 'suite name similar to %s' % suitename) 370 if name_pattern_match: 371 pattern = '^%s$' % name_pattern_match.group(1) 372 return (suite.Suite.test_name_similarity_predicate(pattern), 373 'job name similar to %s' % pattern) 374 if file_pattern_match: 375 pattern = '^%s$' % file_pattern_match.group(1) 376 return (suite.Suite.test_file_similarity_predicate(pattern), 377 'suite to match file name similar to %s' % pattern) 378 return (suite.Suite.test_name_similarity_predicate(test), 379 'job name similar to %s' % test) 380 381 382def add_ssh_identity(temp_directory, ssh_private_key=TEST_KEY_PATH): 383 """Add an ssh identity to the agent. 384 385 TODO (sbasi) b/26186193: Add support for test_droid and make TEST_KEY_PATH 386 not Chrome OS specific. 387 388 @param temp_directory: A directory to copy the |private key| into. 389 @param ssh_private_key: Path to the ssh private key to use for testing. 390 """ 391 # Add the testing key to the current ssh agent. 392 if os.environ.has_key('SSH_AGENT_PID'): 393 # Copy the testing key to the temp directory and make it NOT 394 # world-readable. Otherwise, ssh-add complains. 395 shutil.copy(ssh_private_key, temp_directory) 396 key_copy_path = os.path.join(temp_directory, 397 os.path.basename(ssh_private_key)) 398 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 399 p = subprocess.Popen(['ssh-add', key_copy_path], 400 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 401 p_out, _ = p.communicate() 402 for line in p_out.splitlines(): 403 logging.info(line) 404 else: 405 logging.warning('There appears to be no running ssh-agent. Attempting ' 406 'to continue without running ssh-add, but ssh commands ' 407 'may fail.') 408 409 410def _auto_detect_labels(afe, remote): 411 """Automatically detect host labels and add them to the host in afe. 412 413 Note that the label of board will not be auto-detected. 414 This method assumes the host |remote| has already been added to afe. 415 416 @param afe: A direct_afe object used to interact with local afe database. 417 @param remote: The hostname of the remote device. 418 419 """ 420 cros_host = factory.create_host(remote) 421 labels_to_create = [label for label in cros_host.get_labels() 422 if not label.startswith(constants.BOARD_PREFIX)] 423 labels_to_add_to_afe_host = [] 424 for label in labels_to_create: 425 new_label = afe.create_label(label) 426 labels_to_add_to_afe_host.append(new_label.name) 427 hosts = afe.get_hosts(hostname=remote) 428 if not hosts: 429 raise TestThatRunError('Unexpected error: %s has not ' 430 'been added to afe.' % remote) 431 afe_host = hosts[0] 432 afe_host.add_labels(labels_to_add_to_afe_host) 433 434 435def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 436 build=NO_BUILD, board=NO_BOARD, args=None, 437 pretend=False, no_experimental=False, 438 ignore_deps=True, 439 results_directory=None, ssh_verbosity=0, 440 ssh_options=None, 441 autoserv_verbose=False, 442 iterations=1, 443 host_attributes={}): 444 """Perform local run of tests. 445 446 This method enforces satisfaction of test dependencies for tests that are 447 run as a part of a suite. 448 449 @param afe: A direct_afe object used to interact with local afe database. 450 @param autotest_path: Absolute path of autotest installed in sysroot or 451 custom autotest path set by --autotest_dir. 452 @param tests: List of strings naming tests and suites to run. Suite strings 453 should be formed like "suite:smoke". 454 @param remote: Remote hostname. 455 @param fast_mode: bool to use fast mode (disables slow autotest features). 456 @param build: String specifying build for local run. 457 @param board: String specifyinb board for local run. 458 @param args: String that should be passed as args parameter to autoserv, 459 and then ultimitely to test itself. 460 @param pretend: If True, will print out autoserv commands rather than 461 running them. 462 @param no_experimental: Skip experimental tests when scheduling a suite. 463 @param ignore_deps: If True, test dependencies will be ignored. 464 @param results_directory: Directory to store results in. Defaults to None, 465 in which case results will be stored in a new 466 subdirectory of /tmp 467 @param ssh_verbosity: SSH verbosity level, passed through to 468 autoserv_utils. 469 @param ssh_options: Additional ssh options to be passed to autoserv_utils 470 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 471 @param iterations: int number of times to schedule tests. 472 @param host_attributes: Dict of host attributes to pass into autoserv. 473 474 @returns: A list of return codes each job that has run. 475 """ 476 # Create host in afe, add board and build labels. 477 cros_version_label = provision.cros_version_to_label(build) 478 build_label = afe.create_label(cros_version_label) 479 board_label = afe.create_label(constants.BOARD_PREFIX + board) 480 new_host = afe.create_host(remote) 481 new_host.add_labels([build_label.name, board_label.name]) 482 if not ignore_deps: 483 logging.info('Auto-detecting labels for %s', remote) 484 _auto_detect_labels(afe, remote) 485 # Provision the host to |build|. 486 if build != NO_BUILD: 487 logging.info('Provisioning %s...', cros_version_label) 488 try: 489 run_provisioning_job(cros_version_label, remote, autotest_path, 490 results_directory, fast_mode, 491 ssh_verbosity, ssh_options, 492 pretend, autoserv_verbose) 493 except TestThatProvisioningError as e: 494 logging.error('Provisioning %s to %s failed, tests are aborted, ' 495 'failure reason: %s', 496 remote, cros_version_label, e) 497 return 498 499 # Create suites that will be scheduled. 500 suites_and_descriptions = [] 501 for test in tests: 502 (predicate, description) = get_predicate_for_test_arg(test) 503 logging.info('Fetching suite for %s...', description) 504 suite = fetch_local_suite(autotest_path, predicate, afe, test_arg=test, 505 remote=remote, 506 build=build, board=board, 507 results_directory=results_directory, 508 no_experimental=no_experimental, 509 ignore_deps=ignore_deps) 510 suites_and_descriptions.append((suite, description)) 511 512 # Schedule the suites, looping over iterations if necessary. 513 for iteration in range(iterations): 514 if iteration > 0: 515 logging.info('Repeating scheduling for iteration %d:', iteration) 516 517 for suite, description in suites_and_descriptions: 518 logging.info('Scheduling suite for %s...', description) 519 ntests = suite.schedule( 520 lambda log_entry, log_in_subdir=False: None, 521 add_experimental=not no_experimental) 522 logging.info('... scheduled %s job(s).', ntests) 523 524 if not afe.get_jobs(): 525 logging.info('No jobs scheduled. End of local run.') 526 return 527 528 last_job_id = afe.get_jobs()[-1].id 529 job_id_digits = len(str(last_job_id)) 530 codes = [] 531 for job in afe.get_jobs(): 532 code, _ = run_job(job, remote, autotest_path, results_directory, 533 fast_mode, job_id_digits, ssh_verbosity, ssh_options, args, 534 pretend, autoserv_verbose, host_attributes) 535 codes.append(code) 536 return codes 537 538 539def sigint_handler(signum, stack_frame): 540 #pylint: disable-msg=C0111 541 """Handle SIGINT or SIGTERM to a local test_that run. 542 543 This handler sends a SIGINT to the running autoserv process, 544 if one is running, giving it up to 5 seconds to clean up and exit. After 545 the timeout elapses, autoserv is killed. In either case, after autoserv 546 exits then this process exits with status 1. 547 """ 548 # If multiple signals arrive before handler is unset, ignore duplicates 549 if not _sigint_handler_lock.acquire(False): 550 return 551 try: 552 # Ignore future signals by unsetting handler. 553 signal.signal(signal.SIGINT, signal.SIG_IGN) 554 signal.signal(signal.SIGTERM, signal.SIG_IGN) 555 556 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 557 if _autoserv_proc: 558 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 559 'to %s seconds for cleanup.', 560 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 561 _autoserv_proc.send_signal(signal.SIGINT) 562 timed_out, _ = retry.timeout(_autoserv_proc.wait, 563 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 564 if timed_out: 565 _autoserv_proc.kill() 566 logging.warning('Timed out waiting for autoserv to handle ' 567 'SIGINT. Killed autoserv.') 568 finally: 569 _sigint_handler_lock.release() # this is not really necessary? 570 sys.exit(1) 571 572 573def create_results_directory(results_directory=None): 574 """Create a results directory. 575 576 If no directory is specified this method will create and return a 577 temp directory to hold results. If a directory name is specified this 578 method will create a directory at the given path, provided it doesn't 579 already exist. 580 581 @param results_directory: The path to the results_directory to create. 582 583 @return results_directory: A path to the results_directory, ready for use. 584 """ 585 if results_directory is None: 586 # Create a results_directory as subdir of /tmp 587 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 588 else: 589 # Delete results_directory if it already exists. 590 try: 591 shutil.rmtree(results_directory) 592 except OSError as e: 593 if e.errno != errno.ENOENT: 594 raise 595 596 # Create results_directory if it does not exist 597 try: 598 os.makedirs(results_directory) 599 except OSError as e: 600 if e.errno != errno.EEXIST: 601 raise 602 return results_directory 603 604 605def perform_run_from_autotest_root(autotest_path, argv, tests, remote, 606 build=NO_BUILD, board=NO_BOARD, args=None, 607 pretend=False, no_experimental=False, 608 ignore_deps=True, 609 results_directory=None, ssh_verbosity=0, 610 ssh_options=None, 611 iterations=1, fast_mode=False, debug=False, 612 whitelist_chrome_crashes=False, 613 host_attributes={}): 614 """ 615 Perform a test_that run, from the |autotest_path|. 616 617 This function is to be called from test_that/test_droid's main() script, 618 when tests are executed from the |autotest_path|. It handles all stages 619 of a test run that come after the bootstrap into |autotest_path|. 620 621 @param autotest_path: Full absolute path to the autotest root directory. 622 @param argv: The arguments list, as passed to main(...) 623 @param tests: List of strings naming tests and suites to run. Suite strings 624 should be formed like "suite:smoke". 625 @param remote: Remote hostname. 626 @param build: String specifying build for local run. 627 @param board: String specifyinb board for local run. 628 @param args: String that should be passed as args parameter to autoserv, 629 and then ultimitely to test itself. 630 @param pretend: If True, will print out autoserv commands rather than 631 running them. 632 @param no_experimental: Skip experimental tests when scheduling a suite. 633 @param ignore_deps: If True, test dependencies will be ignored. 634 @param results_directory: Directory to store results in. Defaults to None, 635 in which case results will be stored in a new 636 subdirectory of /tmp 637 @param ssh_verbosity: SSH verbosity level, passed through to 638 autoserv_utils. 639 @param ssh_options: Additional ssh options to be passed to autoserv_utils 640 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 641 @param iterations: int number of times to schedule tests. 642 @param fast_mode: bool to use fast mode (disables slow autotest features). 643 @param debug: Logging and autoserv verbosity. 644 @param whitelist_chrome_crashes: If True, whitelist chrome crashes. 645 @param host_attributes: Dict of host attributes to pass into autoserv. 646 647 @returns: A return code that test_that should exit with. 648 """ 649 if results_directory is None or not os.path.exists(results_directory): 650 raise ValueError('Expected valid results directory, got %s' % 651 results_directory) 652 653 logging_manager.configure_logging( 654 server_logging_config.ServerLoggingConfig(), 655 results_dir=results_directory, 656 use_console=True, 657 verbose=debug, 658 debug_log_name='test_that') 659 logging.info('Began logging to %s', results_directory) 660 661 logging.debug('test_that command line was: %s', argv) 662 663 signal.signal(signal.SIGINT, sigint_handler) 664 signal.signal(signal.SIGTERM, sigint_handler) 665 666 afe = setup_local_afe() 667 codes = perform_local_run(afe, autotest_path, tests, remote, fast_mode, 668 build, board, 669 args=args, 670 pretend=pretend, 671 no_experimental=no_experimental, 672 ignore_deps=ignore_deps, 673 results_directory=results_directory, 674 ssh_verbosity=ssh_verbosity, 675 ssh_options=ssh_options, 676 autoserv_verbose=debug, 677 iterations=iterations, 678 host_attributes=host_attributes) 679 if pretend: 680 logging.info('Finished pretend run. Exiting.') 681 return 0 682 683 test_report_command = [os.path.join(os.path.dirname(__file__), 684 'generate_test_report')] 685 # Experimental test results do not influence the exit code. 686 test_report_command.append('--ignore_experimental_tests') 687 if whitelist_chrome_crashes: 688 test_report_command.append('--whitelist_chrome_crashes') 689 test_report_command.append(results_directory) 690 final_result = subprocess.call(test_report_command) 691 with open(os.path.join(results_directory, 'test_report.log'), 692 'w') as report_log: 693 subprocess.call(test_report_command, stdout=report_log) 694 try: 695 os.unlink(_LATEST_RESULTS_DIRECTORY) 696 except OSError: 697 pass 698 link_target = os.path.relpath(results_directory, 699 os.path.dirname(_LATEST_RESULTS_DIRECTORY)) 700 if any(codes): 701 logging.error('Autoserv encountered unexpected errors ' 702 'when executing jobs.') 703 final_result = final_result or 1 704 os.symlink(link_target, _LATEST_RESULTS_DIRECTORY) 705 logging.info('Finished running tests. Results can be found in %s or %s', 706 results_directory, _LATEST_RESULTS_DIRECTORY) 707 return final_result 708