test_that.py revision bebd62154f7e782793bab4092762b31af286272d
1#!/usr/bin/python 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import errno 8import os 9import re 10import shutil 11import signal 12import stat 13import subprocess 14import sys 15import tempfile 16import threading 17 18import logging 19# Turn the logging level to INFO before importing other autotest 20# code, to avoid having failed import logging messages confuse the 21# test_that user. 22logging.basicConfig(level=logging.INFO) 23 24import common 25from autotest_lib.client.common_lib.cros import dev_server, retry 26from autotest_lib.client.common_lib import logging_manager 27from autotest_lib.server.cros.dynamic_suite import suite 28from autotest_lib.server.cros import provision 29from autotest_lib.server import autoserv_utils 30from autotest_lib.server import server_logging_config 31 32 33try: 34 from chromite.lib import cros_build_lib 35except ImportError: 36 print 'Unable to import chromite.' 37 print 'This script must be either:' 38 print ' - Be run in the chroot.' 39 print ' - (not yet supported) be run after running ' 40 print ' ../utils/build_externals.py' 41 42_autoserv_proc = None 43_sigint_handler_lock = threading.Lock() 44 45_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 46_NO_BOARD = 'ad_hoc_board' 47_NO_BUILD = 'ad_hoc_build' 48 49_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' 50_TEST_KEY_FILENAME = 'testing_rsa' 51_TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 52 'ssh_keys/%s' % _TEST_KEY_FILENAME) 53 54_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' 55 56_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 57 58 59def schedule_local_suite(autotest_path, suite_predicate, afe, build=_NO_BUILD, 60 board=_NO_BOARD, results_directory=None, 61 no_experimental=False): 62 """ 63 Schedule a suite against a mock afe object, for a local suite run. 64 @param autotest_path: Absolute path to autotest (in sysroot). 65 @param suite_predicate: callable that takes ControlData objects, and 66 returns True on those that should be in suite 67 @param afe: afe object to schedule against (typically a directAFE) 68 @param build: Build to schedule suite for. 69 @param board: Board to schedule suite for. 70 @param results_directory: Absolute path of directory to store results in. 71 (results will be stored in subdirectory of this). 72 @param no_experimental: Skip experimental tests when scheduling a suite. 73 @returns: The number of tests scheduled. 74 """ 75 fs_getter = suite.Suite.create_fs_getter(autotest_path) 76 devserver = dev_server.ImageServer('') 77 my_suite = suite.Suite.create_from_predicates([suite_predicate], 78 build, board, devserver, fs_getter, afe=afe, ignore_deps=True, 79 results_dir=results_directory) 80 if len(my_suite.tests) == 0: 81 raise ValueError('Suite contained no tests.') 82 # Schedule tests, discard record calls. 83 return my_suite.schedule(lambda x: None, 84 add_experimental=not no_experimental) 85 86 87def run_job(job, host, sysroot_autotest_path, results_directory, fast_mode, 88 id_digits=1, ssh_verbosity=0, ssh_options=None, 89 args=None, pretend=False, 90 autoserv_verbose=False): 91 """ 92 Shell out to autoserv to run an individual test job. 93 94 @param job: A Job object containing the control file contents and other 95 relevent metadata for this test. 96 @param host: Hostname of DUT to run test against. 97 @param sysroot_autotest_path: Absolute path of autotest directory. 98 @param results_directory: Absolute path of directory to store results in. 99 (results will be stored in subdirectory of this). 100 @param fast_mode: bool to use fast mode (disables slow autotest features). 101 @param id_digits: The minimum number of digits that job ids should be 102 0-padded to when formatting as a string for results 103 directory. 104 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 105 @param ssh_options: Additional ssh options to be passed to autoserv_utils 106 @param args: String that should be passed as args parameter to autoserv, 107 and then ultimitely to test itself. 108 @param pretend: If True, will print out autoserv commands rather than 109 running them. 110 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 111 @returns: Absolute path of directory where results were stored. 112 """ 113 with tempfile.NamedTemporaryFile() as temp_file: 114 temp_file.write(job.control_file) 115 temp_file.flush() 116 name_tail = job.name.split('/')[-1] 117 results_directory = os.path.join(results_directory, 118 'results-%0*d-%s' % (id_digits, job.id, 119 name_tail)) 120 extra_args = [temp_file.name] 121 if args: 122 extra_args.extend(['--args', args]) 123 124 command = autoserv_utils.autoserv_run_job_command( 125 os.path.join(sysroot_autotest_path, 'server'), 126 machines=host, job=job, verbose=autoserv_verbose, 127 results_directory=results_directory, 128 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 129 ssh_options=ssh_options, 130 extra_args=extra_args, 131 no_console_prefix=True) 132 133 if not pretend: 134 logging.debug('Running autoserv command: %s', command) 135 global _autoserv_proc 136 _autoserv_proc = subprocess.Popen(command, 137 stdout=subprocess.PIPE, 138 stderr=subprocess.STDOUT) 139 # This incantation forces unbuffered reading from stdout, 140 # so that autoserv output can be displayed to the user 141 # immediately. 142 for message in iter(_autoserv_proc.stdout.readline, b''): 143 logging.info('autoserv| %s', message.strip()) 144 145 _autoserv_proc.wait() 146 _autoserv_proc = None 147 return results_directory 148 else: 149 logging.info('Pretend mode. Would run autoserv command: %s', 150 command) 151 152 153def setup_local_afe(): 154 """ 155 Setup a local afe database and return a direct_afe object to access it. 156 157 @returns: A autotest_lib.frontend.afe.direct_afe instance. 158 """ 159 # This import statement is delayed until now rather than running at 160 # module load time, because it kicks off a local sqlite :memory: backed 161 # database, and we don't need that unless we are doing a local run. 162 from autotest_lib.frontend import setup_django_lite_environment 163 from autotest_lib.frontend.afe import direct_afe 164 return direct_afe.directAFE() 165 166 167def get_predicate_for_test_arg(test): 168 """ 169 Gets a suite predicte function for a given command-line argument. 170 171 @param test: String. An individual TEST command line argument, e.g. 172 'login_CryptohomeMounted' or 'suite:smoke' 173 @returns: A (predicate, string) tuple with the necessary suite 174 predicate, and a description string of the suite that 175 this predicate will produce. 176 """ 177 suitematch = re.match(r'suite:(.*)', test) 178 name_pattern_match = re.match(r'e:(.*)', test) 179 file_pattern_match = re.match(r'f:(.*)', test) 180 if suitematch: 181 suitename = suitematch.group(1) 182 return (suite.Suite.name_in_tag_predicate(suitename), 183 'suite named %s' % suitename) 184 if name_pattern_match: 185 pattern = '^%s$' % name_pattern_match.group(1) 186 return (suite.Suite.test_name_matches_pattern_predicate(pattern), 187 'suite to match name pattern %s' % pattern) 188 if file_pattern_match: 189 pattern = '^%s$' % file_pattern_match.group(1) 190 return (suite.Suite.test_file_matches_pattern_predicate(pattern), 191 'suite to match file name pattern %s' % pattern) 192 return (suite.Suite.test_name_equals_predicate(test), 193 'job named %s' % test) 194 195 196def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 197 build=_NO_BUILD, board=_NO_BOARD, args=None, 198 pretend=False, no_experimental=False, 199 results_directory=None, ssh_verbosity=0, 200 ssh_options=None, 201 autoserv_verbose=False): 202 """ 203 @param afe: A direct_afe object used to interact with local afe database. 204 @param autotest_path: Absolute path of sysroot installed autotest. 205 @param tests: List of strings naming tests and suites to run. Suite strings 206 should be formed like "suite:smoke". 207 @param remote: Remote hostname. 208 @param fast_mode: bool to use fast mode (disables slow autotest features). 209 @param build: String specifying build for local run. 210 @param board: String specifyinb board for local run. 211 @param args: String that should be passed as args parameter to autoserv, 212 and then ultimitely to test itself. 213 @param pretend: If True, will print out autoserv commands rather than 214 running them. 215 @param no_experimental: Skip experimental tests when scheduling a suite. 216 @param results_directory: Directory to store results in. Defaults to None, 217 in which case results will be stored in a new 218 subdirectory of /tmp 219 @param ssh_verbosity: SSH verbosity level, passed through to 220 autoserv_utils. 221 @param ssh_options: Additional ssh options to be passed to autoserv_utils 222 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 223 """ 224 # Add the testing key to the current ssh agent. 225 if os.environ.has_key('SSH_AGENT_PID'): 226 # Copy the testing key to the results directory and make it NOT 227 # world-readable. Otherwise, ssh-add complains. 228 shutil.copy(_TEST_KEY_PATH, results_directory) 229 key_copy_path = os.path.join(results_directory, _TEST_KEY_FILENAME) 230 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 231 p = subprocess.Popen(['ssh-add', key_copy_path], 232 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 233 p_out, _ = p.communicate() 234 for line in p_out.splitlines(): 235 logging.info(line) 236 else: 237 logging.warning('There appears to be no running ssh-agent. Attempting ' 238 'to continue without running ssh-add, but ssh commands ' 239 'may fail.') 240 241 build_label = afe.create_label(provision.cros_version_to_label(build)) 242 board_label = afe.create_label(board) 243 new_host = afe.create_host(remote) 244 new_host.add_labels([build_label.name, board_label.name]) 245 246 247 # Schedule tests / suites in local afe 248 for test in tests: 249 (predicate, description) = get_predicate_for_test_arg(test) 250 logging.info('Scheduling %s...', description) 251 ntests = schedule_local_suite(autotest_path, predicate, afe, 252 build=build, board=board, 253 results_directory=results_directory, 254 no_experimental=no_experimental) 255 logging.info('... scheduled %s job(s).', ntests) 256 257 if not afe.get_jobs(): 258 logging.info('No jobs scheduled. End of local run.') 259 260 last_job_id = afe.get_jobs()[-1].id 261 job_id_digits=len(str(last_job_id)) 262 for job in afe.get_jobs(): 263 run_job(job, remote, autotest_path, results_directory, fast_mode, 264 job_id_digits, ssh_verbosity, ssh_options, args, pretend, 265 autoserv_verbose) 266 267 268def validate_arguments(arguments): 269 """ 270 Validates parsed arguments. 271 272 @param arguments: arguments object, as parsed by ParseArguments 273 @raises: ValueError if arguments were invalid. 274 """ 275 if arguments.build: 276 raise ValueError('-i/--build flag not yet supported.') 277 278 if not arguments.board: 279 raise ValueError('Board autodetection not yet supported. ' 280 '--board required.') 281 282 if arguments.remote == ':lab:': 283 raise ValueError('Running tests in test lab not yet supported.') 284 if arguments.args: 285 raise ValueError('--args flag not supported when running against ' 286 ':lab:') 287 if arguments.pretend: 288 raise ValueError('--pretend flag not supported when running ' 289 'against :lab:') 290 291 if arguments.ssh_verbosity: 292 raise ValueError('--ssh_verbosity flag not supported when running ' 293 'against :lab:') 294 295 296def parse_arguments(argv): 297 """ 298 Parse command line arguments 299 300 @param argv: argument list to parse 301 @returns: parsed arguments. 302 """ 303 parser = argparse.ArgumentParser(description='Run remote tests.') 304 305 parser.add_argument('remote', metavar='REMOTE', 306 help='hostname[:port] for remote device. Specify ' 307 ':lab: to run in test lab, or :vm:PORT_NUMBER to ' 308 'run in vm.') 309 parser.add_argument('tests', nargs='+', metavar='TEST', 310 help='Run given test(s). Use suite:SUITE to specify ' 311 'test suite. Use e:[NAME_PATTERN] to specify a ' 312 'NAME-matching regular expression. Use ' 313 'f:[FILE_PATTERN] to specify a filename matching ' 314 'regular expression. Specified regular ' 315 'expressiosn will be implicitly wrapped in ' 316 '^ and $.') 317 default_board = cros_build_lib.GetDefaultBoard() 318 parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, 319 action='store', 320 help='Board for which the test will run. Default: %s' % 321 (default_board or 'Not configured')) 322 parser.add_argument('-i', '--build', metavar='BUILD', 323 help='Build to test. Device will be reimaged if ' 324 'necessary. Omit flag to skip reimage and test ' 325 'against already installed DUT image.') 326 parser.add_argument('--fast', action='store_true', dest='fast_mode', 327 default=False, 328 help='Enable fast mode. This will cause test_that to ' 329 'skip time consuming steps like sysinfo and ' 330 'collecting crash information.') 331 parser.add_argument('--args', metavar='ARGS', 332 help='Argument string to pass through to test. Only ' 333 'supported for runs against a local DUT.') 334 parser.add_argument('--results_dir', metavar='RESULTS_DIR', 335 help='Instead of storing results in a new subdirectory' 336 ' of /tmp , store results in RESULTS_DIR. If ' 337 'RESULTS_DIR already exists, will attempt to ' 338 'continue using this directory, which may result ' 339 'in test failures due to file collisions.') 340 parser.add_argument('--pretend', action='store_true', default=False, 341 help='Print autoserv commands that would be run, ' 342 'rather than running them.') 343 parser.add_argument('--no-quickmerge', action='store_true', default=False, 344 dest='no_quickmerge', 345 help='Skip the quickmerge step and use the sysroot ' 346 'as it currently is. May result in un-merged ' 347 'source tree changes not being reflected in run.') 348 parser.add_argument('--no-experimental', action='store_true', 349 default=False, dest='no_experimental', 350 help='When scheduling a suite, skip any tests marked ' 351 'as experimental. Applies only to tests scheduled' 352 ' via suite:[SUITE].') 353 parser.add_argument('--whitelist-chrome-crashes', action='store_true', 354 default=False, dest='whitelist_chrome_crashes', 355 help='Ignore chrome crashes when producing test ' 356 'report. This flag gets passed along to the ' 357 'report generation tool.') 358 parser.add_argument('--ssh_verbosity', action='store', type=int, 359 choices=[0, 1, 2, 3], default=0, 360 help='Verbosity level for ssh, between 0 and 3 ' 361 'inclusive.') 362 parser.add_argument('--ssh_options', action='store', default=None, 363 help='A string giving additional options to be ' 364 'added to ssh commands.') 365 parser.add_argument('--debug', action='store_true', 366 help='Include DEBUG level messages in stdout. Note: ' 367 'these messages will be included in output log ' 368 'file regardless. In addition, turn on autoserv ' 369 'verbosity.') 370 return parser.parse_args(argv) 371 372 373def sigint_handler(signum, stack_frame): 374 #pylint: disable-msg=C0111 375 """Handle SIGINT or SIGTERM to a local test_that run. 376 377 This handler sends a SIGINT to the running autoserv process, 378 if one is running, giving it up to 5 seconds to clean up and exit. After 379 the timeout elapses, autoserv is killed. In either case, after autoserv 380 exits then this process exits with status 1. 381 """ 382 # If multiple signals arrive before handler is unset, ignore duplicates 383 if not _sigint_handler_lock.acquire(False): 384 return 385 try: 386 # Ignore future signals by unsetting handler. 387 signal.signal(signal.SIGINT, signal.SIG_IGN) 388 signal.signal(signal.SIGTERM, signal.SIG_IGN) 389 390 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 391 if _autoserv_proc: 392 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 393 'to %s seconds for cleanup.', 394 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 395 _autoserv_proc.send_signal(signal.SIGINT) 396 timed_out, _ = retry.timeout(_autoserv_proc.wait, 397 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 398 if timed_out: 399 _autoserv_proc.kill() 400 logging.warning('Timed out waiting for autoserv to handle ' 401 'SIGINT. Killed autoserv.') 402 finally: 403 _sigint_handler_lock.release() # this is not really necessary? 404 sys.exit(1) 405 406 407def main(argv): 408 """ 409 Entry point for test_that script. 410 @param argv: arguments list 411 """ 412 413 if not cros_build_lib.IsInsideChroot(): 414 print >> sys.stderr, 'Script must be invoked inside the chroot.' 415 return 1 416 417 arguments = parse_arguments(argv) 418 try: 419 validate_arguments(arguments) 420 except ValueError as err: 421 print >> sys.stderr, ('Invalid arguments. %s' % err.message) 422 return 1 423 424 sysroot_path = os.path.join('/build', arguments.board, '') 425 sysroot_autotest_path = os.path.join(sysroot_path, 'usr', 'local', 426 'autotest', '') 427 sysroot_site_utils_path = os.path.join(sysroot_autotest_path, 428 'site_utils') 429 430 if not os.path.exists(sysroot_path): 431 print >> sys.stderr, ('%s does not exist. Have you run ' 432 'setup_board?' % sysroot_path) 433 return 1 434 if not os.path.exists(sysroot_autotest_path): 435 print >> sys.stderr, ('%s does not exist. Have you run ' 436 'build_packages?' % sysroot_autotest_path) 437 return 1 438 439 # If we are not running the sysroot version of script, perform 440 # a quickmerge if necessary and then re-execute 441 # the sysroot version of script with the same arguments. 442 realpath = os.path.realpath(__file__) 443 if os.path.dirname(realpath) != sysroot_site_utils_path: 444 logging_manager.configure_logging( 445 server_logging_config.ServerLoggingConfig(), 446 use_console=True, 447 verbose=arguments.debug) 448 if arguments.no_quickmerge: 449 logging.info('Skipping quickmerge step as requested.') 450 else: 451 logging.info('Running autotest_quickmerge step.') 452 s = subprocess.Popen([_QUICKMERGE_SCRIPTNAME, 453 '--board='+arguments.board], 454 stdout=subprocess.PIPE, 455 stderr=subprocess.STDOUT) 456 for message in iter(s.stdout.readline, b''): 457 logging.debug('quickmerge| %s', message.strip()) 458 s.wait() 459 460 logging.info('Re-running test_that script in sysroot.') 461 script_command = os.path.join(sysroot_site_utils_path, 462 os.path.basename(realpath)) 463 proc = None 464 def resend_sig(signum, stack_frame): 465 #pylint: disable-msg=C0111 466 if proc: 467 proc.send_signal(signum) 468 signal.signal(signal.SIGINT, resend_sig) 469 signal.signal(signal.SIGTERM, resend_sig) 470 471 proc = subprocess.Popen([script_command] + argv) 472 473 return proc.wait() 474 475 # We are running the sysroot version of the script. 476 # No further levels of bootstrapping that will occur, so 477 # create a results directory and start sending our logging messages 478 # to it. 479 results_directory = arguments.results_dir 480 if results_directory is None: 481 # Create a results_directory as subdir of /tmp 482 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 483 else: 484 # Create results_directory if it does not exist 485 try: 486 os.makedirs(results_directory) 487 except OSError as e: 488 if e.errno != errno.EEXIST: 489 raise 490 491 logging_manager.configure_logging( 492 server_logging_config.ServerLoggingConfig(), 493 results_dir=results_directory, 494 use_console=True, 495 verbose=arguments.debug, 496 debug_log_name='test_that') 497 logging.info('Began logging to %s', results_directory) 498 499 logging.debug('test_that command line was: %s', argv) 500 501 # Hard coded to True temporarily. This will eventually be parsed to false 502 # if we are doing a run in the test lab. 503 local_run = True 504 505 signal.signal(signal.SIGINT, sigint_handler) 506 signal.signal(signal.SIGTERM, sigint_handler) 507 508 if local_run: 509 afe = setup_local_afe() 510 perform_local_run(afe, sysroot_autotest_path, arguments.tests, 511 arguments.remote, arguments.fast_mode, 512 args=arguments.args, 513 pretend=arguments.pretend, 514 no_experimental=arguments.no_experimental, 515 results_directory=results_directory, 516 ssh_verbosity=arguments.ssh_verbosity, 517 ssh_options=arguments.ssh_options, 518 autoserv_verbose=arguments.debug) 519 if arguments.pretend: 520 logging.info('Finished pretend run. Exiting.') 521 return 0 522 523 test_report_command = [_TEST_REPORT_SCRIPTNAME] 524 if arguments.whitelist_chrome_crashes: 525 test_report_command.append('--whitelist_chrome_crashes') 526 test_report_command.append(results_directory) 527 final_result = subprocess.call(test_report_command) 528 with open(os.path.join(results_directory, 'test_report.log'), 529 'w') as report_log: 530 subprocess.call(test_report_command, stdout=report_log) 531 logging.info('Finished running tests. Results can be found in %s', 532 results_directory) 533 try: 534 os.unlink(_LATEST_RESULTS_DIRECTORY) 535 except OSError: 536 pass 537 os.symlink(results_directory, _LATEST_RESULTS_DIRECTORY) 538 return final_result 539 540 541if __name__ == '__main__': 542 sys.exit(main(sys.argv[1:])) 543