test_that.py revision 6a7c7f16c643ad4a77550bf5eb0cd9f96a94e680
1#!/usr/bin/python 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import errno 8import os 9import re 10import shutil 11import signal 12import stat 13import subprocess 14import sys 15import tempfile 16import threading 17 18import logging 19# Turn the logging level to INFO before importing other autotest 20# code, to avoid having failed import logging messages confuse the 21# test_that user. 22logging.basicConfig(level=logging.INFO) 23 24import common 25from autotest_lib.client.common_lib.cros import dev_server, retry 26from autotest_lib.client.common_lib import logging_manager 27from autotest_lib.server.cros.dynamic_suite import suite 28from autotest_lib.server.cros import provision 29from autotest_lib.server import autoserv_utils 30from autotest_lib.server import server_logging_config 31 32 33try: 34 from chromite.lib import cros_build_lib 35except ImportError: 36 print 'Unable to import chromite.' 37 print 'This script must be either:' 38 print ' - Be run in the chroot.' 39 print ' - (not yet supported) be run after running ' 40 print ' ../utils/build_externals.py' 41 42_autoserv_proc = None 43_sigint_handler_lock = threading.Lock() 44 45_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 46_NO_BOARD = 'ad_hoc_board' 47_NO_BUILD = 'ad_hoc_build' 48 49_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' 50_TEST_KEY_FILENAME = 'testing_rsa' 51_TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 52 'ssh_keys/%s' % _TEST_KEY_FILENAME) 53 54_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' 55 56_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 57 58 59def schedule_local_suite(autotest_path, suite_name, afe, build=_NO_BUILD, 60 board=_NO_BOARD, results_directory=None, 61 no_experimental=False): 62 """ 63 Schedule a suite against a mock afe object, for a local suite run. 64 @param autotest_path: Absolute path to autotest (in sysroot). 65 @param suite_name: Name of suite to schedule. 66 @param afe: afe object to schedule against (typically a directAFE) 67 @param build: Build to schedule suite for. 68 @param board: Board to schedule suite for. 69 @param results_directory: Absolute path of directory to store results in. 70 (results will be stored in subdirectory of this). 71 @param no_experimental: Skip experimental tests when scheduling a suite. 72 @returns: The number of tests scheduled. 73 """ 74 fs_getter = suite.Suite.create_fs_getter(autotest_path) 75 devserver = dev_server.ImageServer('') 76 my_suite = suite.Suite.create_from_name(suite_name, build, board, 77 devserver, fs_getter, afe=afe, ignore_deps=True, 78 results_dir=results_directory) 79 if len(my_suite.tests) == 0: 80 raise ValueError('Suite named %s does not exist, or contains no ' 81 'tests.' % suite_name) 82 # Schedule tests, discard record calls. 83 return my_suite.schedule(lambda x: None, 84 add_experimental=not no_experimental) 85 86 87def schedule_local_test(autotest_path, test_name, afe, build=_NO_BUILD, 88 board=_NO_BOARD, results_directory=None): 89 #temporarily disabling pylint 90 #pylint: disable-msg=C0111 91 """ 92 Schedule an individual test against a mock afe object, for a local run. 93 @param autotest_path: Absolute path to autotest (in sysroot). 94 @param test_name: Name of test to schedule. 95 @param afe: afe object to schedule against (typically a directAFE) 96 @param build: Build to schedule suite for. 97 @param board: Board to schedule suite for. 98 @param results_directory: Absolute path of directory to store results in. 99 (results will be stored in subdirectory of this). 100 @returns: The number of tests scheduled (may be >1 if there are 101 multiple tests with the same name). 102 """ 103 fs_getter = suite.Suite.create_fs_getter(autotest_path) 104 devserver = dev_server.ImageServer('') 105 predicates = [suite.Suite.test_name_equals_predicate(test_name)] 106 suite_name = 'suite_' + test_name 107 my_suite = suite.Suite.create_from_predicates(predicates, build, board, 108 devserver, fs_getter, afe=afe, name=suite_name, ignore_deps=True, 109 results_dir=results_directory) 110 if len(my_suite.tests) == 0: 111 raise ValueError('No tests named %s.' % test_name) 112 # Schedule tests, discard record calls. 113 return my_suite.schedule(lambda x: None) 114 115 116def run_job(job, host, sysroot_autotest_path, results_directory, fast_mode, 117 id_digits=1, ssh_verbosity=0, args=None, pretend=False): 118 """ 119 Shell out to autoserv to run an individual test job. 120 121 @param job: A Job object containing the control file contents and other 122 relevent metadata for this test. 123 @param host: Hostname of DUT to run test against. 124 @param sysroot_autotest_path: Absolute path of autotest directory. 125 @param results_directory: Absolute path of directory to store results in. 126 (results will be stored in subdirectory of this). 127 @param fast_mode: bool to use fast mode (disables slow autotest features). 128 @param id_digits: The minimum number of digits that job ids should be 129 0-padded to when formatting as a string for results 130 directory. 131 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 132 @param args: String that should be passed as args parameter to autoserv, 133 and then ultimitely to test itself. 134 @param pretend: If True, will print out autoserv commands rather than 135 running them. 136 @returns: Absolute path of directory where results were stored. 137 """ 138 with tempfile.NamedTemporaryFile() as temp_file: 139 temp_file.write(job.control_file) 140 temp_file.flush() 141 name_tail = job.name.split('/')[-1] 142 results_directory = os.path.join(results_directory, 143 'results-%0*d-%s' % (id_digits, job.id, 144 name_tail)) 145 extra_args = [temp_file.name] 146 if args: 147 extra_args.extend(['--args', args]) 148 149 command = autoserv_utils.autoserv_run_job_command( 150 os.path.join(sysroot_autotest_path, 'server'), 151 machines=host, job=job, verbose=False, 152 results_directory=results_directory, 153 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 154 extra_args=extra_args, 155 no_console_prefix=True) 156 157 if not pretend: 158 logging.debug('Running autoserv command: %s', ' '.join(command)) 159 global _autoserv_proc 160 _autoserv_proc = subprocess.Popen(command, 161 stdout=subprocess.PIPE, 162 stderr=subprocess.STDOUT) 163 # This incantation forces unbuffered reading from stdout, 164 # so that autoserv output can be displayed to the user 165 # immediately. 166 for message in iter(_autoserv_proc.stdout.readline, b''): 167 logging.info('autoserv| %s', message.strip()) 168 169 _autoserv_proc.wait() 170 _autoserv_proc = None 171 return results_directory 172 else: 173 logging.info('Pretend mode. Would run autoserv command: %s', 174 ' '.join(command)) 175 176 177def setup_local_afe(): 178 """ 179 Setup a local afe database and return a direct_afe object to access it. 180 181 @returns: A autotest_lib.frontend.afe.direct_afe instance. 182 """ 183 # This import statement is delayed until now rather than running at 184 # module load time, because it kicks off a local sqlite :memory: backed 185 # database, and we don't need that unless we are doing a local run. 186 from autotest_lib.frontend import setup_django_lite_environment 187 from autotest_lib.frontend.afe import direct_afe 188 return direct_afe.directAFE() 189 190 191def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 192 build=_NO_BUILD, board=_NO_BOARD, args=None, 193 pretend=False, no_experimental=False, 194 results_directory=None, ssh_verbosity=0): 195 """ 196 @param afe: A direct_afe object used to interact with local afe database. 197 @param autotest_path: Absolute path of sysroot installed autotest. 198 @param tests: List of strings naming tests and suites to run. Suite strings 199 should be formed like "suite:smoke". 200 @param remote: Remote hostname. 201 @param fast_mode: bool to use fast mode (disables slow autotest features). 202 @param build: String specifying build for local run. 203 @param board: String specifyinb board for local run. 204 @param args: String that should be passed as args parameter to autoserv, 205 and then ultimitely to test itself. 206 @param pretend: If True, will print out autoserv commands rather than 207 running them. 208 @param no_experimental: Skip experimental tests when scheduling a suite. 209 @param results_directory: Directory to store results in. Defaults to None, 210 in which case results will be stored in a new 211 subdirectory of /tmp 212 @param ssh_verbosity: SSH verbosity level, passed through to 213 autoserv_utils. 214 """ 215 # Add the testing key to the current ssh agent. 216 if os.environ.has_key('SSH_AGENT_PID'): 217 # Copy the testing key to the results directory and make it NOT 218 # world-readable. Otherwise, ssh-add complains. 219 shutil.copy(_TEST_KEY_PATH, results_directory) 220 key_copy_path = os.path.join(results_directory, _TEST_KEY_FILENAME) 221 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 222 p = subprocess.Popen(['ssh-add', key_copy_path], 223 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 224 p_out, _ = p.communicate() 225 for line in p_out.splitlines(): 226 logging.info(line) 227 else: 228 logging.warning('There appears to be no running ssh-agent. Attempting ' 229 'to continue without running ssh-add, but ssh commands ' 230 'may fail.') 231 232 build_label = afe.create_label(provision.cros_version_to_label(build)) 233 board_label = afe.create_label(board) 234 new_host = afe.create_host(remote) 235 new_host.add_labels([build_label.name, board_label.name]) 236 237 238 # Schedule tests / suites in local afe 239 for test in tests: 240 suitematch = re.match(r'suite:(.*)', test) 241 if suitematch: 242 suitename = suitematch.group(1) 243 logging.info('Scheduling suite %s...', suitename) 244 ntests = schedule_local_suite(autotest_path, suitename, afe, 245 build=build, board=board, 246 results_directory=results_directory, 247 no_experimental=no_experimental) 248 else: 249 logging.info('Scheduling test %s...', test) 250 ntests = schedule_local_test(autotest_path, test, afe, 251 build=build, board=board, 252 results_directory=results_directory) 253 logging.info('... scheduled %s tests.', ntests) 254 255 if not afe.get_jobs(): 256 logging.info('No jobs scheduled. End of local run.') 257 258 last_job_id = afe.get_jobs()[-1].id 259 job_id_digits=len(str(last_job_id)) 260 for job in afe.get_jobs(): 261 run_job(job, remote, autotest_path, results_directory, fast_mode, 262 job_id_digits, ssh_verbosity, args, pretend) 263 264 265def validate_arguments(arguments): 266 """ 267 Validates parsed arguments. 268 269 @param arguments: arguments object, as parsed by ParseArguments 270 @raises: ValueError if arguments were invalid. 271 """ 272 if arguments.build: 273 raise ValueError('-i/--build flag not yet supported.') 274 275 if not arguments.board: 276 raise ValueError('Board autodetection not yet supported. ' 277 '--board required.') 278 279 if arguments.remote == ':lab:': 280 raise ValueError('Running tests in test lab not yet supported.') 281 if arguments.args: 282 raise ValueError('--args flag not supported when running against ' 283 ':lab:') 284 if arguments.pretend: 285 raise ValueError('--pretend flag not supported when running ' 286 'against :lab:') 287 288 if arguments.ssh_verbosity: 289 raise ValueError('--ssh_verbosity flag not supported when running ' 290 'against :lab:') 291 292 293def parse_arguments(argv): 294 """ 295 Parse command line arguments 296 297 @param argv: argument list to parse 298 @returns: parsed arguments. 299 """ 300 parser = argparse.ArgumentParser(description='Run remote tests.') 301 302 parser.add_argument('remote', metavar='REMOTE', 303 help='hostname[:port] for remote device. Specify ' 304 ':lab: to run in test lab, or :vm:PORT_NUMBER to ' 305 'run in vm.') 306 parser.add_argument('tests', nargs='+', metavar='TEST', 307 help='Run given test(s). Use suite:SUITE to specify ' 308 'test suite.') 309 default_board = cros_build_lib.GetDefaultBoard() 310 parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, 311 action='store', 312 help='Board for which the test will run. Default: %s' % 313 (default_board or 'Not configured')) 314 parser.add_argument('-i', '--build', metavar='BUILD', 315 help='Build to test. Device will be reimaged if ' 316 'necessary. Omit flag to skip reimage and test ' 317 'against already installed DUT image.') 318 parser.add_argument('--fast', action='store_true', dest='fast_mode', 319 default=False, 320 help='Enable fast mode. This will cause test_that to ' 321 'skip time consuming steps like sysinfo and ' 322 'collecting crash information.') 323 parser.add_argument('--args', metavar='ARGS', 324 help='Argument string to pass through to test. Only ' 325 'supported for runs against a local DUT.') 326 parser.add_argument('--results_dir', metavar='RESULTS_DIR', 327 help='Instead of storing results in a new subdirectory' 328 ' of /tmp , store results in RESULTS_DIR. If ' 329 'RESULTS_DIR already exists, will attempt to ' 330 'continue using this directory, which may result ' 331 'in test failures due to file collisions.') 332 parser.add_argument('--pretend', action='store_true', default=False, 333 help='Print autoserv commands that would be run, ' 334 'rather than running them.') 335 parser.add_argument('--no-quickmerge', action='store_true', default=False, 336 dest='no_quickmerge', 337 help='Skip the quickmerge step and use the sysroot ' 338 'as it currently is. May result in un-merged ' 339 'source tree changes not being reflected in run.') 340 parser.add_argument('--no-experimental', action='store_true', 341 default=False, dest='no_experimental', 342 help='When scheduling a suite, skip any tests marked ' 343 'as experimental. Applies only to tests scheduled' 344 ' via suite:[SUITE].') 345 parser.add_argument('--whitelist-chrome-crashes', action='store_true', 346 default=False, dest='whitelist_chrome_crashes', 347 help='Ignore chrome crashes when producing test ' 348 'report. This flag gets passed along to the ' 349 'report generation tool.') 350 parser.add_argument('--ssh_verbosity', action='store', type=int, 351 choices=[0, 1, 2, 3], default=0, 352 help='Verbosity level for ssh, between 0 and 3 ' 353 'inclusive.') 354 parser.add_argument('--debug', action='store_true', 355 help='Include DEBUG level messages in stdout. Note: ' 356 'these messages will be included in output log ' 357 'file regardless.') 358 return parser.parse_args(argv) 359 360 361def sigint_handler(signum, stack_frame): 362 #pylint: disable-msg=C0111 363 """Handle SIGINT or SIGTERM to a local test_that run. 364 365 This handler sends a SIGINT to the running autoserv process, 366 if one is running, giving it up to 5 seconds to clean up and exit. After 367 the timeout elapses, autoserv is killed. In either case, after autoserv 368 exits then this process exits with status 1. 369 """ 370 # If multiple signals arrive before handler is unset, ignore duplicates 371 if not _sigint_handler_lock.acquire(False): 372 return 373 try: 374 # Ignore future signals by unsetting handler. 375 signal.signal(signal.SIGINT, signal.SIG_IGN) 376 signal.signal(signal.SIGTERM, signal.SIG_IGN) 377 378 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 379 if _autoserv_proc: 380 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 381 'to %s seconds for cleanup.', 382 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 383 _autoserv_proc.send_signal(signal.SIGINT) 384 timed_out, _ = retry.timeout(_autoserv_proc.wait, 385 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 386 if timed_out: 387 _autoserv_proc.kill() 388 logging.warning('Timed out waiting for autoserv to handle ' 389 'SIGINT. Killed autoserv.') 390 finally: 391 _sigint_handler_lock.release() # this is not really necessary? 392 sys.exit(1) 393 394 395def main(argv): 396 """ 397 Entry point for test_that script. 398 @param argv: arguments list 399 """ 400 401 if not cros_build_lib.IsInsideChroot(): 402 print >> sys.stderr, 'Script must be invoked inside the chroot.' 403 return 1 404 405 arguments = parse_arguments(argv) 406 try: 407 validate_arguments(arguments) 408 except ValueError as err: 409 print >> sys.stderr, ('Invalid arguments. %s' % err.message) 410 return 1 411 412 sysroot_path = os.path.join('/build', arguments.board, '') 413 sysroot_autotest_path = os.path.join(sysroot_path, 'usr', 'local', 414 'autotest', '') 415 sysroot_site_utils_path = os.path.join(sysroot_autotest_path, 416 'site_utils') 417 418 if not os.path.exists(sysroot_path): 419 print >> sys.stderr, ('%s does not exist. Have you run ' 420 'setup_board?' % sysroot_path) 421 return 1 422 if not os.path.exists(sysroot_autotest_path): 423 print >> sys.stderr, ('%s does not exist. Have you run ' 424 'build_packages?' % sysroot_autotest_path) 425 return 1 426 427 # If we are not running the sysroot version of script, perform 428 # a quickmerge if necessary and then re-execute 429 # the sysroot version of script with the same arguments. 430 realpath = os.path.realpath(__file__) 431 if os.path.dirname(realpath) != sysroot_site_utils_path: 432 logging_manager.configure_logging( 433 server_logging_config.ServerLoggingConfig(), 434 use_console=True, 435 verbose=arguments.debug) 436 if arguments.no_quickmerge: 437 logging.info('Skipping quickmerge step as requested.') 438 else: 439 logging.info('Running autotest_quickmerge step.') 440 s = subprocess.Popen([_QUICKMERGE_SCRIPTNAME, 441 '--board='+arguments.board], 442 stdout=subprocess.PIPE, 443 stderr=subprocess.STDOUT) 444 for message in iter(s.stdout.readline, b''): 445 logging.debug('quickmerge| %s', message.strip()) 446 s.wait() 447 448 logging.info('Re-running test_that script in sysroot.') 449 script_command = os.path.join(sysroot_site_utils_path, 450 os.path.basename(realpath)) 451 proc = None 452 def resend_sig(signum, stack_frame): 453 #pylint: disable-msg=C0111 454 if proc: 455 proc.send_signal(signum) 456 signal.signal(signal.SIGINT, resend_sig) 457 signal.signal(signal.SIGTERM, resend_sig) 458 459 proc = subprocess.Popen([script_command] + argv) 460 461 return proc.wait() 462 463 # We are running the sysroot version of the script. 464 # No further levels of bootstrapping that will occur, so 465 # create a results directory and start sending our logging messages 466 # to it. 467 results_directory = arguments.results_dir 468 if results_directory is None: 469 # Create a results_directory as subdir of /tmp 470 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 471 else: 472 # Create results_directory if it does not exist 473 try: 474 os.makedirs(results_directory) 475 except OSError as e: 476 if e.errno != errno.EEXIST: 477 raise 478 479 logging_manager.configure_logging( 480 server_logging_config.ServerLoggingConfig(), 481 results_dir=results_directory, 482 use_console=True, 483 verbose=arguments.debug, 484 debug_log_name='test_that') 485 logging.info('Began logging to %s', results_directory) 486 487 # Hard coded to True temporarily. This will eventually be parsed to false 488 # if we are doing a run in the test lab. 489 local_run = True 490 491 signal.signal(signal.SIGINT, sigint_handler) 492 signal.signal(signal.SIGTERM, sigint_handler) 493 494 if local_run: 495 afe = setup_local_afe() 496 perform_local_run(afe, sysroot_autotest_path, arguments.tests, 497 arguments.remote, arguments.fast_mode, 498 args=arguments.args, 499 pretend=arguments.pretend, 500 no_experimental=arguments.no_experimental, 501 results_directory=results_directory, 502 ssh_verbosity=arguments.ssh_verbosity) 503 if arguments.pretend: 504 logging.info('Finished pretend run. Exiting.') 505 return 0 506 507 test_report_command = [_TEST_REPORT_SCRIPTNAME] 508 if arguments.whitelist_chrome_crashes: 509 test_report_command.append('--whitelist_chrome_crashes') 510 test_report_command.append(results_directory) 511 final_result = subprocess.call(test_report_command) 512 with open(os.path.join(results_directory, 'test_report.log'), 513 'w') as report_log: 514 subprocess.call(test_report_command, stdout=report_log) 515 logging.info('Finished running tests. Results can be found in %s', 516 results_directory) 517 try: 518 os.unlink(_LATEST_RESULTS_DIRECTORY) 519 except OSError: 520 pass 521 os.symlink(results_directory, _LATEST_RESULTS_DIRECTORY) 522 return final_result 523 524 525if __name__ == '__main__': 526 sys.exit(main(sys.argv[1:])) 527