test_that.py revision cbee2dc77cea843b7da34b348bf2a35f375d8a09
1#!/usr/bin/python 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import errno 8import os 9import re 10import shutil 11import signal 12import stat 13import subprocess 14import sys 15import tempfile 16import threading 17 18import logging 19# Turn the logging level to INFO before importing other autotest 20# code, to avoid having failed import logging messages confuse the 21# test_that user. 22logging.basicConfig(level=logging.INFO) 23 24import common 25from autotest_lib.client.common_lib.cros import dev_server, retry 26from autotest_lib.client.common_lib import logging_manager 27from autotest_lib.server.cros.dynamic_suite import suite 28from autotest_lib.server.cros.dynamic_suite import constants 29from autotest_lib.server import autoserv_utils 30from autotest_lib.server import server_logging_config 31 32 33try: 34 from chromite.lib import cros_build_lib 35except ImportError: 36 print 'Unable to import chromite.' 37 print 'This script must be either:' 38 print ' - Be run in the chroot.' 39 print ' - (not yet supported) be run after running ' 40 print ' ../utils/build_externals.py' 41 42_autoserv_proc = None 43_sigint_handler_lock = threading.Lock() 44 45_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 46_NO_BOARD = 'ad_hoc_board' 47_NO_BUILD = 'ad_hoc_build' 48 49_QUICKMERGE_SCRIPTNAME = '/mnt/host/source/chromite/bin/autotest_quickmerge' 50_TEST_KEY_FILENAME = 'testing_rsa' 51_TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 52 'ssh_keys/%s' % _TEST_KEY_FILENAME) 53 54_TEST_REPORT_SCRIPTNAME = '/usr/bin/generate_test_report' 55 56_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 57 58 59def schedule_local_suite(autotest_path, suite_name, afe, build=_NO_BUILD, 60 board=_NO_BOARD, results_directory=None, 61 no_experimental=False): 62 """ 63 Schedule a suite against a mock afe object, for a local suite run. 64 @param autotest_path: Absolute path to autotest (in sysroot). 65 @param suite_name: Name of suite to schedule. 66 @param afe: afe object to schedule against (typically a directAFE) 67 @param build: Build to schedule suite for. 68 @param board: Board to schedule suite for. 69 @param results_directory: Absolute path of directory to store results in. 70 (results will be stored in subdirectory of this). 71 @param no_experimental: Skip experimental tests when scheduling a suite. 72 @returns: The number of tests scheduled. 73 """ 74 fs_getter = suite.Suite.create_fs_getter(autotest_path) 75 devserver = dev_server.ImageServer('') 76 my_suite = suite.Suite.create_from_name(suite_name, build, board, 77 devserver, fs_getter, afe=afe, ignore_deps=True, 78 results_dir=results_directory) 79 if len(my_suite.tests) == 0: 80 raise ValueError('Suite named %s does not exist, or contains no ' 81 'tests.' % suite_name) 82 # Schedule tests, discard record calls. 83 return my_suite.schedule(lambda x: None, 84 add_experimental=not no_experimental) 85 86 87def schedule_local_test(autotest_path, test_name, afe, build=_NO_BUILD, 88 board=_NO_BOARD, results_directory=None): 89 #temporarily disabling pylint 90 #pylint: disable-msg=C0111 91 """ 92 Schedule an individual test against a mock afe object, for a local run. 93 @param autotest_path: Absolute path to autotest (in sysroot). 94 @param test_name: Name of test to schedule. 95 @param afe: afe object to schedule against (typically a directAFE) 96 @param build: Build to schedule suite for. 97 @param board: Board to schedule suite for. 98 @param results_directory: Absolute path of directory to store results in. 99 (results will be stored in subdirectory of this). 100 @returns: The number of tests scheduled (may be >1 if there are 101 multiple tests with the same name). 102 """ 103 fs_getter = suite.Suite.create_fs_getter(autotest_path) 104 devserver = dev_server.ImageServer('') 105 predicates = [suite.Suite.test_name_equals_predicate(test_name)] 106 suite_name = 'suite_' + test_name 107 my_suite = suite.Suite.create_from_predicates(predicates, build, board, 108 devserver, fs_getter, afe=afe, name=suite_name, ignore_deps=True, 109 results_dir=results_directory) 110 if len(my_suite.tests) == 0: 111 raise ValueError('No tests named %s.' % test_name) 112 # Schedule tests, discard record calls. 113 return my_suite.schedule(lambda x: None) 114 115 116def run_job(job, host, sysroot_autotest_path, results_directory, fast_mode, 117 id_digits=1, ssh_verbosity=0, args=None, pretend=False): 118 """ 119 Shell out to autoserv to run an individual test job. 120 121 @param job: A Job object containing the control file contents and other 122 relevent metadata for this test. 123 @param host: Hostname of DUT to run test against. 124 @param sysroot_autotest_path: Absolute path of autotest directory. 125 @param results_directory: Absolute path of directory to store results in. 126 (results will be stored in subdirectory of this). 127 @param fast_mode: bool to use fast mode (disables slow autotest features). 128 @param id_digits: The minimum number of digits that job ids should be 129 0-padded to when formatting as a string for results 130 directory. 131 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 132 @param args: String that should be passed as args parameter to autoserv, 133 and then ultimitely to test itself. 134 @param pretend: If True, will print out autoserv commands rather than 135 running them. 136 @returns: Absolute path of directory where results were stored. 137 """ 138 with tempfile.NamedTemporaryFile() as temp_file: 139 temp_file.write(job.control_file) 140 temp_file.flush() 141 name_tail = job.name.split('/')[-1] 142 results_directory = os.path.join(results_directory, 143 'results-%0*d-%s' % (id_digits, job.id, 144 name_tail)) 145 extra_args = [temp_file.name] 146 if args: 147 extra_args.extend(['--args', args]) 148 149 command = autoserv_utils.autoserv_run_job_command( 150 os.path.join(sysroot_autotest_path, 'server'), 151 machines=host, job=job, verbose=False, 152 results_directory=results_directory, 153 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 154 extra_args=extra_args, 155 no_console_prefix=True) 156 157 if not pretend: 158 logging.debug('Running autoserv command: %s', ' '.join(command)) 159 global _autoserv_proc 160 _autoserv_proc = subprocess.Popen(command, 161 stdout=subprocess.PIPE, 162 stderr=subprocess.STDOUT) 163 # This incantation forces unbuffered reading from stdout, 164 # so that autoserv output can be displayed to the user 165 # immediately. 166 for message in iter(_autoserv_proc.stdout.readline, b''): 167 logging.info('autoserv| %s', message.strip()) 168 169 _autoserv_proc.wait() 170 _autoserv_proc = None 171 return results_directory 172 else: 173 logging.info('Pretend mode. Would run autoserv command: %s', 174 ' '.join(command)) 175 176 177def setup_local_afe(): 178 """ 179 Setup a local afe database and return a direct_afe object to access it. 180 181 @returns: A autotest_lib.frontend.afe.direct_afe instance. 182 """ 183 # This import statement is delayed until now rather than running at 184 # module load time, because it kicks off a local sqlite :memory: backed 185 # database, and we don't need that unless we are doing a local run. 186 from autotest_lib.frontend import setup_django_lite_environment 187 from autotest_lib.frontend.afe import direct_afe 188 return direct_afe.directAFE() 189 190 191def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 192 build=_NO_BUILD, board=_NO_BOARD, args=None, 193 pretend=False, no_experimental=False, 194 results_directory=None, ssh_verbosity=0): 195 """ 196 @param afe: A direct_afe object used to interact with local afe database. 197 @param autotest_path: Absolute path of sysroot installed autotest. 198 @param tests: List of strings naming tests and suites to run. Suite strings 199 should be formed like "suite:smoke". 200 @param remote: Remote hostname. 201 @param fast_mode: bool to use fast mode (disables slow autotest features). 202 @param build: String specifying build for local run. 203 @param board: String specifyinb board for local run. 204 @param args: String that should be passed as args parameter to autoserv, 205 and then ultimitely to test itself. 206 @param pretend: If True, will print out autoserv commands rather than 207 running them. 208 @param no_experimental: Skip experimental tests when scheduling a suite. 209 @param results_directory: Directory to store results in. Defaults to None, 210 in which case results will be stored in a new 211 subdirectory of /tmp 212 @param ssh_verbosity: SSH verbosity level, passed through to 213 autoserv_utils. 214 """ 215 # Add the testing key to the current ssh agent. 216 if os.environ.has_key('SSH_AGENT_PID'): 217 # Copy the testing key to the results directory and make it NOT 218 # world-readable. Otherwise, ssh-add complains. 219 shutil.copy(_TEST_KEY_PATH, results_directory) 220 key_copy_path = os.path.join(results_directory, _TEST_KEY_FILENAME) 221 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 222 p = subprocess.Popen(['ssh-add', key_copy_path], 223 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 224 p_out, _ = p.communicate() 225 for line in p_out.splitlines(): 226 logging.info(line) 227 else: 228 logging.warning('There appears to be no running ssh-agent. Attempting ' 229 'to continue without running ssh-add, but ssh commands ' 230 'may fail.') 231 232 afe.create_label(constants.VERSION_PREFIX + build) 233 afe.create_label(board) 234 afe.create_host(remote) 235 236 # Schedule tests / suites in local afe 237 for test in tests: 238 suitematch = re.match(r'suite:(.*)', test) 239 if suitematch: 240 suitename = suitematch.group(1) 241 logging.info('Scheduling suite %s...', suitename) 242 ntests = schedule_local_suite(autotest_path, suitename, afe, 243 build=build, board=board, 244 results_directory=results_directory, 245 no_experimental=no_experimental) 246 else: 247 logging.info('Scheduling test %s...', test) 248 ntests = schedule_local_test(autotest_path, test, afe, 249 build=build, board=board, 250 results_directory=results_directory) 251 logging.info('... scheduled %s tests.', ntests) 252 253 if not afe.get_jobs(): 254 logging.info('No jobs scheduled. End of local run.') 255 256 last_job_id = afe.get_jobs()[-1].id 257 job_id_digits=len(str(last_job_id)) 258 for job in afe.get_jobs(): 259 run_job(job, remote, autotest_path, results_directory, fast_mode, 260 job_id_digits, ssh_verbosity, args, pretend) 261 262 263def validate_arguments(arguments): 264 """ 265 Validates parsed arguments. 266 267 @param arguments: arguments object, as parsed by ParseArguments 268 @raises: ValueError if arguments were invalid. 269 """ 270 if arguments.build: 271 raise ValueError('-i/--build flag not yet supported.') 272 273 if not arguments.board: 274 raise ValueError('Board autodetection not yet supported. ' 275 '--board required.') 276 277 if arguments.remote == ':lab:': 278 raise ValueError('Running tests in test lab not yet supported.') 279 if arguments.args: 280 raise ValueError('--args flag not supported when running against ' 281 ':lab:') 282 if arguments.pretend: 283 raise ValueError('--pretend flag not supported when running ' 284 'against :lab:') 285 286 if arguments.ssh_verbosity: 287 raise ValueError('--ssh_verbosity flag not supported when running ' 288 'against :lab:') 289 290 291def parse_arguments(argv): 292 """ 293 Parse command line arguments 294 295 @param argv: argument list to parse 296 @returns: parsed arguments. 297 """ 298 parser = argparse.ArgumentParser(description='Run remote tests.') 299 300 parser.add_argument('remote', metavar='REMOTE', 301 help='hostname[:port] for remote device. Specify ' 302 ':lab: to run in test lab, or :vm:PORT_NUMBER to ' 303 'run in vm.') 304 parser.add_argument('tests', nargs='+', metavar='TEST', 305 help='Run given test(s). Use suite:SUITE to specify ' 306 'test suite.') 307 default_board = cros_build_lib.GetDefaultBoard() 308 parser.add_argument('-b', '--board', metavar='BOARD', default=default_board, 309 action='store', 310 help='Board for which the test will run. Default: %s' % 311 (default_board or 'Not configured')) 312 parser.add_argument('-i', '--build', metavar='BUILD', 313 help='Build to test. Device will be reimaged if ' 314 'necessary. Omit flag to skip reimage and test ' 315 'against already installed DUT image.') 316 parser.add_argument('--fast', action='store_true', dest='fast_mode', 317 default=False, 318 help='Enable fast mode. This will cause test_that to ' 319 'skip time consuming steps like sysinfo and ' 320 'collecting crash information.') 321 parser.add_argument('--args', metavar='ARGS', 322 help='Argument string to pass through to test. Only ' 323 'supported for runs against a local DUT.') 324 parser.add_argument('--results_dir', metavar='RESULTS_DIR', 325 help='Instead of storing results in a new subdirectory' 326 ' of /tmp , store results in RESULTS_DIR. If ' 327 'RESULTS_DIR already exists, will attempt to ' 328 'continue using this directory, which may result ' 329 'in test failures due to file collisions.') 330 parser.add_argument('--pretend', action='store_true', default=False, 331 help='Print autoserv commands that would be run, ' 332 'rather than running them.') 333 parser.add_argument('--no-quickmerge', action='store_true', default=False, 334 dest='no_quickmerge', 335 help='Skip the quickmerge step and use the sysroot ' 336 'as it currently is. May result in un-merged ' 337 'source tree changes not being reflected in run.') 338 parser.add_argument('--no-experimental', action='store_true', 339 default=False, dest='no_experimental', 340 help='When scheduling a suite, skip any tests marked ' 341 'as experimental. Applies only to tests scheduled' 342 ' via suite:[SUITE].') 343 parser.add_argument('--whitelist-chrome-crashes', action='store_true', 344 default=False, dest='whitelist_chrome_crashes', 345 help='Ignore chrome crashes when producing test ' 346 'report. This flag gets passed along to the ' 347 'report generation tool.') 348 parser.add_argument('--ssh_verbosity', action='store', type=int, 349 choices=[0, 1, 2, 3], default=0, 350 help='Verbosity level for ssh, between 0 and 3 ' 351 'inclusive.') 352 parser.add_argument('--debug', action='store_true', 353 help='Include DEBUG level messages in stdout. Note: ' 354 'these messages will be included in output log ' 355 'file regardless.') 356 return parser.parse_args(argv) 357 358 359def sigint_handler(signum, stack_frame): 360 #pylint: disable-msg=C0111 361 """Handle SIGINT or SIGTERM to a local test_that run. 362 363 This handler sends a SIGINT to the running autoserv process, 364 if one is running, giving it up to 5 seconds to clean up and exit. After 365 the timeout elapses, autoserv is killed. In either case, after autoserv 366 exits then this process exits with status 1. 367 """ 368 # If multiple signals arrive before handler is unset, ignore duplicates 369 if not _sigint_handler_lock.acquire(False): 370 return 371 try: 372 # Ignore future signals by unsetting handler. 373 signal.signal(signal.SIGINT, signal.SIG_IGN) 374 signal.signal(signal.SIGTERM, signal.SIG_IGN) 375 376 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 377 if _autoserv_proc: 378 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 379 'to %s seconds for cleanup.', 380 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 381 _autoserv_proc.send_signal(signal.SIGINT) 382 timed_out, _ = retry.timeout(_autoserv_proc.wait, 383 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 384 if timed_out: 385 _autoserv_proc.kill() 386 logging.warning('Timed out waiting for autoserv to handle ' 387 'SIGINT. Killed autoserv.') 388 finally: 389 _sigint_handler_lock.release() # this is not really necessary? 390 sys.exit(1) 391 392 393def main(argv): 394 """ 395 Entry point for test_that script. 396 @param argv: arguments list 397 """ 398 399 if not cros_build_lib.IsInsideChroot(): 400 print >> sys.stderr, 'Script must be invoked inside the chroot.' 401 return 1 402 403 arguments = parse_arguments(argv) 404 try: 405 validate_arguments(arguments) 406 except ValueError as err: 407 print >> sys.stderr, ('Invalid arguments. %s' % err.message) 408 return 1 409 410 sysroot_path = os.path.join('/build', arguments.board, '') 411 sysroot_autotest_path = os.path.join(sysroot_path, 'usr', 'local', 412 'autotest', '') 413 sysroot_site_utils_path = os.path.join(sysroot_autotest_path, 414 'site_utils') 415 416 if not os.path.exists(sysroot_path): 417 print >> sys.stderr, ('%s does not exist. Have you run ' 418 'setup_board?' % sysroot_path) 419 return 1 420 if not os.path.exists(sysroot_autotest_path): 421 print >> sys.stderr, ('%s does not exist. Have you run ' 422 'build_packages?' % sysroot_autotest_path) 423 return 1 424 425 # If we are not running the sysroot version of script, perform 426 # a quickmerge if necessary and then re-execute 427 # the sysroot version of script with the same arguments. 428 realpath = os.path.realpath(__file__) 429 if os.path.dirname(realpath) != sysroot_site_utils_path: 430 logging_manager.configure_logging( 431 server_logging_config.ServerLoggingConfig(), 432 use_console=True, 433 verbose=arguments.debug) 434 if arguments.no_quickmerge: 435 logging.info('Skipping quickmerge step as requested.') 436 else: 437 logging.info('Running autotest_quickmerge step.') 438 s = subprocess.Popen([_QUICKMERGE_SCRIPTNAME, 439 '--board='+arguments.board], 440 stdout=subprocess.PIPE, 441 stderr=subprocess.STDOUT) 442 for message in iter(s.stdout.readline, b''): 443 logging.debug('quickmerge| %s', message.strip()) 444 s.wait() 445 446 logging.info('Re-running test_that script in sysroot.') 447 script_command = os.path.join(sysroot_site_utils_path, 448 os.path.basename(realpath)) 449 proc = None 450 def resend_sig(signum, stack_frame): 451 #pylint: disable-msg=C0111 452 if proc: 453 proc.send_signal(signum) 454 signal.signal(signal.SIGINT, resend_sig) 455 signal.signal(signal.SIGTERM, resend_sig) 456 457 proc = subprocess.Popen([script_command] + argv) 458 459 return proc.wait() 460 461 # We are running the sysroot version of the script. 462 # No further levels of bootstrapping that will occur, so 463 # create a results directory and start sending our logging messages 464 # to it. 465 results_directory = arguments.results_dir 466 if results_directory is None: 467 # Create a results_directory as subdir of /tmp 468 results_directory = tempfile.mkdtemp(prefix='test_that_results_') 469 else: 470 # Create results_directory if it does not exist 471 try: 472 os.makedirs(results_directory) 473 except OSError as e: 474 if e.errno != errno.EEXIST: 475 raise 476 477 logging_manager.configure_logging( 478 server_logging_config.ServerLoggingConfig(), 479 results_dir=results_directory, 480 use_console=True, 481 verbose=arguments.debug, 482 debug_log_name='test_that') 483 logging.info('Began logging to %s', results_directory) 484 485 # Hard coded to True temporarily. This will eventually be parsed to false 486 # if we are doing a run in the test lab. 487 local_run = True 488 489 signal.signal(signal.SIGINT, sigint_handler) 490 signal.signal(signal.SIGTERM, sigint_handler) 491 492 if local_run: 493 afe = setup_local_afe() 494 perform_local_run(afe, sysroot_autotest_path, arguments.tests, 495 arguments.remote, arguments.fast_mode, 496 args=arguments.args, 497 pretend=arguments.pretend, 498 no_experimental=arguments.no_experimental, 499 results_directory=results_directory, 500 ssh_verbosity=arguments.ssh_verbosity) 501 if arguments.pretend: 502 logging.info('Finished pretend run. Exiting.') 503 return 0 504 505 test_report_command = [_TEST_REPORT_SCRIPTNAME] 506 if arguments.whitelist_chrome_crashes: 507 test_report_command.append('--whitelist_chrome_crashes') 508 test_report_command.append(results_directory) 509 final_result = subprocess.call(test_report_command) 510 logging.info('Finished running tests. Results can be found in %s', 511 results_directory) 512 try: 513 os.unlink(_LATEST_RESULTS_DIRECTORY) 514 except OSError: 515 pass 516 os.symlink(results_directory, _LATEST_RESULTS_DIRECTORY) 517 return final_result 518 519 520if __name__ == '__main__': 521 sys.exit(main(sys.argv[1:])) 522