autotest.py revision 517d95a1ef4edb04da427763f86068a447d45ec7
1# Copyright 2007 Google Inc. Released under the GPL v2 2 3import re, os, sys, traceback, subprocess, time, pickle, glob, tempfile 4import logging, getpass 5from autotest_lib.server import installable_object, prebuild, utils 6from autotest_lib.client.common_lib import base_job, log, error, autotemp 7from autotest_lib.client.common_lib import global_config, packages 8from autotest_lib.client.common_lib import utils as client_utils 9 10AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' 11AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client' 12 13# Timeouts for powering down and up respectively 14HALT_TIME = 300 15BOOT_TIME = 1800 16CRASH_RECOVERY_TIME = 9000 17 18 19get_value = global_config.global_config.get_config_value 20autoserv_prebuild = get_value('AUTOSERV', 'enable_server_prebuild', 21 type=bool, default=False) 22 23 24class AutodirNotFoundError(Exception): 25 """No Autotest installation could be found.""" 26 27 28class BaseAutotest(installable_object.InstallableObject): 29 """ 30 This class represents the Autotest program. 31 32 Autotest is used to run tests automatically and collect the results. 33 It also supports profilers. 34 35 Implementation details: 36 This is a leaf class in an abstract class hierarchy, it must 37 implement the unimplemented methods in parent classes. 38 """ 39 40 def __init__(self, host = None): 41 self.host = host 42 self.got = False 43 self.installed = False 44 self.serverdir = utils.get_server_dir() 45 super(BaseAutotest, self).__init__() 46 47 48 install_in_tmpdir = False 49 @classmethod 50 def set_install_in_tmpdir(cls, flag): 51 """ Sets a flag that controls whether or not Autotest should by 52 default be installed in a "standard" directory (e.g. 53 /home/autotest, /usr/local/autotest) or a temporary directory. """ 54 cls.install_in_tmpdir = flag 55 56 57 @classmethod 58 def get_client_autodir_paths(cls, host): 59 return global_config.global_config.get_config_value( 60 'AUTOSERV', 'client_autodir_paths', type=list) 61 62 63 @classmethod 64 def get_installed_autodir(cls, host): 65 """ 66 Find where the Autotest client is installed on the host. 67 @returns an absolute path to an installed Autotest client root. 68 @raises AutodirNotFoundError if no Autotest installation can be found. 69 """ 70 autodir = host.get_autodir() 71 if autodir: 72 logging.debug('Using existing host autodir: %s', autodir) 73 return autodir 74 75 for path in Autotest.get_client_autodir_paths(host): 76 try: 77 autotest_binary = os.path.join(path, 'bin', 'autotest') 78 host.run('test -x %s' % utils.sh_escape(autotest_binary)) 79 logging.debug('Found existing autodir at %s', path) 80 return path 81 except error.AutoservRunError: 82 logging.debug('%s does not exist on %s', autotest_binary, 83 host.hostname) 84 raise AutodirNotFoundError 85 86 87 @classmethod 88 def get_install_dir(cls, host): 89 """ 90 Determines the location where autotest should be installed on 91 host. If self.install_in_tmpdir is set, it will return a unique 92 temporary directory that autotest can be installed in. Otherwise, looks 93 for an existing installation to use; if none is found, looks for a 94 usable directory in the global config client_autodir_paths. 95 """ 96 try: 97 install_dir = cls.get_installed_autodir(host) 98 except AutodirNotFoundError: 99 install_dir = cls._find_installable_dir(host) 100 101 if cls.install_in_tmpdir: 102 return host.get_tmp_dir(parent=install_dir) 103 return install_dir 104 105 106 @classmethod 107 def _find_installable_dir(cls, host): 108 client_autodir_paths = cls.get_client_autodir_paths(host) 109 for path in client_autodir_paths: 110 try: 111 host.run('mkdir -p %s' % utils.sh_escape(path)) 112 return path 113 except error.AutoservRunError: 114 logging.debug('Failed to create %s', path) 115 raise error.AutoservInstallError( 116 'Unable to find a place to install Autotest; tried %s', 117 ', '.join(client_autodir_paths)) 118 119 120 def install(self, host=None, autodir=None): 121 self._install(host=host, autodir=autodir) 122 123 124 def install_full_client(self, host=None, autodir=None): 125 self._install(host=host, autodir=autodir, use_autoserv=False, 126 use_packaging=False) 127 128 129 def install_no_autoserv(self, host=None, autodir=None): 130 self._install(host=host, autodir=autodir, use_autoserv=False) 131 132 133 def _install_using_packaging(self, host, autodir): 134 c = global_config.global_config 135 repos = c.get_config_value("PACKAGES", 'fetch_location', type=list, 136 default=[]) 137 repos.reverse() 138 if not repos: 139 raise error.PackageInstallError("No repos to install an " 140 "autotest client from") 141 pkgmgr = packages.PackageManager(autodir, hostname=host.hostname, 142 repo_urls=repos, 143 do_locking=False, 144 run_function=host.run, 145 run_function_dargs=dict(timeout=600)) 146 # The packages dir is used to store all the packages that 147 # are fetched on that client. (for the tests,deps etc. 148 # too apart from the client) 149 pkg_dir = os.path.join(autodir, 'packages') 150 # clean up the autodir except for the packages directory 151 host.run('cd %s && ls | grep -v "^packages$"' 152 ' | xargs rm -rf && rm -rf .[^.]*' % autodir) 153 pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir, 154 preserve_install_dir=True) 155 self.installed = True 156 157 158 def _install_using_send_file(self, host, autodir): 159 dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"]) 160 light_files = [os.path.join(self.source_material, f) 161 for f in os.listdir(self.source_material) 162 if f not in dirs_to_exclude] 163 host.send_file(light_files, autodir, delete_dest=True) 164 165 # create empty dirs for all the stuff we excluded 166 commands = [] 167 for path in dirs_to_exclude: 168 abs_path = os.path.join(autodir, path) 169 abs_path = utils.sh_escape(abs_path) 170 commands.append("mkdir -p '%s'" % abs_path) 171 commands.append("touch '%s'/__init__.py" % abs_path) 172 host.run(';'.join(commands)) 173 174 175 def _install(self, host=None, autodir=None, use_autoserv=True, 176 use_packaging=True): 177 """ 178 Install autotest. If get() was not called previously, an 179 attempt will be made to install from the autotest svn 180 repository. 181 182 @param host A Host instance on which autotest will be installed 183 @param autodir Location on the remote host to install to 184 @param use_autoserv Enable install modes that depend on the client 185 running with the autoserv harness 186 @param use_packaging Enable install modes that use the packaging system 187 188 @exception AutoservError if a tarball was not specified and 189 the target host does not have svn installed in its path 190 """ 191 if not host: 192 host = self.host 193 if not self.got: 194 self.get() 195 host.wait_up(timeout=30) 196 host.setup() 197 logging.info("Installing autotest on %s", host.hostname) 198 199 # set up the autotest directory on the remote machine 200 if not autodir: 201 autodir = self.get_install_dir(host) 202 logging.info('Using installation dir %s', autodir) 203 host.set_autodir(autodir) 204 host.run('mkdir -p %s' % utils.sh_escape(autodir)) 205 206 # make sure there are no files in $AUTODIR/results 207 results_path = os.path.join(autodir, 'results') 208 host.run('rm -rf %s/*' % utils.sh_escape(results_path), 209 ignore_status=True) 210 211 # Fetch the autotest client from the nearest repository 212 if use_packaging: 213 try: 214 self._install_using_packaging(host, autodir) 215 return 216 except global_config.ConfigError, e: 217 logging.info("Could not install autotest using the packaging " 218 "system: %s", e) 219 except (error.PackageInstallError, error.AutoservRunError), e: 220 logging.error("Could not install autotest from repos") 221 222 # try to install from file or directory 223 if self.source_material: 224 c = global_config.global_config 225 supports_autoserv_packaging = c.get_config_value( 226 "PACKAGES", "serve_packages_from_autoserv", type=bool) 227 # Copy autotest recursively 228 if supports_autoserv_packaging and use_autoserv: 229 self._install_using_send_file(host, autodir) 230 else: 231 host.send_file(self.source_material, autodir, delete_dest=True) 232 logging.info("Installation of autotest completed") 233 self.installed = True 234 return 235 236 # if that fails try to install using svn 237 if utils.run('which svn').exit_status: 238 raise error.AutoservError('svn not found on target machine: %s' 239 % host.name) 240 try: 241 host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir)) 242 except error.AutoservRunError, e: 243 host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir)) 244 logging.info("Installation of autotest completed") 245 self.installed = True 246 247 248 def uninstall(self, host=None): 249 """ 250 Uninstall (i.e. delete) autotest. Removes the autotest client install 251 from the specified host. 252 253 @params host a Host instance from which the client will be removed 254 """ 255 if not self.installed: 256 return 257 if not host: 258 host = self.host 259 autodir = host.get_autodir() 260 if not autodir: 261 return 262 263 # perform the actual uninstall 264 host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True) 265 host.set_autodir(None) 266 self.installed = False 267 268 269 def get(self, location = None): 270 if not location: 271 location = os.path.join(self.serverdir, '../client') 272 location = os.path.abspath(location) 273 # If there's stuff run on our client directory already, it 274 # can cause problems. Try giving it a quick clean first. 275 cwd = os.getcwd() 276 os.chdir(location) 277 try: 278 utils.system('tools/make_clean', ignore_status=True) 279 finally: 280 os.chdir(cwd) 281 super(BaseAutotest, self).get(location) 282 self.got = True 283 284 285 def run(self, control_file, results_dir='.', host=None, timeout=None, 286 tag=None, parallel_flag=False, background=False, 287 client_disconnect_timeout=1800): 288 """ 289 Run an autotest job on the remote machine. 290 291 @param control_file: An open file-like-obj of the control file. 292 @param results_dir: A str path where the results should be stored 293 on the local filesystem. 294 @param host: A Host instance on which the control file should 295 be run. 296 @param timeout: Maximum number of seconds to wait for the run or None. 297 @param tag: Tag name for the client side instance of autotest. 298 @param parallel_flag: Flag set when multiple jobs are run at the 299 same time. 300 @param background: Indicates that the client should be launched as 301 a background job; the code calling run will be responsible 302 for monitoring the client and collecting the results. 303 @param client_disconnect_timeout: Seconds to wait for the remote host 304 to come back after a reboot. [default: 30 minutes] 305 306 @raises AutotestRunError: If there is a problem executing 307 the control file. 308 """ 309 host = self._get_host_and_setup(host) 310 results_dir = os.path.abspath(results_dir) 311 312 if tag: 313 results_dir = os.path.join(results_dir, tag) 314 315 atrun = _Run(host, results_dir, tag, parallel_flag, background) 316 self._do_run(control_file, results_dir, host, atrun, timeout, 317 client_disconnect_timeout) 318 319 320 def _get_host_and_setup(self, host): 321 if not host: 322 host = self.host 323 if not self.installed: 324 self.install(host) 325 326 host.wait_up(timeout=30) 327 return host 328 329 330 def _do_run(self, control_file, results_dir, host, atrun, timeout, 331 client_disconnect_timeout): 332 try: 333 atrun.verify_machine() 334 except: 335 logging.error("Verify failed on %s. Reinstalling autotest", 336 host.hostname) 337 self.install(host) 338 atrun.verify_machine() 339 debug = os.path.join(results_dir, 'debug') 340 try: 341 os.makedirs(debug) 342 except Exception: 343 pass 344 345 delete_file_list = [atrun.remote_control_file, 346 atrun.remote_control_file + '.state', 347 atrun.manual_control_file, 348 atrun.manual_control_file + '.state'] 349 cmd = ';'.join('rm -f ' + control for control in delete_file_list) 350 host.run(cmd, ignore_status=True) 351 352 tmppath = utils.get(control_file) 353 354 # build up the initialization prologue for the control file 355 prologue_lines = [] 356 357 # Add the additional user arguments 358 prologue_lines.append("args = %r\n" % self.job.args) 359 360 # If the packaging system is being used, add the repository list. 361 repos = None 362 try: 363 c = global_config.global_config 364 repos = c.get_config_value("PACKAGES", 'fetch_location', type=list) 365 repos.reverse() # high priority packages should be added last 366 pkgmgr = packages.PackageManager('autotest', hostname=host.hostname, 367 repo_urls=repos) 368 prologue_lines.append('job.add_repository(%s)\n' % repos) 369 except global_config.ConfigError, e: 370 # If repos is defined packaging is enabled so log the error 371 if repos: 372 logging.error(e) 373 374 # on full-size installs, turn on any profilers the server is using 375 if not atrun.background: 376 running_profilers = host.job.profilers.add_log.iteritems() 377 for profiler, (args, dargs) in running_profilers: 378 call_args = [repr(profiler)] 379 call_args += [repr(arg) for arg in args] 380 call_args += ["%s=%r" % item for item in dargs.iteritems()] 381 prologue_lines.append("job.profilers.add(%s)\n" 382 % ", ".join(call_args)) 383 cfile = "".join(prologue_lines) 384 385 cfile += open(tmppath).read() 386 open(tmppath, "w").write(cfile) 387 388 # Create and copy state file to remote_control_file + '.state' 389 state_file = host.job.preprocess_client_state() 390 host.send_file(state_file, atrun.remote_control_file + '.init.state') 391 os.remove(state_file) 392 393 # Copy control_file to remote_control_file on the host 394 host.send_file(tmppath, atrun.remote_control_file) 395 if os.path.abspath(tmppath) != os.path.abspath(control_file): 396 os.remove(tmppath) 397 398 atrun.execute_control( 399 timeout=timeout, 400 client_disconnect_timeout=client_disconnect_timeout) 401 402 403 def run_timed_test(self, test_name, results_dir='.', host=None, 404 timeout=None, *args, **dargs): 405 """ 406 Assemble a tiny little control file to just run one test, 407 and run it as an autotest client-side test 408 """ 409 if not host: 410 host = self.host 411 if not self.installed: 412 self.install(host) 413 opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()] 414 cmd = ", ".join([repr(test_name)] + map(repr, args) + opts) 415 control = "job.run_test(%s)\n" % cmd 416 self.run(control, results_dir, host, timeout=timeout) 417 418 419 def run_test(self, test_name, results_dir='.', host=None, *args, **dargs): 420 self.run_timed_test(test_name, results_dir, host, timeout=None, 421 *args, **dargs) 422 423 424class _BaseRun(object): 425 """ 426 Represents a run of autotest control file. This class maintains 427 all the state necessary as an autotest control file is executed. 428 429 It is not intended to be used directly, rather control files 430 should be run using the run method in Autotest. 431 """ 432 def __init__(self, host, results_dir, tag, parallel_flag, background): 433 self.host = host 434 self.results_dir = results_dir 435 self.env = host.env 436 self.tag = tag 437 self.parallel_flag = parallel_flag 438 self.background = background 439 self.autodir = Autotest.get_installed_autodir(self.host) 440 control = os.path.join(self.autodir, 'control') 441 if tag: 442 control += '.' + tag 443 self.manual_control_file = control 444 self.remote_control_file = control + '.autoserv' 445 self.config_file = os.path.join(self.autodir, 'global_config.ini') 446 447 448 def verify_machine(self): 449 binary = os.path.join(self.autodir, 'bin/autotest') 450 try: 451 self.host.run('ls %s > /dev/null 2>&1' % binary) 452 except: 453 raise error.AutoservInstallError( 454 "Autotest does not appear to be installed") 455 456 if not self.parallel_flag: 457 tmpdir = os.path.join(self.autodir, 'tmp') 458 download = os.path.join(self.autodir, 'tests/download') 459 self.host.run('umount %s' % tmpdir, ignore_status=True) 460 self.host.run('umount %s' % download, ignore_status=True) 461 462 463 def get_base_cmd_args(self, section): 464 args = ['--verbose'] 465 if section > 0: 466 args.append('-c') 467 if self.tag: 468 args.append('-t %s' % self.tag) 469 if self.host.job.use_external_logging(): 470 args.append('-l') 471 if self.host.hostname: 472 args.append('--hostname=%s' % self.host.hostname) 473 args.append('--user=%s' % self.host.job.user) 474 475 args.append(self.remote_control_file) 476 return args 477 478 479 def get_background_cmd(self, section): 480 cmd = ['nohup', os.path.join(self.autodir, 'bin/autotest_client')] 481 cmd += self.get_base_cmd_args(section) 482 cmd += ['>/dev/null', '2>/dev/null', '&'] 483 return ' '.join(cmd) 484 485 486 def get_daemon_cmd(self, section, monitor_dir): 487 cmd = ['nohup', os.path.join(self.autodir, 'bin/autotestd'), 488 monitor_dir, '-H autoserv'] 489 cmd += self.get_base_cmd_args(section) 490 cmd += ['>/dev/null', '2>/dev/null', '&'] 491 return ' '.join(cmd) 492 493 494 def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read): 495 cmd = [os.path.join(self.autodir, 'bin', 'autotestd_monitor'), 496 monitor_dir, str(stdout_read), str(stderr_read)] 497 return ' '.join(cmd) 498 499 500 def get_client_log(self): 501 """Find what the "next" client.* prefix should be 502 503 @returns A string of the form client.INTEGER that should be prefixed 504 to all client debug log files. 505 """ 506 max_digit = -1 507 debug_dir = os.path.join(self.results_dir, 'debug') 508 client_logs = glob.glob(os.path.join(debug_dir, 'client.*.*')) 509 for log in client_logs: 510 _, number, _ = log.split('.', 2) 511 if number.isdigit(): 512 max_digit = max(max_digit, int(number)) 513 return 'client.%d' % (max_digit + 1) 514 515 516 def copy_client_config_file(self, client_log_prefix=None): 517 """ 518 Create and copy the client config file based on the server config. 519 520 @param client_log_prefix: Optional prefix to prepend to log files. 521 """ 522 client_config_file = self._create_client_config_file(client_log_prefix) 523 self.host.send_file(client_config_file, self.config_file) 524 os.remove(client_config_file) 525 526 527 def _create_client_config_file(self, client_log_prefix=None): 528 """ 529 Create a temporary file with the [CLIENT] section configuration values 530 taken from the server global_config.ini. 531 532 @param client_log_prefix: Optional prefix to prepend to log files. 533 534 @return: Path of the temporary file generated. 535 """ 536 config = global_config.global_config.get_section_values('CLIENT') 537 if client_log_prefix: 538 config.set('CLIENT', 'default_logging_name', client_log_prefix) 539 return self._create_aux_file(config.write) 540 541 542 def _create_aux_file(self, func, *args): 543 """ 544 Creates a temporary file and writes content to it according to a 545 content creation function. The file object is appended to *args, which 546 is then passed to the content creation function 547 548 @param func: Function that will be used to write content to the 549 temporary file. 550 @param *args: List of parameters that func takes. 551 @return: Path to the temporary file that was created. 552 """ 553 fd, path = tempfile.mkstemp(dir=self.host.job.tmpdir) 554 aux_file = os.fdopen(fd, "w") 555 try: 556 list_args = list(args) 557 list_args.append(aux_file) 558 func(*list_args) 559 finally: 560 aux_file.close() 561 return path 562 563 564 @staticmethod 565 def is_client_job_finished(last_line): 566 return bool(re.match(r'^END .*\t----\t----\t.*$', last_line)) 567 568 569 @staticmethod 570 def is_client_job_rebooting(last_line): 571 return bool(re.match(r'^\t*GOOD\t----\treboot\.start.*$', last_line)) 572 573 574 def log_unexpected_abort(self, stderr_redirector): 575 stderr_redirector.flush_all_buffers() 576 msg = "Autotest client terminated unexpectedly" 577 self.host.job.record("END ABORT", None, None, msg) 578 579 580 def _execute_in_background(self, section, timeout): 581 full_cmd = self.get_background_cmd(section) 582 devnull = open(os.devnull, "w") 583 584 self.copy_client_config_file(self.get_client_log()) 585 586 self.host.job.push_execution_context(self.results_dir) 587 try: 588 result = self.host.run(full_cmd, ignore_status=True, 589 timeout=timeout, 590 stdout_tee=devnull, 591 stderr_tee=devnull) 592 finally: 593 self.host.job.pop_execution_context() 594 595 return result 596 597 598 @staticmethod 599 def _strip_stderr_prologue(stderr): 600 """Strips the 'standard' prologue that get pre-pended to every 601 remote command and returns the text that was actually written to 602 stderr by the remote command.""" 603 stderr_lines = stderr.split("\n")[1:] 604 if not stderr_lines: 605 return "" 606 elif stderr_lines[0].startswith("NOTE: autotestd_monitor"): 607 del stderr_lines[0] 608 return "\n".join(stderr_lines) 609 610 611 def _execute_daemon(self, section, timeout, stderr_redirector, 612 client_disconnect_timeout): 613 monitor_dir = self.host.get_tmp_dir() 614 daemon_cmd = self.get_daemon_cmd(section, monitor_dir) 615 616 # grab the location for the server-side client log file 617 client_log_prefix = self.get_client_log() 618 client_log_path = os.path.join(self.results_dir, 'debug', 619 client_log_prefix + '.log') 620 client_log = open(client_log_path, 'w', 0) 621 self.copy_client_config_file(client_log_prefix) 622 623 stdout_read = stderr_read = 0 624 self.host.job.push_execution_context(self.results_dir) 625 try: 626 self.host.run(daemon_cmd, ignore_status=True, timeout=timeout) 627 disconnect_warnings = [] 628 while True: 629 monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read, 630 stderr_read) 631 try: 632 result = self.host.run(monitor_cmd, ignore_status=True, 633 timeout=timeout, 634 stdout_tee=client_log, 635 stderr_tee=stderr_redirector) 636 except error.AutoservRunError, e: 637 result = e.result_obj 638 result.exit_status = None 639 disconnect_warnings.append(e.description) 640 641 stderr_redirector.log_warning( 642 "Autotest client was disconnected: %s" % e.description, 643 "NETWORK") 644 except error.AutoservSSHTimeout: 645 result = utils.CmdResult(monitor_cmd, "", "", None, 0) 646 stderr_redirector.log_warning( 647 "Attempt to connect to Autotest client timed out", 648 "NETWORK") 649 650 stdout_read += len(result.stdout) 651 stderr_read += len(self._strip_stderr_prologue(result.stderr)) 652 653 if result.exit_status is not None: 654 return result 655 elif not self.host.wait_up(client_disconnect_timeout): 656 raise error.AutoservSSHTimeout( 657 "client was disconnected, reconnect timed out") 658 finally: 659 client_log.close() 660 self.host.job.pop_execution_context() 661 662 663 def execute_section(self, section, timeout, stderr_redirector, 664 client_disconnect_timeout): 665 logging.info("Executing %s/bin/autotest %s/control phase %d", 666 self.autodir, self.autodir, section) 667 668 if self.background: 669 result = self._execute_in_background(section, timeout) 670 else: 671 result = self._execute_daemon(section, timeout, stderr_redirector, 672 client_disconnect_timeout) 673 674 last_line = stderr_redirector.last_line 675 676 # check if we failed hard enough to warrant an exception 677 if result.exit_status == 1: 678 err = error.AutotestRunError("client job was aborted") 679 elif not self.background and not result.stderr: 680 err = error.AutotestRunError( 681 "execute_section %s failed to return anything\n" 682 "stdout:%s\n" % (section, result.stdout)) 683 else: 684 err = None 685 686 # log something if the client failed AND never finished logging 687 if err and not self.is_client_job_finished(last_line): 688 self.log_unexpected_abort(stderr_redirector) 689 690 if err: 691 raise err 692 else: 693 return stderr_redirector.last_line 694 695 696 def _wait_for_reboot(self, old_boot_id): 697 logging.info("Client is rebooting") 698 logging.info("Waiting for client to halt") 699 if not self.host.wait_down(HALT_TIME, old_boot_id=old_boot_id): 700 err = "%s failed to shutdown after %d" 701 err %= (self.host.hostname, HALT_TIME) 702 raise error.AutotestRunError(err) 703 logging.info("Client down, waiting for restart") 704 if not self.host.wait_up(BOOT_TIME): 705 # since reboot failed 706 # hardreset the machine once if possible 707 # before failing this control file 708 warning = "%s did not come back up, hard resetting" 709 warning %= self.host.hostname 710 logging.warning(warning) 711 try: 712 self.host.hardreset(wait=False) 713 except (AttributeError, error.AutoservUnsupportedError): 714 warning = "Hard reset unsupported on %s" 715 warning %= self.host.hostname 716 logging.warning(warning) 717 raise error.AutotestRunError("%s failed to boot after %ds" % 718 (self.host.hostname, BOOT_TIME)) 719 self.host.reboot_followup() 720 721 722 def execute_control(self, timeout=None, client_disconnect_timeout=None): 723 if not self.background: 724 collector = log_collector(self.host, self.tag, self.results_dir) 725 hostname = self.host.hostname 726 remote_results = collector.client_results_dir 727 local_results = collector.server_results_dir 728 self.host.job.add_client_log(hostname, remote_results, 729 local_results) 730 job_record_context = self.host.job.get_record_context() 731 732 section = 0 733 start_time = time.time() 734 735 logger = client_logger(self.host, self.tag, self.results_dir) 736 try: 737 while not timeout or time.time() < start_time + timeout: 738 if timeout: 739 section_timeout = start_time + timeout - time.time() 740 else: 741 section_timeout = None 742 boot_id = self.host.get_boot_id() 743 last = self.execute_section(section, section_timeout, 744 logger, client_disconnect_timeout) 745 if self.background: 746 return 747 section += 1 748 if self.is_client_job_finished(last): 749 logging.info("Client complete") 750 return 751 elif self.is_client_job_rebooting(last): 752 try: 753 self._wait_for_reboot(boot_id) 754 except error.AutotestRunError, e: 755 self.host.job.record("ABORT", None, "reboot", str(e)) 756 self.host.job.record("END ABORT", None, None, str(e)) 757 raise 758 continue 759 760 # if we reach here, something unexpected happened 761 self.log_unexpected_abort(logger) 762 763 # give the client machine a chance to recover from a crash 764 self.host.wait_up(CRASH_RECOVERY_TIME) 765 msg = ("Aborting - unexpected final status message from " 766 "client on %s: %s\n") % (self.host.hostname, last) 767 raise error.AutotestRunError(msg) 768 finally: 769 logger.close() 770 if not self.background: 771 collector.collect_client_job_results() 772 collector.remove_redundant_client_logs() 773 state_file = os.path.basename(self.remote_control_file 774 + '.state') 775 state_path = os.path.join(self.results_dir, state_file) 776 self.host.job.postprocess_client_state(state_path) 777 self.host.job.remove_client_log(hostname, remote_results, 778 local_results) 779 job_record_context.restore() 780 781 # should only get here if we timed out 782 assert timeout 783 raise error.AutotestTimeoutError() 784 785 786class log_collector(object): 787 def __init__(self, host, client_tag, results_dir): 788 self.host = host 789 if not client_tag: 790 client_tag = "default" 791 self.client_results_dir = os.path.join(host.get_autodir(), "results", 792 client_tag) 793 self.server_results_dir = results_dir 794 795 796 def collect_client_job_results(self): 797 """ A method that collects all the current results of a running 798 client job into the results dir. By default does nothing as no 799 client job is running, but when running a client job you can override 800 this with something that will actually do something. """ 801 802 # make an effort to wait for the machine to come up 803 try: 804 self.host.wait_up(timeout=30) 805 except error.AutoservError: 806 # don't worry about any errors, we'll try and 807 # get the results anyway 808 pass 809 810 # Copy all dirs in default to results_dir 811 try: 812 self.host.get_file(self.client_results_dir + '/', 813 self.server_results_dir, preserve_symlinks=True) 814 except Exception: 815 # well, don't stop running just because we couldn't get logs 816 e_msg = "Unexpected error copying test result logs, continuing ..." 817 logging.error(e_msg) 818 traceback.print_exc(file=sys.stdout) 819 820 821 def remove_redundant_client_logs(self): 822 """Remove client.*.log files in favour of client.*.DEBUG files.""" 823 debug_dir = os.path.join(self.server_results_dir, 'debug') 824 debug_files = [f for f in os.listdir(debug_dir) 825 if re.search(r'^client\.\d+\.DEBUG$', f)] 826 for debug_file in debug_files: 827 log_file = debug_file.replace('DEBUG', 'log') 828 log_file = os.path.join(debug_dir, log_file) 829 if os.path.exists(log_file): 830 os.remove(log_file) 831 832 833# a file-like object for catching stderr from an autotest client and 834# extracting status logs from it 835class client_logger(object): 836 """Partial file object to write to both stdout and 837 the status log file. We only implement those methods 838 utils.run() actually calls. 839 """ 840 status_parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$") 841 test_complete_parser = re.compile(r"^AUTOTEST_TEST_COMPLETE:(.*)$") 842 fetch_package_parser = re.compile( 843 r"^AUTOTEST_FETCH_PACKAGE:([^:]*):([^:]*):(.*)$") 844 extract_indent = re.compile(r"^(\t*).*$") 845 extract_timestamp = re.compile(r".*\ttimestamp=(\d+)\t.*$") 846 847 def __init__(self, host, tag, server_results_dir): 848 self.host = host 849 self.job = host.job 850 self.log_collector = log_collector(host, tag, server_results_dir) 851 self.leftover = "" 852 self.last_line = "" 853 self.logs = {} 854 855 856 def _process_log_dict(self, log_dict): 857 log_list = log_dict.pop("logs", []) 858 for key in sorted(log_dict.iterkeys()): 859 log_list += self._process_log_dict(log_dict.pop(key)) 860 return log_list 861 862 863 def _process_logs(self): 864 """Go through the accumulated logs in self.log and print them 865 out to stdout and the status log. Note that this processes 866 logs in an ordering where: 867 868 1) logs to different tags are never interleaved 869 2) logs to x.y come before logs to x.y.z for all z 870 3) logs to x.y come before x.z whenever y < z 871 872 Note that this will in general not be the same as the 873 chronological ordering of the logs. However, if a chronological 874 ordering is desired that one can be reconstructed from the 875 status log by looking at timestamp lines.""" 876 log_list = self._process_log_dict(self.logs) 877 for entry in log_list: 878 self.job.record_entry(entry, log_in_subdir=False) 879 if log_list: 880 self.last_line = log_list[-1].render() 881 882 883 def _process_quoted_line(self, tag, line): 884 """Process a line quoted with an AUTOTEST_STATUS flag. If the 885 tag is blank then we want to push out all the data we've been 886 building up in self.logs, and then the newest line. If the 887 tag is not blank, then push the line into the logs for handling 888 later.""" 889 entry = base_job.status_log_entry.parse(line) 890 if entry is None: 891 return # the line contains no status lines 892 if tag == "": 893 self._process_logs() 894 self.job.record_entry(entry, log_in_subdir=False) 895 self.last_line = line 896 else: 897 tag_parts = [int(x) for x in tag.split(".")] 898 log_dict = self.logs 899 for part in tag_parts: 900 log_dict = log_dict.setdefault(part, {}) 901 log_list = log_dict.setdefault("logs", []) 902 log_list.append(entry) 903 904 905 def _process_info_line(self, line): 906 """Check if line is an INFO line, and if it is, interpret any control 907 messages (e.g. enabling/disabling warnings) that it may contain.""" 908 match = re.search(r"^\t*INFO\t----\t----(.*)\t[^\t]*$", line) 909 if not match: 910 return # not an INFO line 911 for field in match.group(1).split('\t'): 912 if field.startswith("warnings.enable="): 913 func = self.job.warning_manager.enable_warnings 914 elif field.startswith("warnings.disable="): 915 func = self.job.warning_manager.disable_warnings 916 else: 917 continue 918 warning_type = field.split("=", 1)[1] 919 func(warning_type) 920 921 922 def _process_line(self, line): 923 """Write out a line of data to the appropriate stream. Status 924 lines sent by autotest will be prepended with 925 "AUTOTEST_STATUS", and all other lines are ssh error 926 messages.""" 927 status_match = self.status_parser.search(line) 928 test_complete_match = self.test_complete_parser.search(line) 929 fetch_package_match = self.fetch_package_parser.search(line) 930 if status_match: 931 tag, line = status_match.groups() 932 self._process_info_line(line) 933 self._process_quoted_line(tag, line) 934 elif test_complete_match: 935 self._process_logs() 936 fifo_path, = test_complete_match.groups() 937 try: 938 self.log_collector.collect_client_job_results() 939 self.host.run("echo A > %s" % fifo_path) 940 except Exception: 941 msg = "Post-test log collection failed, continuing anyway" 942 logging.exception(msg) 943 elif fetch_package_match: 944 pkg_name, dest_path, fifo_path = fetch_package_match.groups() 945 serve_packages = global_config.global_config.get_config_value( 946 "PACKAGES", "serve_packages_from_autoserv", type=bool) 947 if serve_packages and pkg_name.endswith(".tar.bz2"): 948 try: 949 self._send_tarball(pkg_name, dest_path) 950 except Exception: 951 msg = "Package tarball creation failed, continuing anyway" 952 logging.exception(msg) 953 try: 954 self.host.run("echo B > %s" % fifo_path) 955 except Exception: 956 msg = "Package tarball installation failed, continuing anyway" 957 logging.exception(msg) 958 else: 959 logging.info(line) 960 961 962 def _send_tarball(self, pkg_name, remote_dest): 963 name, pkg_type = self.job.pkgmgr.parse_tarball_name(pkg_name) 964 src_dirs = [] 965 if pkg_type == 'test': 966 for test_dir in ['site_tests', 'tests']: 967 src_dir = os.path.join(self.job.clientdir, test_dir, name) 968 if os.path.exists(src_dir): 969 src_dirs += [src_dir] 970 if autoserv_prebuild: 971 prebuild.setup(self.job.clientdir, src_dir) 972 break 973 elif pkg_type == 'profiler': 974 src_dirs += [os.path.join(self.job.clientdir, 'profilers', name)] 975 if autoserv_prebuild: 976 prebuild.setup(self.job.clientdir, src_dir) 977 elif pkg_type == 'dep': 978 src_dirs += [os.path.join(self.job.clientdir, 'deps', name)] 979 elif pkg_type == 'client': 980 return # you must already have a client to hit this anyway 981 else: 982 return # no other types are supported 983 984 # iterate over src_dirs until we find one that exists, then tar it 985 for src_dir in src_dirs: 986 if os.path.exists(src_dir): 987 try: 988 logging.info('Bundling %s into %s', src_dir, pkg_name) 989 temp_dir = autotemp.tempdir(unique_id='autoserv-packager', 990 dir=self.job.tmpdir) 991 tarball_path = self.job.pkgmgr.tar_package( 992 pkg_name, src_dir, temp_dir.name, " .") 993 self.host.send_file(tarball_path, remote_dest) 994 finally: 995 temp_dir.clean() 996 return 997 998 999 def log_warning(self, msg, warning_type): 1000 """Injects a WARN message into the current status logging stream.""" 1001 timestamp = int(time.time()) 1002 if self.job.warning_manager.is_valid(timestamp, warning_type): 1003 self.job.record('WARN', None, None, {}, msg) 1004 1005 1006 def write(self, data): 1007 # now start processing the existing buffer and the new data 1008 data = self.leftover + data 1009 lines = data.split('\n') 1010 processed_lines = 0 1011 try: 1012 # process all the buffered data except the last line 1013 # ignore the last line since we may not have all of it yet 1014 for line in lines[:-1]: 1015 self._process_line(line) 1016 processed_lines += 1 1017 finally: 1018 # save any unprocessed lines for future processing 1019 self.leftover = '\n'.join(lines[processed_lines:]) 1020 1021 1022 def flush(self): 1023 sys.stdout.flush() 1024 1025 1026 def flush_all_buffers(self): 1027 if self.leftover: 1028 self._process_line(self.leftover) 1029 self.leftover = "" 1030 self._process_logs() 1031 self.flush() 1032 1033 1034 def close(self): 1035 self.flush_all_buffers() 1036 1037 1038SiteAutotest = client_utils.import_site_class( 1039 __file__, "autotest_lib.server.site_autotest", "SiteAutotest", 1040 BaseAutotest) 1041 1042 1043_SiteRun = client_utils.import_site_class( 1044 __file__, "autotest_lib.server.site_autotest", "_SiteRun", _BaseRun) 1045 1046 1047class Autotest(SiteAutotest): 1048 pass 1049 1050 1051class _Run(_SiteRun): 1052 pass 1053 1054 1055class AutotestHostMixin(object): 1056 """A generic mixin to add a run_test method to classes, which will allow 1057 you to run an autotest client test on a machine directly.""" 1058 1059 # for testing purposes 1060 _Autotest = Autotest 1061 1062 def run_test(self, test_name, **dargs): 1063 """Run an autotest client test on the host. 1064 1065 @param test_name: The name of the client test. 1066 @param dargs: Keyword arguments to pass to the test. 1067 1068 @returns: True if the test passes, False otherwise.""" 1069 at = self._Autotest() 1070 control_file = ('result = job.run_test(%s)\n' 1071 'job.set_state("test_result", result)\n') 1072 test_args = [repr(test_name)] 1073 test_args += ['%s=%r' % (k, v) for k, v in dargs.iteritems()] 1074 control_file %= ', '.join(test_args) 1075 at.run(control_file, host=self) 1076 return at.job.get_state('test_result', default=False) 1077