1dcd57a85fc9233c4dc9f45da55338a2c09f5a37embligh# Copyright 2007 Google Inc. Released under the GPL v2 2b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi#pylint: disable-msg=C0111 3dcd57a85fc9233c4dc9f45da55338a2c09f5a37embligh 407f53b97c4ca29c9613c5eff2948e10e520f37bdbeepsimport re, os, sys, traceback, time, glob, tempfile 507f53b97c4ca29c9613c5eff2948e10e520f37bdbeepsimport logging 61f572e5efbaaa8bc0d1246123ab897607c9a011bmblighfrom autotest_lib.server import installable_object, prebuild, utils 707f53b97c4ca29c9613c5eff2948e10e520f37bdbeepsfrom autotest_lib.client.common_lib import base_job, error, autotemp 81889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shifrom autotest_lib.client.common_lib import base_packages, packages 91889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shifrom autotest_lib.client.common_lib import global_config 10a700772e70548b10721ba709c7ec5194bb6d7597mblighfrom autotest_lib.client.common_lib import utils as client_utils 111e1c41b1b4a1b97c0b7086b8430856ed45e064d3Gabe Blackfrom autotest_lib.client.common_lib.cros.graphite import autotest_stats 123c7a1500cd0e67ce3ee4752453ea3e76eb088ccdmbligh 13cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale CurtisAUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' 14dcd57a85fc9233c4dc9f45da55338a2c09f5a37emblighAUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client' 15dcd57a85fc9233c4dc9f45da55338a2c09f5a37embligh 16dcd57a85fc9233c4dc9f45da55338a2c09f5a37embligh 171f572e5efbaaa8bc0d1246123ab897607c9a011bmblighget_value = global_config.global_config.get_config_value 181f572e5efbaaa8bc0d1246123ab897607c9a011bmblighautoserv_prebuild = get_value('AUTOSERV', 'enable_server_prebuild', 191f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh type=bool, default=False) 201f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh 212a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski 22ad812bf3cf46bd9da1871401042b2b0abf1a4220showardclass AutodirNotFoundError(Exception): 23ad812bf3cf46bd9da1871401042b2b0abf1a4220showard """No Autotest installation could be found.""" 24ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 25ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 26607428b251f437f924932a7b5386cbc06d63b22bbeepsclass AutotestFailure(Exception): 27607428b251f437f924932a7b5386cbc06d63b22bbeeps """Gereric exception class for failures during a test run.""" 28607428b251f437f924932a7b5386cbc06d63b22bbeeps 29607428b251f437f924932a7b5386cbc06d63b22bbeeps 30607428b251f437f924932a7b5386cbc06d63b22bbeepsclass AutotestAbort(AutotestFailure): 31607428b251f437f924932a7b5386cbc06d63b22bbeeps """ 32607428b251f437f924932a7b5386cbc06d63b22bbeeps AutotestAborts are thrown when the DUT seems fine, 33607428b251f437f924932a7b5386cbc06d63b22bbeeps and the test doesn't give us an explicit reason for 34607428b251f437f924932a7b5386cbc06d63b22bbeeps failure; In this case we have no choice but to abort. 35607428b251f437f924932a7b5386cbc06d63b22bbeeps """ 36607428b251f437f924932a7b5386cbc06d63b22bbeeps 37607428b251f437f924932a7b5386cbc06d63b22bbeeps 38607428b251f437f924932a7b5386cbc06d63b22bbeepsclass AutotestDeviceError(AutotestFailure): 39607428b251f437f924932a7b5386cbc06d63b22bbeeps """ 40607428b251f437f924932a7b5386cbc06d63b22bbeeps Exceptions that inherit from AutotestDeviceError 41607428b251f437f924932a7b5386cbc06d63b22bbeeps are thrown when we can determine the current 42607428b251f437f924932a7b5386cbc06d63b22bbeeps state of the DUT and conclude that it probably 43607428b251f437f924932a7b5386cbc06d63b22bbeeps lead to the test failing; these exceptions lead 44607428b251f437f924932a7b5386cbc06d63b22bbeeps to failures instead of aborts. 45607428b251f437f924932a7b5386cbc06d63b22bbeeps """ 46607428b251f437f924932a7b5386cbc06d63b22bbeeps 47607428b251f437f924932a7b5386cbc06d63b22bbeeps 48607428b251f437f924932a7b5386cbc06d63b22bbeepsclass AutotestDeviceNotPingable(AutotestDeviceError): 49607428b251f437f924932a7b5386cbc06d63b22bbeeps """Error for when a DUT becomes unpingable.""" 50607428b251f437f924932a7b5386cbc06d63b22bbeeps 51607428b251f437f924932a7b5386cbc06d63b22bbeeps 52607428b251f437f924932a7b5386cbc06d63b22bbeepsclass AutotestDeviceNotSSHable(AutotestDeviceError): 53607428b251f437f924932a7b5386cbc06d63b22bbeeps """Error for when a DUT is pingable but not SSHable.""" 54607428b251f437f924932a7b5386cbc06d63b22bbeeps 55607428b251f437f924932a7b5386cbc06d63b22bbeeps 56607428b251f437f924932a7b5386cbc06d63b22bbeepsclass AutotestDeviceRebooted(AutotestDeviceError): 57607428b251f437f924932a7b5386cbc06d63b22bbeeps """Error for when a DUT rebooted unexpectedly.""" 58607428b251f437f924932a7b5386cbc06d63b22bbeeps 59607428b251f437f924932a7b5386cbc06d63b22bbeeps 60d8b39259ca4f011cea6f804e5b407262cad57057mblighclass BaseAutotest(installable_object.InstallableObject): 610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski This class represents the Autotest program. 630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Autotest is used to run tests automatically and collect the results. 650afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski It also supports profilers. 660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 670afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Implementation details: 680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski This is a leaf class in an abstract class hierarchy, it must 690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski implement the unimplemented methods in parent classes. 700afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 710afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 72cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis def __init__(self, host=None): 730afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.host = host 740afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.got = False 750afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.installed = False 760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.serverdir = utils.get_server_dir() 770afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski super(BaseAutotest, self).__init__() 780afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 790afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 80f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski install_in_tmpdir = False 81f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski @classmethod 82f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski def set_install_in_tmpdir(cls, flag): 83f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski """ Sets a flag that controls whether or not Autotest should by 84f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski default be installed in a "standard" directory (e.g. 85f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski /home/autotest, /usr/local/autotest) or a temporary directory. """ 86f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski cls.install_in_tmpdir = flag 87f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski 88f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski 89ad812bf3cf46bd9da1871401042b2b0abf1a4220showard @classmethod 90ad812bf3cf46bd9da1871401042b2b0abf1a4220showard def get_client_autodir_paths(cls, host): 91ad812bf3cf46bd9da1871401042b2b0abf1a4220showard return global_config.global_config.get_config_value( 92ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 'AUTOSERV', 'client_autodir_paths', type=list) 93ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 94ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 95ad812bf3cf46bd9da1871401042b2b0abf1a4220showard @classmethod 96ad812bf3cf46bd9da1871401042b2b0abf1a4220showard def get_installed_autodir(cls, host): 97ad812bf3cf46bd9da1871401042b2b0abf1a4220showard """ 98ad812bf3cf46bd9da1871401042b2b0abf1a4220showard Find where the Autotest client is installed on the host. 99ad812bf3cf46bd9da1871401042b2b0abf1a4220showard @returns an absolute path to an installed Autotest client root. 100ad812bf3cf46bd9da1871401042b2b0abf1a4220showard @raises AutodirNotFoundError if no Autotest installation can be found. 101ad812bf3cf46bd9da1871401042b2b0abf1a4220showard """ 102ad812bf3cf46bd9da1871401042b2b0abf1a4220showard autodir = host.get_autodir() 103ad812bf3cf46bd9da1871401042b2b0abf1a4220showard if autodir: 104ad812bf3cf46bd9da1871401042b2b0abf1a4220showard logging.debug('Using existing host autodir: %s', autodir) 105ad812bf3cf46bd9da1871401042b2b0abf1a4220showard return autodir 106ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 107ad812bf3cf46bd9da1871401042b2b0abf1a4220showard for path in Autotest.get_client_autodir_paths(host): 108ad812bf3cf46bd9da1871401042b2b0abf1a4220showard try: 109ad812bf3cf46bd9da1871401042b2b0abf1a4220showard autotest_binary = os.path.join(path, 'bin', 'autotest') 110ad812bf3cf46bd9da1871401042b2b0abf1a4220showard host.run('test -x %s' % utils.sh_escape(autotest_binary)) 111861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li host.run('test -w %s' % utils.sh_escape(path)) 112ad812bf3cf46bd9da1871401042b2b0abf1a4220showard logging.debug('Found existing autodir at %s', path) 113ad812bf3cf46bd9da1871401042b2b0abf1a4220showard return path 114ad812bf3cf46bd9da1871401042b2b0abf1a4220showard except error.AutoservRunError: 115ad812bf3cf46bd9da1871401042b2b0abf1a4220showard logging.debug('%s does not exist on %s', autotest_binary, 116ad812bf3cf46bd9da1871401042b2b0abf1a4220showard host.hostname) 117ad812bf3cf46bd9da1871401042b2b0abf1a4220showard raise AutodirNotFoundError 118ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 119ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 120ad812bf3cf46bd9da1871401042b2b0abf1a4220showard @classmethod 121ad812bf3cf46bd9da1871401042b2b0abf1a4220showard def get_install_dir(cls, host): 122ad812bf3cf46bd9da1871401042b2b0abf1a4220showard """ 123ad812bf3cf46bd9da1871401042b2b0abf1a4220showard Determines the location where autotest should be installed on 124f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski host. If self.install_in_tmpdir is set, it will return a unique 125ad812bf3cf46bd9da1871401042b2b0abf1a4220showard temporary directory that autotest can be installed in. Otherwise, looks 126ad812bf3cf46bd9da1871401042b2b0abf1a4220showard for an existing installation to use; if none is found, looks for a 127ad812bf3cf46bd9da1871401042b2b0abf1a4220showard usable directory in the global config client_autodir_paths. 128ad812bf3cf46bd9da1871401042b2b0abf1a4220showard """ 129f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski try: 1309dcf083957dff6257b7cbaa23db253b65bd92a25lmr install_dir = cls.get_installed_autodir(host) 131ad812bf3cf46bd9da1871401042b2b0abf1a4220showard except AutodirNotFoundError: 1329dcf083957dff6257b7cbaa23db253b65bd92a25lmr install_dir = cls._find_installable_dir(host) 1339dcf083957dff6257b7cbaa23db253b65bd92a25lmr 1349dcf083957dff6257b7cbaa23db253b65bd92a25lmr if cls.install_in_tmpdir: 1359dcf083957dff6257b7cbaa23db253b65bd92a25lmr return host.get_tmp_dir(parent=install_dir) 1369dcf083957dff6257b7cbaa23db253b65bd92a25lmr return install_dir 137ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 138ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 139ad812bf3cf46bd9da1871401042b2b0abf1a4220showard @classmethod 140ad812bf3cf46bd9da1871401042b2b0abf1a4220showard def _find_installable_dir(cls, host): 141ad812bf3cf46bd9da1871401042b2b0abf1a4220showard client_autodir_paths = cls.get_client_autodir_paths(host) 142ad812bf3cf46bd9da1871401042b2b0abf1a4220showard for path in client_autodir_paths: 143ad812bf3cf46bd9da1871401042b2b0abf1a4220showard try: 144f44ff961880151407c051a6b657ebdc9019d6945J. Richard Barnette host.run('mkdir -p %s' % utils.sh_escape(path)) 145861b2d54aec24228cdb3895dbc40062cb40cb2adEric Li host.run('test -w %s' % utils.sh_escape(path)) 146ad812bf3cf46bd9da1871401042b2b0abf1a4220showard return path 147ad812bf3cf46bd9da1871401042b2b0abf1a4220showard except error.AutoservRunError: 148ad812bf3cf46bd9da1871401042b2b0abf1a4220showard logging.debug('Failed to create %s', path) 149a8ff78d6cc8514ad8d52274ab4b144d9e2aa3e94Dan Shi metadata = {'_type': 'AutoservInstallError', 150a8ff78d6cc8514ad8d52274ab4b144d9e2aa3e94Dan Shi 'hostname': host.hostname} 151a8ff78d6cc8514ad8d52274ab4b144d9e2aa3e94Dan Shi autotest_stats.Counter('AutoservInstallError', 152a8ff78d6cc8514ad8d52274ab4b144d9e2aa3e94Dan Shi metadata=metadata).increment() 153ad812bf3cf46bd9da1871401042b2b0abf1a4220showard raise error.AutoservInstallError( 15474a314b490ff542c4dd2ae4aa0d11c6394d92960Dale Curtis 'Unable to find a place to install Autotest; tried %s' % 155ad812bf3cf46bd9da1871401042b2b0abf1a4220showard ', '.join(client_autodir_paths)) 156f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski 157f22fea8f311c4d955b171a2e7869732d6a1067e4jadmanski 158d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li def get_fetch_location(self): 159d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li c = global_config.global_config 160d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li repos = c.get_config_value("PACKAGES", 'fetch_location', type=list, 161d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li default=[]) 162d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li repos.reverse() 163d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li return repos 164d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li 165d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li 166b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi def install(self, host=None, autodir=None, use_packaging=True): 167b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi self._install(host=host, autodir=autodir, use_packaging=use_packaging) 16854f90af69daac8a781678f0ccd42d6728d5d81d1jadmanski 16954f90af69daac8a781678f0ccd42d6728d5d81d1jadmanski 170b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh def install_full_client(self, host=None, autodir=None): 171b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh self._install(host=host, autodir=autodir, use_autoserv=False, 172b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh use_packaging=False) 173bccad480b242b63f93f35b55a0e77517e3a76febmbligh 174bccad480b242b63f93f35b55a0e77517e3a76febmbligh 175b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh def install_no_autoserv(self, host=None, autodir=None): 176b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh self._install(host=host, autodir=autodir, use_autoserv=False) 177b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 178b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 179b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh def _install_using_packaging(self, host, autodir): 180d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li repos = self.get_fetch_location() 181b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh if not repos: 182b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh raise error.PackageInstallError("No repos to install an " 183b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh "autotest client from") 184b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh pkgmgr = packages.PackageManager(autodir, hostname=host.hostname, 185b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh repo_urls=repos, 186b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh do_locking=False, 187b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh run_function=host.run, 188b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh run_function_dargs=dict(timeout=600)) 189b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh # The packages dir is used to store all the packages that 190b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh # are fetched on that client. (for the tests,deps etc. 191b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh # too apart from the client) 192b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh pkg_dir = os.path.join(autodir, 'packages') 193b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh # clean up the autodir except for the packages directory 194b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh host.run('cd %s && ls | grep -v "^packages$"' 1952de320dc5137d947ecd12859c400009a5fce2051Laurence Goodby ' | xargs rm -rf && rm -rf .[!.]*' % autodir) 196b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir, 197b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh preserve_install_dir=True) 198b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh self.installed = True 199b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 200b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 201b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh def _install_using_send_file(self, host, autodir): 2023ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers", 2033ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa "packages"]) 204b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh light_files = [os.path.join(self.source_material, f) 205b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh for f in os.listdir(self.source_material) 206b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh if f not in dirs_to_exclude] 207b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh host.send_file(light_files, autodir, delete_dest=True) 208b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 209b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh # create empty dirs for all the stuff we excluded 210b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh commands = [] 211b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh for path in dirs_to_exclude: 212b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh abs_path = os.path.join(autodir, path) 213b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh abs_path = utils.sh_escape(abs_path) 214b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh commands.append("mkdir -p '%s'" % abs_path) 215b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh commands.append("touch '%s'/__init__.py" % abs_path) 216b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh host.run(';'.join(commands)) 217b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 218b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh 219b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh def _install(self, host=None, autodir=None, use_autoserv=True, 220b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh use_packaging=True): 2210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 2220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Install autotest. If get() was not called previously, an 2230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski attempt will be made to install from the autotest svn 2240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski repository. 2250afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 226bccad480b242b63f93f35b55a0e77517e3a76febmbligh @param host A Host instance on which autotest will be installed 227bccad480b242b63f93f35b55a0e77517e3a76febmbligh @param autodir Location on the remote host to install to 228b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh @param use_autoserv Enable install modes that depend on the client 229bccad480b242b63f93f35b55a0e77517e3a76febmbligh running with the autoserv harness 230b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh @param use_packaging Enable install modes that use the packaging system 2310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 232bccad480b242b63f93f35b55a0e77517e3a76febmbligh @exception AutoservError if a tarball was not specified and 233bccad480b242b63f93f35b55a0e77517e3a76febmbligh the target host does not have svn installed in its path 234bccad480b242b63f93f35b55a0e77517e3a76febmbligh """ 2350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not host: 2360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host = self.host 2370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not self.got: 2380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.get() 2390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host.wait_up(timeout=30) 2400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host.setup() 241b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info("Installing autotest on %s", host.hostname) 2420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 24354f90af69daac8a781678f0ccd42d6728d5d81d1jadmanski # set up the autotest directory on the remote machine 24454f90af69daac8a781678f0ccd42d6728d5d81d1jadmanski if not autodir: 245ad812bf3cf46bd9da1871401042b2b0abf1a4220showard autodir = self.get_install_dir(host) 246ad812bf3cf46bd9da1871401042b2b0abf1a4220showard logging.info('Using installation dir %s', autodir) 2470562e658835ab8d604f56a4b9b3a84bcadafc143mbligh host.set_autodir(autodir) 2483c2369470f809d4a920e8cc102c9ad54fdb5c839jadmanski host.run('mkdir -p %s' % utils.sh_escape(autodir)) 2490afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 2501c3c07b9c7100c9ae28d609a4d94764a5ce0d9abjadmanski # make sure there are no files in $AUTODIR/results 2511c3c07b9c7100c9ae28d609a4d94764a5ce0d9abjadmanski results_path = os.path.join(autodir, 'results') 2523c2369470f809d4a920e8cc102c9ad54fdb5c839jadmanski host.run('rm -rf %s/*' % utils.sh_escape(results_path), 2531c3c07b9c7100c9ae28d609a4d94764a5ce0d9abjadmanski ignore_status=True) 2541c3c07b9c7100c9ae28d609a4d94764a5ce0d9abjadmanski 255c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh # Fetch the autotest client from the nearest repository 256b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh if use_packaging: 257b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh try: 258b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh self._install_using_packaging(host, autodir) 259b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi logging.info("Installation of autotest completed using the " 260b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi "packaging system.") 261b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh return 2626f27d4f22a1ba5063968b8c322fa0845f3279adeEric Li except (error.PackageInstallError, error.AutoservRunError, 2636f27d4f22a1ba5063968b8c322fa0845f3279adeEric Li global_config.ConfigError), e: 264b8aa75bded69fd07b808c61eeed8eff9c4b12d63mbligh logging.info("Could not install autotest using the packaging " 265cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis "system: %s. Trying other methods", e) 2661889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shi else: 2671889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shi # Delete the package checksum file to force dut updating local 2681889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shi # packages. 2691889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shi command = ('rm -f "%s"' % 2701889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shi (os.path.join(autodir, base_packages.CHECKSUM_FILE))) 2711889ca1f96b253d6054fd2c0e9b31e06a34d1035Dan Shi host.run(command) 272c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh 2730afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # try to install from file or directory 2740afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if self.source_material: 27569bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski c = global_config.global_config 27669bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski supports_autoserv_packaging = c.get_config_value( 27769bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski "PACKAGES", "serve_packages_from_autoserv", type=bool) 27869bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski # Copy autotest recursively 27969bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski if supports_autoserv_packaging and use_autoserv: 28069bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski self._install_using_send_file(host, autodir) 2810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 28269bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski host.send_file(self.source_material, autodir, delete_dest=True) 283b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi logging.info("Installation of autotest completed from %s", 284b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi self.source_material) 2850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.installed = True 2860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 2870afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 2880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # if that fails try to install using svn 2890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if utils.run('which svn').exit_status: 29074a314b490ff542c4dd2ae4aa0d11c6394d92960Dale Curtis raise error.AutoservError('svn not found on target machine: %s' % 29174a314b490ff542c4dd2ae4aa0d11c6394d92960Dale Curtis host.name) 2920afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 29378bf5355ae3455f6e18fde2b4a839b16a59db337mbligh host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir)) 2940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except error.AutoservRunError, e: 29578bf5355ae3455f6e18fde2b4a839b16a59db337mbligh host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir)) 296b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi logging.info("Installation of autotest completed using SVN.") 2970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.installed = True 2980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 2990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 3007c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski def uninstall(self, host=None): 3017c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski """ 3027c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski Uninstall (i.e. delete) autotest. Removes the autotest client install 3037c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski from the specified host. 3047c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski 3057c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski @params host a Host instance from which the client will be removed 3067c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski """ 3077c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski if not self.installed: 3087c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski return 3097c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski if not host: 3107c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski host = self.host 3117c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski autodir = host.get_autodir() 3127c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski if not autodir: 3137c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski return 3147c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski 3157c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski # perform the actual uninstall 3167c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True) 3177c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski host.set_autodir(None) 3187c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski self.installed = False 3197c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski 3207c7aff3a2ed5787d502642a3f5eaa0107aad1661jadmanski 321cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis def get(self, location=None): 3220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not location: 3230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski location = os.path.join(self.serverdir, '../client') 3240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski location = os.path.abspath(location) 3250afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # If there's stuff run on our client directory already, it 3260afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # can cause problems. Try giving it a quick clean first. 3270afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cwd = os.getcwd() 3280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.chdir(location) 3294b976078babcf8f652a44aa84537861ac1e4d6d0showard try: 3304b976078babcf8f652a44aa84537861ac1e4d6d0showard utils.system('tools/make_clean', ignore_status=True) 3314b976078babcf8f652a44aa84537861ac1e4d6d0showard finally: 3324b976078babcf8f652a44aa84537861ac1e4d6d0showard os.chdir(cwd) 3330afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski super(BaseAutotest, self).get(location) 3340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.got = True 3350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 3360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 337e7d9c605bacd7b1816987994ae18a68c63306a16mbligh def run(self, control_file, results_dir='.', host=None, timeout=None, 338e7d9c605bacd7b1816987994ae18a68c63306a16mbligh tag=None, parallel_flag=False, background=False, 339b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi client_disconnect_timeout=None, use_packaging=True): 3400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 3410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Run an autotest job on the remote machine. 3420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 343e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param control_file: An open file-like-obj of the control file. 344e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param results_dir: A str path where the results should be stored 345e7d9c605bacd7b1816987994ae18a68c63306a16mbligh on the local filesystem. 346e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param host: A Host instance on which the control file should 347e7d9c605bacd7b1816987994ae18a68c63306a16mbligh be run. 348e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param timeout: Maximum number of seconds to wait for the run or None. 349e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param tag: Tag name for the client side instance of autotest. 350e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param parallel_flag: Flag set when multiple jobs are run at the 351e7d9c605bacd7b1816987994ae18a68c63306a16mbligh same time. 352e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param background: Indicates that the client should be launched as 353e7d9c605bacd7b1816987994ae18a68c63306a16mbligh a background job; the code calling run will be responsible 354e7d9c605bacd7b1816987994ae18a68c63306a16mbligh for monitoring the client and collecting the results. 355e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @param client_disconnect_timeout: Seconds to wait for the remote host 356cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis to come back after a reboot. Defaults to the host setting for 357cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis DEFAULT_REBOOT_TIMEOUT. 358e7d9c605bacd7b1816987994ae18a68c63306a16mbligh 359e7d9c605bacd7b1816987994ae18a68c63306a16mbligh @raises AutotestRunError: If there is a problem executing 360e7d9c605bacd7b1816987994ae18a68c63306a16mbligh the control file. 3610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 362b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi host = self._get_host_and_setup(host, use_packaging=use_packaging) 3630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski results_dir = os.path.abspath(results_dir) 3640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 365cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis if client_disconnect_timeout is None: 366cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis client_disconnect_timeout = host.DEFAULT_REBOOT_TIMEOUT 367cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis 3680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if tag: 3690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski results_dir = os.path.join(results_dir, tag) 3700afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 371b3c0c91ff9020a7259377e3d607c3aa0cae9c804mbligh atrun = _Run(host, results_dir, tag, parallel_flag, background) 3726dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski self._do_run(control_file, results_dir, host, atrun, timeout, 373b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi client_disconnect_timeout, use_packaging=use_packaging) 3740afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 3750afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 376b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi def _get_host_and_setup(self, host, use_packaging=True): 3770afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not host: 3780afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host = self.host 3790afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not self.installed: 380b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi self.install(host, use_packaging=use_packaging) 3810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 3820afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host.wait_up(timeout=30) 3830afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return host 3840afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 3850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 3866dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def _do_run(self, control_file, results_dir, host, atrun, timeout, 387b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi client_disconnect_timeout, use_packaging=True): 3880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 3890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski atrun.verify_machine() 3900afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except: 391b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.error("Verify failed on %s. Reinstalling autotest", 392b18134f8faa7c5b4623760bc88650a65e70b2cacshoward host.hostname) 3930afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.install(host) 394b9cd83c205116d37128db5599ea272fc0cf0df8bFang Deng atrun.verify_machine() 3950afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski debug = os.path.join(results_dir, 'debug') 3960afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 3970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.makedirs(debug) 3980910844396c68a8747dc1ab580fb898ecff2469embligh except Exception: 3990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski pass 4000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4010910844396c68a8747dc1ab580fb898ecff2469embligh delete_file_list = [atrun.remote_control_file, 4020910844396c68a8747dc1ab580fb898ecff2469embligh atrun.remote_control_file + '.state', 4030910844396c68a8747dc1ab580fb898ecff2469embligh atrun.manual_control_file, 4040910844396c68a8747dc1ab580fb898ecff2469embligh atrun.manual_control_file + '.state'] 4050910844396c68a8747dc1ab580fb898ecff2469embligh cmd = ';'.join('rm -f ' + control for control in delete_file_list) 4060910844396c68a8747dc1ab580fb898ecff2469embligh host.run(cmd, ignore_status=True) 4070afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 408386eea7066cbd9a35599e76eee56b2c98ecec9a1Dale Curtis tmppath = utils.get(control_file, local_copy=True) 409c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh 410cb0e16134a76b3badc0307cd804a9e510f3e232ejadmanski # build up the initialization prologue for the control file 411cb0e16134a76b3badc0307cd804a9e510f3e232ejadmanski prologue_lines = [] 41223afbec41f6275bef4e80c36726916b0be564204jadmanski 4132f076833aba3e9cc6ddc8a7e1b02b45005770deembligh # Add the additional user arguments 414808f4b1663600ba7e735c4d78125825c52bb804fjadmanski prologue_lines.append("args = %r\n" % self.job.args) 4152f076833aba3e9cc6ddc8a7e1b02b45005770deembligh 4160910844396c68a8747dc1ab580fb898ecff2469embligh # If the packaging system is being used, add the repository list. 417ddc9a40e2f313399c2d3eca1125e39e4bf65882fmbligh repos = None 418c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh try: 419b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi if use_packaging: 420b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi repos = self.get_fetch_location() 421b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi prologue_lines.append('job.add_repository(%s)\n' % repos) 422b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi else: 423b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi logging.debug('use_packaging is set to False, do not add any ' 424b669cbdea7f4bfc9cbfcea48d9cf9ce62f0deff5Dan Shi 'repository.') 425c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh except global_config.ConfigError, e: 426ddc9a40e2f313399c2d3eca1125e39e4bf65882fmbligh # If repos is defined packaging is enabled so log the error 427ddc9a40e2f313399c2d3eca1125e39e4bf65882fmbligh if repos: 428ddc9a40e2f313399c2d3eca1125e39e4bf65882fmbligh logging.error(e) 429c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh 430e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski # on full-size installs, turn on any profilers the server is using 43127b529165c63274395ff1d207e346201cea4cb25jadmanski if not atrun.background: 432e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski running_profilers = host.job.profilers.add_log.iteritems() 433e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski for profiler, (args, dargs) in running_profilers: 434e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski call_args = [repr(profiler)] 435e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski call_args += [repr(arg) for arg in args] 436e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski call_args += ["%s=%r" % item for item in dargs.iteritems()] 437e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski prologue_lines.append("job.profilers.add(%s)\n" 438e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski % ", ".join(call_args)) 439e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski cfile = "".join(prologue_lines) 440e2eef7bedc043857306fc2ea7211cbf6f48a4f09jadmanski 4410910844396c68a8747dc1ab580fb898ecff2469embligh cfile += open(tmppath).read() 4420910844396c68a8747dc1ab580fb898ecff2469embligh open(tmppath, "w").write(cfile) 443c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh 444c09fc1543a3f77026bab728770aa0191feaddd07jadmanski # Create and copy state file to remote_control_file + '.state' 445fc3da5bbdcd63840cd224282812c050c5520db11mbligh state_file = host.job.preprocess_client_state() 446fbf73aecdd357094ae05e7d1e4ea99b1ecf93ee4mbligh host.send_file(state_file, atrun.remote_control_file + '.init.state') 447c09fc1543a3f77026bab728770aa0191feaddd07jadmanski os.remove(state_file) 448c09fc1543a3f77026bab728770aa0191feaddd07jadmanski 449c5ddfd1f71caef9ec0c84c53ef7db42fcdc33e1cmbligh # Copy control_file to remote_control_file on the host 4500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host.send_file(tmppath, atrun.remote_control_file) 4510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.abspath(tmppath) != os.path.abspath(control_file): 4520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.remove(tmppath) 4530afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4546bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski atrun.execute_control( 4556dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski timeout=timeout, 4569285ddf562c698fda569390be6763170215663d7Dale Curtis client_disconnect_timeout=client_disconnect_timeout) 45723afbec41f6275bef4e80c36726916b0be564204jadmanski 45823afbec41f6275bef4e80c36726916b0be564204jadmanski 4590afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def run_timed_test(self, test_name, results_dir='.', host=None, 460b0be88bc1176099c85abe4a5999e55fd7ca8dc1cDennis Jeffrey timeout=None, parallel_flag=False, background=False, 461b0be88bc1176099c85abe4a5999e55fd7ca8dc1cDennis Jeffrey client_disconnect_timeout=None, *args, **dargs): 4620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Assemble a tiny little control file to just run one test, 4640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski and run it as an autotest client-side test 4650afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not host: 4670afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host = self.host 4680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not self.installed: 4690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.install(host) 4700afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()] 4710afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cmd = ", ".join([repr(test_name)] + map(repr, args) + opts) 4720afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski control = "job.run_test(%s)\n" % cmd 473b0be88bc1176099c85abe4a5999e55fd7ca8dc1cDennis Jeffrey self.run(control, results_dir, host, timeout=timeout, 4742da1b76ed7190a8e6c8791997624d6d07b89f650Andrew Bresticker parallel_flag=parallel_flag, background=background, 4752da1b76ed7190a8e6c8791997624d6d07b89f650Andrew Bresticker client_disconnect_timeout=client_disconnect_timeout) 4760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4770afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 478b0be88bc1176099c85abe4a5999e55fd7ca8dc1cDennis Jeffrey def run_test(self, test_name, results_dir='.', host=None, 4792da1b76ed7190a8e6c8791997624d6d07b89f650Andrew Bresticker parallel_flag=False, background=False, 4802da1b76ed7190a8e6c8791997624d6d07b89f650Andrew Bresticker client_disconnect_timeout=None, *args, **dargs): 4810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.run_timed_test(test_name, results_dir, host, timeout=None, 482b0be88bc1176099c85abe4a5999e55fd7ca8dc1cDennis Jeffrey parallel_flag=parallel_flag, background=background, 4832da1b76ed7190a8e6c8791997624d6d07b89f650Andrew Bresticker client_disconnect_timeout=client_disconnect_timeout, 484c98c47002070d684f022bf4e1d957929c309ab25jadmanski *args, **dargs) 485d54832be0608b7381cba6b4ac9dae92a4bb2715dmbligh 486d54832be0608b7381cba6b4ac9dae92a4bb2715dmbligh 48769bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanskiclass _BaseRun(object): 4880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Represents a run of autotest control file. This class maintains 4900afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski all the state necessary as an autotest control file is executed. 4910afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4920afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski It is not intended to be used directly, rather control files 4930afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski should be run using the run method in Autotest. 4940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 495b3c0c91ff9020a7259377e3d607c3aa0cae9c804mbligh def __init__(self, host, results_dir, tag, parallel_flag, background): 4960afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.host = host 4970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.results_dir = results_dir 4980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.env = host.env 4990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.tag = tag 5000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.parallel_flag = parallel_flag 501b3c0c91ff9020a7259377e3d607c3aa0cae9c804mbligh self.background = background 502ad812bf3cf46bd9da1871401042b2b0abf1a4220showard self.autodir = Autotest.get_installed_autodir(self.host) 50378bf5355ae3455f6e18fde2b4a839b16a59db337mbligh control = os.path.join(self.autodir, 'control') 5040afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if tag: 50578bf5355ae3455f6e18fde2b4a839b16a59db337mbligh control += '.' + tag 50678bf5355ae3455f6e18fde2b4a839b16a59db337mbligh self.manual_control_file = control 50778bf5355ae3455f6e18fde2b4a839b16a59db337mbligh self.remote_control_file = control + '.autoserv' 5086d08b3c3fc2858ccd60a323008703179f93ae38blmr self.config_file = os.path.join(self.autodir, 'global_config.ini') 5090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def verify_machine(self): 5120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski binary = os.path.join(self.autodir, 'bin/autotest') 5130afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 5140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.host.run('ls %s > /dev/null 2>&1' % binary) 5150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except: 516d6d27ed06b7e264b8c99862414b13a443197d64clmr raise error.AutoservInstallError( 517d6d27ed06b7e264b8c99862414b13a443197d64clmr "Autotest does not appear to be installed") 5180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not self.parallel_flag: 5200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski tmpdir = os.path.join(self.autodir, 'tmp') 5210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski download = os.path.join(self.autodir, 'tests/download') 5220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.host.run('umount %s' % tmpdir, ignore_status=True) 5230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.host.run('umount %s' % download, ignore_status=True) 5240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5256dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5266dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def get_base_cmd_args(self, section): 527234b296fb6a81f67fd429c9be87b62708738311fshoward args = ['--verbose'] 5280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if section > 0: 5296dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski args.append('-c') 5300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if self.tag: 5316dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski args.append('-t %s' % self.tag) 5320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if self.host.job.use_external_logging(): 5336dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski args.append('-l') 534ce955fcf58207d795ff1fe0001851b8b3728452cmbligh if self.host.hostname: 535ce955fcf58207d795ff1fe0001851b8b3728452cmbligh args.append('--hostname=%s' % self.host.hostname) 5360d0f67d6abcf6599ad626042d3236020e093558fmbligh args.append('--user=%s' % self.host.job.user) 537ce955fcf58207d795ff1fe0001851b8b3728452cmbligh 5386dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski args.append(self.remote_control_file) 5396dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return args 5406dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5416dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5426dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def get_background_cmd(self, section): 5436dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski cmd = ['nohup', os.path.join(self.autodir, 'bin/autotest_client')] 5446dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski cmd += self.get_base_cmd_args(section) 54569bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski cmd += ['>/dev/null', '2>/dev/null', '&'] 5466dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return ' '.join(cmd) 5476dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5486dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5496dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def get_daemon_cmd(self, section, monitor_dir): 5506dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski cmd = ['nohup', os.path.join(self.autodir, 'bin/autotestd'), 5516dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski monitor_dir, '-H autoserv'] 5526dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski cmd += self.get_base_cmd_args(section) 55369bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski cmd += ['>/dev/null', '2>/dev/null', '&'] 5546dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return ' '.join(cmd) 5556dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5566dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 5576dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read): 5586dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski cmd = [os.path.join(self.autodir, 'bin', 'autotestd_monitor'), 5596dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski monitor_dir, str(stdout_read), str(stderr_read)] 5600afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return ' '.join(cmd) 5610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5634d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski def get_client_log(self): 5644d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """Find what the "next" client.* prefix should be 5654d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5664d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @returns A string of the form client.INTEGER that should be prefixed 5674d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski to all client debug log files. 5684d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 5694d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski max_digit = -1 5704d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski debug_dir = os.path.join(self.results_dir, 'debug') 5714d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_logs = glob.glob(os.path.join(debug_dir, 'client.*.*')) 5724d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski for log in client_logs: 5734d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski _, number, _ = log.split('.', 2) 5744d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski if number.isdigit(): 5754d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski max_digit = max(max_digit, int(number)) 5764d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski return 'client.%d' % (max_digit + 1) 5774d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5784d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5794d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski def copy_client_config_file(self, client_log_prefix=None): 5804d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 5814d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski Create and copy the client config file based on the server config. 5824d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5834d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @param client_log_prefix: Optional prefix to prepend to log files. 5844d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 5854d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_config_file = self._create_client_config_file(client_log_prefix) 5864d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski self.host.send_file(client_config_file, self.config_file) 5874d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski os.remove(client_config_file) 5884d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5894d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5904d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski def _create_client_config_file(self, client_log_prefix=None): 5914d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 5924d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski Create a temporary file with the [CLIENT] section configuration values 5934d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski taken from the server global_config.ini. 5944d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5954d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @param client_log_prefix: Optional prefix to prepend to log files. 5964d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 5974d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @return: Path of the temporary file generated. 5984d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 5994d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski config = global_config.global_config.get_section_values('CLIENT') 6004d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski if client_log_prefix: 6014d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski config.set('CLIENT', 'default_logging_name', client_log_prefix) 6024d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski return self._create_aux_file(config.write) 6034d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 6044d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 6054d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski def _create_aux_file(self, func, *args): 6064d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 6074d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski Creates a temporary file and writes content to it according to a 6084d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski content creation function. The file object is appended to *args, which 6094d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski is then passed to the content creation function 6104d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 6114d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @param func: Function that will be used to write content to the 6124d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski temporary file. 6134d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @param *args: List of parameters that func takes. 6144d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski @return: Path to the temporary file that was created. 6154d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """ 6164d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski fd, path = tempfile.mkstemp(dir=self.host.job.tmpdir) 6174d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski aux_file = os.fdopen(fd, "w") 6184d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski try: 6194d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski list_args = list(args) 6204d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski list_args.append(aux_file) 6214d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski func(*list_args) 6224d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski finally: 6234d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski aux_file.close() 6244d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski return path 6250afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 6260afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 627b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski @staticmethod 628b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski def is_client_job_finished(last_line): 629b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski return bool(re.match(r'^END .*\t----\t----\t.*$', last_line)) 630b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 631b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 632b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski @staticmethod 633b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski def is_client_job_rebooting(last_line): 634b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski return bool(re.match(r'^\t*GOOD\t----\treboot\.start.*$', last_line)) 635b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 636b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 637607428b251f437f924932a7b5386cbc06d63b22bbeeps def _diagnose_dut(self, old_boot_id=None): 63807f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps """ 639607428b251f437f924932a7b5386cbc06d63b22bbeeps Run diagnostic checks on a DUT. 640607428b251f437f924932a7b5386cbc06d63b22bbeeps 641607428b251f437f924932a7b5386cbc06d63b22bbeeps 1. ping: A dead host will not respond to pings. 642607428b251f437f924932a7b5386cbc06d63b22bbeeps 2. ssh (happens with 3.): DUT hangs usually fail in authentication 643607428b251f437f924932a7b5386cbc06d63b22bbeeps but respond to pings. 644607428b251f437f924932a7b5386cbc06d63b22bbeeps 3. Check if a reboot occured: A healthy but unexpected reboot leaves the 645607428b251f437f924932a7b5386cbc06d63b22bbeeps host running with a new boot id. 646607428b251f437f924932a7b5386cbc06d63b22bbeeps 647607428b251f437f924932a7b5386cbc06d63b22bbeeps This method will always raise an exception from the AutotestFailure 648607428b251f437f924932a7b5386cbc06d63b22bbeeps family and should only get called when the reason for a test failing 649607428b251f437f924932a7b5386cbc06d63b22bbeeps is ambiguous. 650607428b251f437f924932a7b5386cbc06d63b22bbeeps 651607428b251f437f924932a7b5386cbc06d63b22bbeeps @raises AutotestDeviceNotPingable: If the DUT doesn't respond to ping. 652607428b251f437f924932a7b5386cbc06d63b22bbeeps @raises AutotestDeviceNotSSHable: If we cannot SSH into the DUT. 653607428b251f437f924932a7b5386cbc06d63b22bbeeps @raises AutotestDeviceRebooted: If the boot id changed. 654607428b251f437f924932a7b5386cbc06d63b22bbeeps @raises AutotestAbort: If none of the above exceptions were raised. 655607428b251f437f924932a7b5386cbc06d63b22bbeeps Since we have no recourse we must abort at this stage. 65607f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps """ 657607428b251f437f924932a7b5386cbc06d63b22bbeeps msg = 'Autotest client terminated unexpectedly: ' 65807f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps if utils.ping(self.host.hostname, tries=1, deadline=1) != 0: 659607428b251f437f924932a7b5386cbc06d63b22bbeeps msg += 'DUT is no longer pingable, it may have rebooted or hung.\n' 660607428b251f437f924932a7b5386cbc06d63b22bbeeps raise AutotestDeviceNotPingable(msg) 66107f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps 66207f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps if old_boot_id: 66307f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps try: 66407f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps new_boot_id = self.host.get_boot_id(timeout=60) 665607428b251f437f924932a7b5386cbc06d63b22bbeeps except Exception as e: 66607f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps msg += ('DUT is pingable but not SSHable, it most likely' 66714768813d90a8729c76350fb6f4861a07e4cc86bbeeps ' sporadically rebooted during testing. %s\n' % str(e)) 668607428b251f437f924932a7b5386cbc06d63b22bbeeps raise AutotestDeviceNotSSHable(msg) 66907f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps else: 67007f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps if new_boot_id != old_boot_id: 67107f53b97c4ca29c9613c5eff2948e10e520f37bdbeeps msg += 'DUT rebooted during the test run.\n' 672607428b251f437f924932a7b5386cbc06d63b22bbeeps raise AutotestDeviceRebooted(msg) 673607428b251f437f924932a7b5386cbc06d63b22bbeeps 674607428b251f437f924932a7b5386cbc06d63b22bbeeps msg += ('DUT is pingable, SSHable and did NOT restart ' 675607428b251f437f924932a7b5386cbc06d63b22bbeeps 'un-expectedly. We probably lost connectivity during the ' 676607428b251f437f924932a7b5386cbc06d63b22bbeeps 'test.') 677607428b251f437f924932a7b5386cbc06d63b22bbeeps else: 678607428b251f437f924932a7b5386cbc06d63b22bbeeps msg += ('DUT is pingable, could not determine if an un-expected ' 679607428b251f437f924932a7b5386cbc06d63b22bbeeps 'reboot occured during the test.') 680607428b251f437f924932a7b5386cbc06d63b22bbeeps 681607428b251f437f924932a7b5386cbc06d63b22bbeeps raise AutotestAbort(msg) 682607428b251f437f924932a7b5386cbc06d63b22bbeeps 683607428b251f437f924932a7b5386cbc06d63b22bbeeps 684607428b251f437f924932a7b5386cbc06d63b22bbeeps def log_unexpected_abort(self, stderr_redirector, old_boot_id=None): 685607428b251f437f924932a7b5386cbc06d63b22bbeeps """ 686607428b251f437f924932a7b5386cbc06d63b22bbeeps Logs that something unexpected happened, then tries to diagnose the 687607428b251f437f924932a7b5386cbc06d63b22bbeeps failure. The purpose of this function is only to close out the status 688607428b251f437f924932a7b5386cbc06d63b22bbeeps log with the appropriate error message, not to critically terminate 689607428b251f437f924932a7b5386cbc06d63b22bbeeps the program. 690607428b251f437f924932a7b5386cbc06d63b22bbeeps 691607428b251f437f924932a7b5386cbc06d63b22bbeeps @param stderr_redirector: log stream. 692607428b251f437f924932a7b5386cbc06d63b22bbeeps @param old_boot_id: boot id used to infer if a reboot occured. 693607428b251f437f924932a7b5386cbc06d63b22bbeeps """ 694607428b251f437f924932a7b5386cbc06d63b22bbeeps stderr_redirector.flush_all_buffers() 695607428b251f437f924932a7b5386cbc06d63b22bbeeps try: 696607428b251f437f924932a7b5386cbc06d63b22bbeeps self._diagnose_dut(old_boot_id) 697607428b251f437f924932a7b5386cbc06d63b22bbeeps except AutotestFailure as e: 698607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.job.record('END ABORT', None, None, str(e)) 699b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 700b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 7016dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def _execute_in_background(self, section, timeout): 7026dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski full_cmd = self.get_background_cmd(section) 7036dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski devnull = open(os.devnull, "w") 7040afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 7054d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski self.copy_client_config_file(self.get_client_log()) 7064d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 7070d0f67d6abcf6599ad626042d3236020e093558fmbligh self.host.job.push_execution_context(self.results_dir) 7080afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 7090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski result = self.host.run(full_cmd, ignore_status=True, 7100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski timeout=timeout, 7116dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stdout_tee=devnull, 7126dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_tee=devnull) 7136dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski finally: 7140d0f67d6abcf6599ad626042d3236020e093558fmbligh self.host.job.pop_execution_context() 7156dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7166dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return result 7176dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7186dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7196dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski @staticmethod 7206dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def _strip_stderr_prologue(stderr): 7216dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski """Strips the 'standard' prologue that get pre-pended to every 7226dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski remote command and returns the text that was actually written to 7236dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr by the remote command.""" 7246dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_lines = stderr.split("\n")[1:] 7256dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski if not stderr_lines: 7266dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return "" 7276dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski elif stderr_lines[0].startswith("NOTE: autotestd_monitor"): 7286dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski del stderr_lines[0] 7296dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return "\n".join(stderr_lines) 7306dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7316dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7326dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def _execute_daemon(self, section, timeout, stderr_redirector, 7336dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski client_disconnect_timeout): 7346dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski monitor_dir = self.host.get_tmp_dir() 7356dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski daemon_cmd = self.get_daemon_cmd(section, monitor_dir) 7364d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 7374d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski # grab the location for the server-side client log file 7384d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_log_prefix = self.get_client_log() 7394d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_log_path = os.path.join(self.results_dir, 'debug', 7404d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_log_prefix + '.log') 7414d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_log = open(client_log_path, 'w', 0) 7424d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski self.copy_client_config_file(client_log_prefix) 7436dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7446dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stdout_read = stderr_read = 0 7450d0f67d6abcf6599ad626042d3236020e093558fmbligh self.host.job.push_execution_context(self.results_dir) 7466dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski try: 7476dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski self.host.run(daemon_cmd, ignore_status=True, timeout=timeout) 74891d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski disconnect_warnings = [] 7496dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski while True: 7506dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read, 7516dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_read) 7526dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski try: 7536dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski result = self.host.run(monitor_cmd, ignore_status=True, 7546dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski timeout=timeout, 7556dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stdout_tee=client_log, 7566dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_tee=stderr_redirector) 7576dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski except error.AutoservRunError, e: 7586dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski result = e.result_obj 7596dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski result.exit_status = None 76091d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski disconnect_warnings.append(e.description) 76191d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski 7626dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_redirector.log_warning( 76391d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski "Autotest client was disconnected: %s" % e.description, 76491d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski "NETWORK") 7656dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski except error.AutoservSSHTimeout: 7666dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski result = utils.CmdResult(monitor_cmd, "", "", None, 0) 7676dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_redirector.log_warning( 76891d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski "Attempt to connect to Autotest client timed out", 76991d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski "NETWORK") 7706dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7716dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stdout_read += len(result.stdout) 7726dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski stderr_read += len(self._strip_stderr_prologue(result.stderr)) 7736dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7746dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski if result.exit_status is not None: 775bca10a6fc4527d43199f4ad7f489127662fe6c29Simran Basi # TODO (crosbug.com/38224)- sbasi: Remove extra logging. 776bca10a6fc4527d43199f4ad7f489127662fe6c29Simran Basi logging.debug('Result exit status is %d.', 777bca10a6fc4527d43199f4ad7f489127662fe6c29Simran Basi result.exit_status) 7786dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski return result 7796dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski elif not self.host.wait_up(client_disconnect_timeout): 7806dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski raise error.AutoservSSHTimeout( 7816dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski "client was disconnected, reconnect timed out") 7820afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski finally: 7834d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski client_log.close() 7840d0f67d6abcf6599ad626042d3236020e093558fmbligh self.host.job.pop_execution_context() 7856dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7866dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7876dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski def execute_section(self, section, timeout, stderr_redirector, 7889285ddf562c698fda569390be6763170215663d7Dale Curtis client_disconnect_timeout): 789b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info("Executing %s/bin/autotest %s/control phase %d", 790b18134f8faa7c5b4623760bc88650a65e70b2cacshoward self.autodir, self.autodir, section) 7916dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7926dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski if self.background: 7936dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski result = self._execute_in_background(section, timeout) 7946dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski else: 7956dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski result = self._execute_daemon(section, timeout, stderr_redirector, 7966dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski client_disconnect_timeout) 7976dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 7986dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski last_line = stderr_redirector.last_line 7990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 800b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski # check if we failed hard enough to warrant an exception 8010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if result.exit_status == 1: 802b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski err = error.AutotestRunError("client job was aborted") 803b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski elif not self.background and not result.stderr: 804b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski err = error.AutotestRunError( 805e41305384c80bd6d8a126b3ace503e179e13bf8djadmanski "execute_section %s failed to return anything\n" 806e41305384c80bd6d8a126b3ace503e179e13bf8djadmanski "stdout:%s\n" % (section, result.stdout)) 807b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski else: 808b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski err = None 809b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski 810b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski # log something if the client failed AND never finished logging 8119285ddf562c698fda569390be6763170215663d7Dale Curtis if err and not self.is_client_job_finished(last_line): 812a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski self.log_unexpected_abort(stderr_redirector) 8130afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 814b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski if err: 815b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski raise err 816b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski else: 817b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski return stderr_redirector.last_line 8184600e343febea26110a920c598870f75ac394952jadmanski 8194600e343febea26110a920c598870f75ac394952jadmanski 820c035491ba24efea9e4343982f0a7c4b92e0a8c72jadmanski def _wait_for_reboot(self, old_boot_id): 821b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info("Client is rebooting") 822b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info("Waiting for client to halt") 823cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis if not self.host.wait_down(self.host.WAIT_DOWN_REBOOT_TIMEOUT, 824cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis old_boot_id=old_boot_id): 8254600e343febea26110a920c598870f75ac394952jadmanski err = "%s failed to shutdown after %d" 826cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis err %= (self.host.hostname, self.host.WAIT_DOWN_REBOOT_TIMEOUT) 8274600e343febea26110a920c598870f75ac394952jadmanski raise error.AutotestRunError(err) 828b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info("Client down, waiting for restart") 829cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis if not self.host.wait_up(self.host.DEFAULT_REBOOT_TIMEOUT): 8304600e343febea26110a920c598870f75ac394952jadmanski # since reboot failed 8314600e343febea26110a920c598870f75ac394952jadmanski # hardreset the machine once if possible 8324600e343febea26110a920c598870f75ac394952jadmanski # before failing this control file 8334600e343febea26110a920c598870f75ac394952jadmanski warning = "%s did not come back up, hard resetting" 8344600e343febea26110a920c598870f75ac394952jadmanski warning %= self.host.hostname 835b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.warning(warning) 8364600e343febea26110a920c598870f75ac394952jadmanski try: 8374600e343febea26110a920c598870f75ac394952jadmanski self.host.hardreset(wait=False) 838d99d3b27fb076377354fa388e0a81e97bda443b8mbligh except (AttributeError, error.AutoservUnsupportedError): 8394600e343febea26110a920c598870f75ac394952jadmanski warning = "Hard reset unsupported on %s" 8404600e343febea26110a920c598870f75ac394952jadmanski warning %= self.host.hostname 841b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.warning(warning) 8424600e343febea26110a920c598870f75ac394952jadmanski raise error.AutotestRunError("%s failed to boot after %ds" % 843cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis (self.host.hostname, 844cb7bfaf79c567031a93da7390fed5d57b4c7da85Dale Curtis self.host.DEFAULT_REBOOT_TIMEOUT)) 8454600e343febea26110a920c598870f75ac394952jadmanski self.host.reboot_followup() 8460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 8470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 8489285ddf562c698fda569390be6763170215663d7Dale Curtis def execute_control(self, timeout=None, client_disconnect_timeout=None): 8496bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski if not self.background: 8506bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski collector = log_collector(self.host, self.tag, self.results_dir) 8516bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski hostname = self.host.hostname 8526bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski remote_results = collector.client_results_dir 8536bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski local_results = collector.server_results_dir 8546bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski self.host.job.add_client_log(hostname, remote_results, 8556bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski local_results) 85652053636199c9dca7cd7a4df4d8913c866dff507jadmanski job_record_context = self.host.job.get_record_context() 8576bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski 8580afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski section = 0 8594600e343febea26110a920c598870f75ac394952jadmanski start_time = time.time() 8604600e343febea26110a920c598870f75ac394952jadmanski 861043e1132d1af7969a8f67871188ef34e050c2186jadmanski logger = client_logger(self.host, self.tag, self.results_dir) 8624600e343febea26110a920c598870f75ac394952jadmanski try: 8634600e343febea26110a920c598870f75ac394952jadmanski while not timeout or time.time() < start_time + timeout: 8644600e343febea26110a920c598870f75ac394952jadmanski if timeout: 8654600e343febea26110a920c598870f75ac394952jadmanski section_timeout = start_time + timeout - time.time() 8664600e343febea26110a920c598870f75ac394952jadmanski else: 8674600e343febea26110a920c598870f75ac394952jadmanski section_timeout = None 868c035491ba24efea9e4343982f0a7c4b92e0a8c72jadmanski boot_id = self.host.get_boot_id() 8694600e343febea26110a920c598870f75ac394952jadmanski last = self.execute_section(section, section_timeout, 8709285ddf562c698fda569390be6763170215663d7Dale Curtis logger, client_disconnect_timeout) 871b3c0c91ff9020a7259377e3d607c3aa0cae9c804mbligh if self.background: 872b3c0c91ff9020a7259377e3d607c3aa0cae9c804mbligh return 8734600e343febea26110a920c598870f75ac394952jadmanski section += 1 874b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski if self.is_client_job_finished(last): 875b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info("Client complete") 8764600e343febea26110a920c598870f75ac394952jadmanski return 877b264ed09d1300aed3a6353eaa9a8545ca9c03f35jadmanski elif self.is_client_job_rebooting(last): 87879ab928f306a222db195e0ef75806be071f8b3ddjadmanski try: 879c035491ba24efea9e4343982f0a7c4b92e0a8c72jadmanski self._wait_for_reboot(boot_id) 88079ab928f306a222db195e0ef75806be071f8b3ddjadmanski except error.AutotestRunError, e: 881043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.host.job.record("ABORT", None, "reboot", str(e)) 882043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.host.job.record("END ABORT", None, None, str(e)) 88379ab928f306a222db195e0ef75806be071f8b3ddjadmanski raise 8844600e343febea26110a920c598870f75ac394952jadmanski continue 8854600e343febea26110a920c598870f75ac394952jadmanski 886607428b251f437f924932a7b5386cbc06d63b22bbeeps # If a test fails without probable cause we try to bucket it's 887607428b251f437f924932a7b5386cbc06d63b22bbeeps # failure into one of 2 categories. If we can determine the 888607428b251f437f924932a7b5386cbc06d63b22bbeeps # current state of the device and it is suspicious, we close the 889607428b251f437f924932a7b5386cbc06d63b22bbeeps # status lines indicating a failure. If we either cannot 890607428b251f437f924932a7b5386cbc06d63b22bbeeps # determine the state of the device, or it appears totally 891607428b251f437f924932a7b5386cbc06d63b22bbeeps # healthy, we give up and abort. 892607428b251f437f924932a7b5386cbc06d63b22bbeeps try: 893607428b251f437f924932a7b5386cbc06d63b22bbeeps self._diagnose_dut(boot_id) 894607428b251f437f924932a7b5386cbc06d63b22bbeeps except AutotestDeviceError as e: 895607428b251f437f924932a7b5386cbc06d63b22bbeeps # The status lines of the test are pretty much tailed to 896607428b251f437f924932a7b5386cbc06d63b22bbeeps # our log, with indentation, from the client job on the DUT. 897607428b251f437f924932a7b5386cbc06d63b22bbeeps # So if the DUT goes down unexpectedly we'll end up with a 898607428b251f437f924932a7b5386cbc06d63b22bbeeps # malformed status log unless we manually unwind the status 899607428b251f437f924932a7b5386cbc06d63b22bbeeps # stack. Ideally we would want to write a nice wrapper like 900607428b251f437f924932a7b5386cbc06d63b22bbeeps # server_job methods run_reboot, run_group but they expect 901607428b251f437f924932a7b5386cbc06d63b22bbeeps # reboots and we don't. 902607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.job.record('FAIL', None, None, str(e)) 903607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.job.record('END FAIL', None, None) 904607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.job.record('END GOOD', None, None) 905b03ea9d5209ff5096556d6dc85b9a2cc1c38e417Dan Shi self.host.job.failed_with_device_error = True 906607428b251f437f924932a7b5386cbc06d63b22bbeeps return 907607428b251f437f924932a7b5386cbc06d63b22bbeeps except AutotestAbort as e: 908607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.job.record('ABORT', None, None, str(e)) 909607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.job.record('END ABORT', None, None) 910607428b251f437f924932a7b5386cbc06d63b22bbeeps 911607428b251f437f924932a7b5386cbc06d63b22bbeeps # give the client machine a chance to recover from a crash 912607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.wait_up( 913607428b251f437f924932a7b5386cbc06d63b22bbeeps self.host.HOURS_TO_WAIT_FOR_RECOVERY * 3600) 914607428b251f437f924932a7b5386cbc06d63b22bbeeps msg = ("Aborting - unexpected final status message from " 915607428b251f437f924932a7b5386cbc06d63b22bbeeps "client on %s: %s\n") % (self.host.hostname, last) 916607428b251f437f924932a7b5386cbc06d63b22bbeeps raise error.AutotestRunError(msg) 9174600e343febea26110a920c598870f75ac394952jadmanski finally: 918043e1132d1af7969a8f67871188ef34e050c2186jadmanski logger.close() 9196bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski if not self.background: 9206bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski collector.collect_client_job_results() 9214d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski collector.remove_redundant_client_logs() 922fc3da5bbdcd63840cd224282812c050c5520db11mbligh state_file = os.path.basename(self.remote_control_file 923fc3da5bbdcd63840cd224282812c050c5520db11mbligh + '.state') 924fc3da5bbdcd63840cd224282812c050c5520db11mbligh state_path = os.path.join(self.results_dir, state_file) 925fc3da5bbdcd63840cd224282812c050c5520db11mbligh self.host.job.postprocess_client_state(state_path) 9266bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski self.host.job.remove_client_log(hostname, remote_results, 9276bb32d74bb4c5c6ae8b76e01d58b8963bbf61849jadmanski local_results) 92852053636199c9dca7cd7a4df4d8913c866dff507jadmanski job_record_context.restore() 9290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 9300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # should only get here if we timed out 9310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski assert timeout 9320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise error.AutotestTimeoutError() 9330e4613bf6bc2fd5e5e3c327dbf568c305ede9681mbligh 934dcd57a85fc9233c4dc9f45da55338a2c09f5a37embligh 935043e1132d1af7969a8f67871188ef34e050c2186jadmanskiclass log_collector(object): 936043e1132d1af7969a8f67871188ef34e050c2186jadmanski def __init__(self, host, client_tag, results_dir): 937043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.host = host 938043e1132d1af7969a8f67871188ef34e050c2186jadmanski if not client_tag: 939043e1132d1af7969a8f67871188ef34e050c2186jadmanski client_tag = "default" 940043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.client_results_dir = os.path.join(host.get_autodir(), "results", 941043e1132d1af7969a8f67871188ef34e050c2186jadmanski client_tag) 942043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.server_results_dir = results_dir 943043e1132d1af7969a8f67871188ef34e050c2186jadmanski 944043e1132d1af7969a8f67871188ef34e050c2186jadmanski 945043e1132d1af7969a8f67871188ef34e050c2186jadmanski def collect_client_job_results(self): 946043e1132d1af7969a8f67871188ef34e050c2186jadmanski """ A method that collects all the current results of a running 947043e1132d1af7969a8f67871188ef34e050c2186jadmanski client job into the results dir. By default does nothing as no 948043e1132d1af7969a8f67871188ef34e050c2186jadmanski client job is running, but when running a client job you can override 949043e1132d1af7969a8f67871188ef34e050c2186jadmanski this with something that will actually do something. """ 950043e1132d1af7969a8f67871188ef34e050c2186jadmanski 951043e1132d1af7969a8f67871188ef34e050c2186jadmanski # make an effort to wait for the machine to come up 952043e1132d1af7969a8f67871188ef34e050c2186jadmanski try: 953043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.host.wait_up(timeout=30) 954043e1132d1af7969a8f67871188ef34e050c2186jadmanski except error.AutoservError: 955043e1132d1af7969a8f67871188ef34e050c2186jadmanski # don't worry about any errors, we'll try and 956043e1132d1af7969a8f67871188ef34e050c2186jadmanski # get the results anyway 957043e1132d1af7969a8f67871188ef34e050c2186jadmanski pass 958043e1132d1af7969a8f67871188ef34e050c2186jadmanski 959043e1132d1af7969a8f67871188ef34e050c2186jadmanski # Copy all dirs in default to results_dir 9601e1c41b1b4a1b97c0b7086b8430856ed45e064d3Gabe Black timer = autotest_stats.Timer('collect_client_job_results') 961209eefd78dbb3eb8f8575aa0e0f088fa0452abc0Dan Shi timer.start() 962043e1132d1af7969a8f67871188ef34e050c2186jadmanski try: 963043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.host.get_file(self.client_results_dir + '/', 9644556178ac97dc6160a31ac105f5d987b6c9933d8mbligh self.server_results_dir, preserve_symlinks=True) 965b03ea9d5209ff5096556d6dc85b9a2cc1c38e417Dan Shi 966209eefd78dbb3eb8f8575aa0e0f088fa0452abc0Dan Shi # Only report time used for successful get_file calls. 967209eefd78dbb3eb8f8575aa0e0f088fa0452abc0Dan Shi timer.stop(); 968043e1132d1af7969a8f67871188ef34e050c2186jadmanski except Exception: 969043e1132d1af7969a8f67871188ef34e050c2186jadmanski # well, don't stop running just because we couldn't get logs 970b18134f8faa7c5b4623760bc88650a65e70b2cacshoward e_msg = "Unexpected error copying test result logs, continuing ..." 971b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.error(e_msg) 972043e1132d1af7969a8f67871188ef34e050c2186jadmanski traceback.print_exc(file=sys.stdout) 973043e1132d1af7969a8f67871188ef34e050c2186jadmanski 974043e1132d1af7969a8f67871188ef34e050c2186jadmanski 9754d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski def remove_redundant_client_logs(self): 9764d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski """Remove client.*.log files in favour of client.*.DEBUG files.""" 9774d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski debug_dir = os.path.join(self.server_results_dir, 'debug') 9784d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski debug_files = [f for f in os.listdir(debug_dir) 9794d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski if re.search(r'^client\.\d+\.DEBUG$', f)] 9804d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski for debug_file in debug_files: 9814d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski log_file = debug_file.replace('DEBUG', 'log') 9824d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski log_file = os.path.join(debug_dir, log_file) 9834d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski if os.path.exists(log_file): 9844d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski os.remove(log_file) 9854d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 9864d03cf61d998e13d4a19a37ad92e4f0f39998322jadmanski 987043e1132d1af7969a8f67871188ef34e050c2186jadmanski# a file-like object for catching stderr from an autotest client and 988043e1132d1af7969a8f67871188ef34e050c2186jadmanski# extracting status logs from it 9893ee5d5c574d4e6d397aff36136101c657664c26bChris Sosaclass BaseClientLogger(object): 990043e1132d1af7969a8f67871188ef34e050c2186jadmanski """Partial file object to write to both stdout and 991043e1132d1af7969a8f67871188ef34e050c2186jadmanski the status log file. We only implement those methods 992043e1132d1af7969a8f67871188ef34e050c2186jadmanski utils.run() actually calls. 993043e1132d1af7969a8f67871188ef34e050c2186jadmanski """ 994043e1132d1af7969a8f67871188ef34e050c2186jadmanski status_parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$") 995043e1132d1af7969a8f67871188ef34e050c2186jadmanski test_complete_parser = re.compile(r"^AUTOTEST_TEST_COMPLETE:(.*)$") 996b1a5113d43c8f0a8b115c541d3e590d280e75565jadmanski fetch_package_parser = re.compile( 997b1a5113d43c8f0a8b115c541d3e590d280e75565jadmanski r"^AUTOTEST_FETCH_PACKAGE:([^:]*):([^:]*):(.*)$") 998043e1132d1af7969a8f67871188ef34e050c2186jadmanski extract_indent = re.compile(r"^(\t*).*$") 999efe4ebff90b023edf8e146b7bfa4a796d52eb1abjadmanski extract_timestamp = re.compile(r".*\ttimestamp=(\d+)\t.*$") 1000043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1001043e1132d1af7969a8f67871188ef34e050c2186jadmanski def __init__(self, host, tag, server_results_dir): 1002043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.host = host 1003043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.job = host.job 1004043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.log_collector = log_collector(host, tag, server_results_dir) 1005043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.leftover = "" 1006043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.last_line = "" 1007043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.logs = {} 1008efe4ebff90b023edf8e146b7bfa4a796d52eb1abjadmanski 1009efe4ebff90b023edf8e146b7bfa4a796d52eb1abjadmanski 1010043e1132d1af7969a8f67871188ef34e050c2186jadmanski def _process_log_dict(self, log_dict): 1011043e1132d1af7969a8f67871188ef34e050c2186jadmanski log_list = log_dict.pop("logs", []) 1012043e1132d1af7969a8f67871188ef34e050c2186jadmanski for key in sorted(log_dict.iterkeys()): 1013043e1132d1af7969a8f67871188ef34e050c2186jadmanski log_list += self._process_log_dict(log_dict.pop(key)) 1014043e1132d1af7969a8f67871188ef34e050c2186jadmanski return log_list 1015043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1016043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1017043e1132d1af7969a8f67871188ef34e050c2186jadmanski def _process_logs(self): 1018043e1132d1af7969a8f67871188ef34e050c2186jadmanski """Go through the accumulated logs in self.log and print them 1019043e1132d1af7969a8f67871188ef34e050c2186jadmanski out to stdout and the status log. Note that this processes 1020043e1132d1af7969a8f67871188ef34e050c2186jadmanski logs in an ordering where: 1021043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1022043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1) logs to different tags are never interleaved 1023043e1132d1af7969a8f67871188ef34e050c2186jadmanski 2) logs to x.y come before logs to x.y.z for all z 1024043e1132d1af7969a8f67871188ef34e050c2186jadmanski 3) logs to x.y come before x.z whenever y < z 1025043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1026043e1132d1af7969a8f67871188ef34e050c2186jadmanski Note that this will in general not be the same as the 1027043e1132d1af7969a8f67871188ef34e050c2186jadmanski chronological ordering of the logs. However, if a chronological 1028043e1132d1af7969a8f67871188ef34e050c2186jadmanski ordering is desired that one can be reconstructed from the 1029043e1132d1af7969a8f67871188ef34e050c2186jadmanski status log by looking at timestamp lines.""" 1030043e1132d1af7969a8f67871188ef34e050c2186jadmanski log_list = self._process_log_dict(self.logs) 10312a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski for entry in log_list: 10322a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self.job.record_entry(entry, log_in_subdir=False) 1033043e1132d1af7969a8f67871188ef34e050c2186jadmanski if log_list: 10342a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self.last_line = log_list[-1].render() 1035043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1036043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1037043e1132d1af7969a8f67871188ef34e050c2186jadmanski def _process_quoted_line(self, tag, line): 1038043e1132d1af7969a8f67871188ef34e050c2186jadmanski """Process a line quoted with an AUTOTEST_STATUS flag. If the 1039043e1132d1af7969a8f67871188ef34e050c2186jadmanski tag is blank then we want to push out all the data we've been 1040043e1132d1af7969a8f67871188ef34e050c2186jadmanski building up in self.logs, and then the newest line. If the 1041043e1132d1af7969a8f67871188ef34e050c2186jadmanski tag is not blank, then push the line into the logs for handling 1042043e1132d1af7969a8f67871188ef34e050c2186jadmanski later.""" 10432a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski entry = base_job.status_log_entry.parse(line) 10442a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski if entry is None: 10452a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski return # the line contains no status lines 1046043e1132d1af7969a8f67871188ef34e050c2186jadmanski if tag == "": 1047043e1132d1af7969a8f67871188ef34e050c2186jadmanski self._process_logs() 10482a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski self.job.record_entry(entry, log_in_subdir=False) 1049043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.last_line = line 1050043e1132d1af7969a8f67871188ef34e050c2186jadmanski else: 1051043e1132d1af7969a8f67871188ef34e050c2186jadmanski tag_parts = [int(x) for x in tag.split(".")] 1052043e1132d1af7969a8f67871188ef34e050c2186jadmanski log_dict = self.logs 1053043e1132d1af7969a8f67871188ef34e050c2186jadmanski for part in tag_parts: 1054043e1132d1af7969a8f67871188ef34e050c2186jadmanski log_dict = log_dict.setdefault(part, {}) 1055043e1132d1af7969a8f67871188ef34e050c2186jadmanski log_list = log_dict.setdefault("logs", []) 10562a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski log_list.append(entry) 1057043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1058043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1059f37df84b97538540185f91e2e5b62890ca407b3bjadmanski def _process_info_line(self, line): 1060f37df84b97538540185f91e2e5b62890ca407b3bjadmanski """Check if line is an INFO line, and if it is, interpret any control 1061f37df84b97538540185f91e2e5b62890ca407b3bjadmanski messages (e.g. enabling/disabling warnings) that it may contain.""" 1062f37df84b97538540185f91e2e5b62890ca407b3bjadmanski match = re.search(r"^\t*INFO\t----\t----(.*)\t[^\t]*$", line) 1063f37df84b97538540185f91e2e5b62890ca407b3bjadmanski if not match: 1064f37df84b97538540185f91e2e5b62890ca407b3bjadmanski return # not an INFO line 1065f37df84b97538540185f91e2e5b62890ca407b3bjadmanski for field in match.group(1).split('\t'): 1066f37df84b97538540185f91e2e5b62890ca407b3bjadmanski if field.startswith("warnings.enable="): 106716a7ff7af7fe2ae3093c5ad7452c516257184ffcjadmanski func = self.job.warning_manager.enable_warnings 1068f37df84b97538540185f91e2e5b62890ca407b3bjadmanski elif field.startswith("warnings.disable="): 106916a7ff7af7fe2ae3093c5ad7452c516257184ffcjadmanski func = self.job.warning_manager.disable_warnings 1070f37df84b97538540185f91e2e5b62890ca407b3bjadmanski else: 1071f37df84b97538540185f91e2e5b62890ca407b3bjadmanski continue 1072f37df84b97538540185f91e2e5b62890ca407b3bjadmanski warning_type = field.split("=", 1)[1] 107316a7ff7af7fe2ae3093c5ad7452c516257184ffcjadmanski func(warning_type) 1074f37df84b97538540185f91e2e5b62890ca407b3bjadmanski 1075f37df84b97538540185f91e2e5b62890ca407b3bjadmanski 1076043e1132d1af7969a8f67871188ef34e050c2186jadmanski def _process_line(self, line): 1077043e1132d1af7969a8f67871188ef34e050c2186jadmanski """Write out a line of data to the appropriate stream. Status 1078043e1132d1af7969a8f67871188ef34e050c2186jadmanski lines sent by autotest will be prepended with 1079043e1132d1af7969a8f67871188ef34e050c2186jadmanski "AUTOTEST_STATUS", and all other lines are ssh error 1080043e1132d1af7969a8f67871188ef34e050c2186jadmanski messages.""" 1081043e1132d1af7969a8f67871188ef34e050c2186jadmanski status_match = self.status_parser.search(line) 1082043e1132d1af7969a8f67871188ef34e050c2186jadmanski test_complete_match = self.test_complete_parser.search(line) 1083b1a5113d43c8f0a8b115c541d3e590d280e75565jadmanski fetch_package_match = self.fetch_package_parser.search(line) 1084043e1132d1af7969a8f67871188ef34e050c2186jadmanski if status_match: 1085043e1132d1af7969a8f67871188ef34e050c2186jadmanski tag, line = status_match.groups() 1086f37df84b97538540185f91e2e5b62890ca407b3bjadmanski self._process_info_line(line) 1087043e1132d1af7969a8f67871188ef34e050c2186jadmanski self._process_quoted_line(tag, line) 1088043e1132d1af7969a8f67871188ef34e050c2186jadmanski elif test_complete_match: 1089fcc0d5d352f3b6e031a2080e5474bd3a70ef2ec7jadmanski self._process_logs() 1090043e1132d1af7969a8f67871188ef34e050c2186jadmanski fifo_path, = test_complete_match.groups() 1091060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh try: 1092060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh self.log_collector.collect_client_job_results() 1093060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh self.host.run("echo A > %s" % fifo_path) 1094060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh except Exception: 1095060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh msg = "Post-test log collection failed, continuing anyway" 1096060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh logging.exception(msg) 1097b1a5113d43c8f0a8b115c541d3e590d280e75565jadmanski elif fetch_package_match: 1098b1a5113d43c8f0a8b115c541d3e590d280e75565jadmanski pkg_name, dest_path, fifo_path = fetch_package_match.groups() 1099ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski serve_packages = global_config.global_config.get_config_value( 1100ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski "PACKAGES", "serve_packages_from_autoserv", type=bool) 1101ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski if serve_packages and pkg_name.endswith(".tar.bz2"): 1102ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski try: 1103ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski self._send_tarball(pkg_name, dest_path) 1104ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski except Exception: 1105ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski msg = "Package tarball creation failed, continuing anyway" 1106ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski logging.exception(msg) 1107060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh try: 1108060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh self.host.run("echo B > %s" % fifo_path) 1109060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh except Exception: 1110060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh msg = "Package tarball installation failed, continuing anyway" 1111060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh logging.exception(msg) 1112043e1132d1af7969a8f67871188ef34e050c2186jadmanski else: 1113b18134f8faa7c5b4623760bc88650a65e70b2cacshoward logging.info(line) 1114043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1115043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1116ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski def _send_tarball(self, pkg_name, remote_dest): 1117ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski name, pkg_type = self.job.pkgmgr.parse_tarball_name(pkg_name) 1118ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski src_dirs = [] 1119ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski if pkg_type == 'test': 11201f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh for test_dir in ['site_tests', 'tests']: 11211f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh src_dir = os.path.join(self.job.clientdir, test_dir, name) 11221f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh if os.path.exists(src_dir): 11231f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh src_dirs += [src_dir] 11241f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh if autoserv_prebuild: 11251f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh prebuild.setup(self.job.clientdir, src_dir) 11261f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh break 1127ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski elif pkg_type == 'profiler': 1128ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski src_dirs += [os.path.join(self.job.clientdir, 'profilers', name)] 11291f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh if autoserv_prebuild: 11301f572e5efbaaa8bc0d1246123ab897607c9a011bmbligh prebuild.setup(self.job.clientdir, src_dir) 1131ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski elif pkg_type == 'dep': 1132ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski src_dirs += [os.path.join(self.job.clientdir, 'deps', name)] 1133ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski elif pkg_type == 'client': 1134ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski return # you must already have a client to hit this anyway 1135ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski else: 1136ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski return # no other types are supported 1137ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski 1138ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski # iterate over src_dirs until we find one that exists, then tar it 1139ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski for src_dir in src_dirs: 1140ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski if os.path.exists(src_dir): 1141ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski try: 1142ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski logging.info('Bundling %s into %s', src_dir, pkg_name) 1143ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski temp_dir = autotemp.tempdir(unique_id='autoserv-packager', 1144ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski dir=self.job.tmpdir) 1145ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski tarball_path = self.job.pkgmgr.tar_package( 1146bccad480b242b63f93f35b55a0e77517e3a76febmbligh pkg_name, src_dir, temp_dir.name, " .") 1147ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski self.host.send_file(tarball_path, remote_dest) 1148ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski finally: 1149ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski temp_dir.clean() 1150ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski return 1151ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski 1152ede7e248313e4670e69b2c9c28583cbc91d6f6bcjadmanski 115391d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski def log_warning(self, msg, warning_type): 11546dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski """Injects a WARN message into the current status logging stream.""" 115591d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski timestamp = int(time.time()) 115691d56a9bbeb29f4f80cd36072254977fe4530eaejadmanski if self.job.warning_manager.is_valid(timestamp, warning_type): 1157d656d56473f50b9c1a9f5e2b2f5a9472181ee342Eric Li self.job.record('WARN', None, None, msg) 11586dadd8360710eb88fb80a9ce85433e13e8dcabe9jadmanski 1159043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1160043e1132d1af7969a8f67871188ef34e050c2186jadmanski def write(self, data): 11612a89dac0b6e319ef58d41c7a591c3d88cf6dd8a1jadmanski # now start processing the existing buffer and the new data 1162043e1132d1af7969a8f67871188ef34e050c2186jadmanski data = self.leftover + data 1163060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh lines = data.split('\n') 1164060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh processed_lines = 0 1165060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh try: 1166060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh # process all the buffered data except the last line 1167060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh # ignore the last line since we may not have all of it yet 1168060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh for line in lines[:-1]: 1169060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh self._process_line(line) 1170060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh processed_lines += 1 1171060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh finally: 1172060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh # save any unprocessed lines for future processing 1173060c471022558cd91a03b6e8da5a98ce35cd4b54mbligh self.leftover = '\n'.join(lines[processed_lines:]) 1174043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1175043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1176043e1132d1af7969a8f67871188ef34e050c2186jadmanski def flush(self): 1177043e1132d1af7969a8f67871188ef34e050c2186jadmanski sys.stdout.flush() 1178043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1179043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1180a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski def flush_all_buffers(self): 1181043e1132d1af7969a8f67871188ef34e050c2186jadmanski if self.leftover: 1182043e1132d1af7969a8f67871188ef34e050c2186jadmanski self._process_line(self.leftover) 1183a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski self.leftover = "" 1184043e1132d1af7969a8f67871188ef34e050c2186jadmanski self._process_logs() 1185043e1132d1af7969a8f67871188ef34e050c2186jadmanski self.flush() 1186043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1187043e1132d1af7969a8f67871188ef34e050c2186jadmanski 1188a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski def close(self): 1189a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski self.flush_all_buffers() 1190a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski 1191a61edad29fd9f81e0c20190a16d0ea9cb9f0b9cdjadmanski 1192a700772e70548b10721ba709c7ec5194bb6d7597mblighSiteAutotest = client_utils.import_site_class( 1193a700772e70548b10721ba709c7ec5194bb6d7597mbligh __file__, "autotest_lib.server.site_autotest", "SiteAutotest", 1194a700772e70548b10721ba709c7ec5194bb6d7597mbligh BaseAutotest) 1195d8b39259ca4f011cea6f804e5b407262cad57057mbligh 1196ad812bf3cf46bd9da1871401042b2b0abf1a4220showard 119769bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski_SiteRun = client_utils.import_site_class( 119869bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski __file__, "autotest_lib.server.site_autotest", "_SiteRun", _BaseRun) 119969bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski 120069bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski 12013ee5d5c574d4e6d397aff36136101c657664c26bChris SosaSiteClientLogger = client_utils.import_site_class( 12023ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa __file__, "autotest_lib.server.site_autotest", "SiteClientLogger", 12033ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa BaseClientLogger) 12043ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa 12053ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa 1206d8b39259ca4f011cea6f804e5b407262cad57057mblighclass Autotest(SiteAutotest): 12070afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski pass 1208c6136e92bb5aef1acda979fd303fc855cf704434jadmanski 1209c6136e92bb5aef1acda979fd303fc855cf704434jadmanski 12103ee5d5c574d4e6d397aff36136101c657664c26bChris Sosaclass client_logger(SiteClientLogger): 12113ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa pass 12123ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa 12133ee5d5c574d4e6d397aff36136101c657664c26bChris Sosa 121469bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanskiclass _Run(_SiteRun): 121569bdaacdf2b8161ae95dded60ed6eedbeb5f8f47jadmanski pass 1216