abstract_ssh.py revision e863384bf392d8a1b7326adc60483f4eacf7ab85
1import os, time, types, socket, shutil, glob, logging, traceback 2from autotest_lib.client.common_lib import error, logging_manager 3from autotest_lib.server import utils, autotest 4from autotest_lib.server.hosts import remote 5 6 7def make_ssh_command(user="root", port=22, opts='', connect_timeout=30): 8 base_command = ("/usr/bin/ssh -a -x %s -o BatchMode=yes " 9 "-o ConnectTimeout=%d -o ServerAliveInterval=300 " 10 "-l %s -p %d") 11 assert isinstance(connect_timeout, (int, long)) 12 assert connect_timeout > 0 # can't disable the timeout 13 return base_command % (opts, connect_timeout, user, port) 14 15 16# import site specific Host class 17SiteHost = utils.import_site_class( 18 __file__, "autotest_lib.server.hosts.site_host", "SiteHost", 19 remote.RemoteHost) 20 21 22class AbstractSSHHost(SiteHost): 23 """ 24 This class represents a generic implementation of most of the 25 framework necessary for controlling a host via ssh. It implements 26 almost all of the abstract Host methods, except for the core 27 Host.run method. 28 """ 29 30 def _initialize(self, hostname, user="root", port=22, password="", 31 *args, **dargs): 32 super(AbstractSSHHost, self)._initialize(hostname=hostname, 33 *args, **dargs) 34 self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0] 35 self.user = user 36 self.port = port 37 self.password = password 38 39 40 def _encode_remote_paths(self, paths, escape=True): 41 """ 42 Given a list of file paths, encodes it as a single remote path, in 43 the style used by rsync and scp. 44 """ 45 if escape: 46 paths = [utils.scp_remote_escape(path) for path in paths] 47 return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths)) 48 49 50 def _make_rsync_cmd(self, sources, dest, delete_dest, preserve_symlinks): 51 """ 52 Given a list of source paths and a destination path, produces the 53 appropriate rsync command for copying them. Remote paths must be 54 pre-encoded. 55 """ 56 ssh_cmd = make_ssh_command(self.user, self.port) 57 if delete_dest: 58 delete_flag = "--delete" 59 else: 60 delete_flag = "" 61 if preserve_symlinks: 62 symlink_flag = "" 63 else: 64 symlink_flag = "-L" 65 command = "rsync %s %s --timeout=1800 --rsh='%s' -az %s %s" 66 return command % (symlink_flag, delete_flag, ssh_cmd, 67 " ".join(sources), dest) 68 69 70 def _make_scp_cmd(self, sources, dest): 71 """ 72 Given a list of source paths and a destination path, produces the 73 appropriate scp command for encoding it. Remote paths must be 74 pre-encoded. 75 """ 76 command = "scp -rq -P %d %s '%s'" 77 return command % (self.port, " ".join(sources), dest) 78 79 80 def _make_rsync_compatible_globs(self, path, is_local): 81 """ 82 Given an rsync-style path, returns a list of globbed paths 83 that will hopefully provide equivalent behaviour for scp. Does not 84 support the full range of rsync pattern matching behaviour, only that 85 exposed in the get/send_file interface (trailing slashes). 86 87 The is_local param is flag indicating if the paths should be 88 interpreted as local or remote paths. 89 """ 90 91 # non-trailing slash paths should just work 92 if len(path) == 0 or path[-1] != "/": 93 return [path] 94 95 # make a function to test if a pattern matches any files 96 if is_local: 97 def glob_matches_files(path, pattern): 98 return len(glob.glob(path + pattern)) > 0 99 else: 100 def glob_matches_files(path, pattern): 101 result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), 102 pattern), 103 stdout_tee=None, ignore_status=True) 104 return result.exit_status == 0 105 106 # take a set of globs that cover all files, and see which are needed 107 patterns = ["*", ".[!.]*"] 108 patterns = [p for p in patterns if glob_matches_files(path, p)] 109 110 # convert them into a set of paths suitable for the commandline 111 if is_local: 112 return ["\"%s\"%s" % (utils.sh_escape(path), pattern) 113 for pattern in patterns] 114 else: 115 return [utils.scp_remote_escape(path) + pattern 116 for pattern in patterns] 117 118 119 def _make_rsync_compatible_source(self, source, is_local): 120 """ 121 Applies the same logic as _make_rsync_compatible_globs, but 122 applies it to an entire list of sources, producing a new list of 123 sources, properly quoted. 124 """ 125 return sum((self._make_rsync_compatible_globs(path, is_local) 126 for path in source), []) 127 128 129 def _set_umask_perms(self, dest): 130 """ 131 Given a destination file/dir (recursively) set the permissions on 132 all the files and directories to the max allowed by running umask. 133 """ 134 135 # now this looks strange but I haven't found a way in Python to _just_ 136 # get the umask, apparently the only option is to try to set it 137 umask = os.umask(0) 138 os.umask(umask) 139 140 max_privs = 0777 & ~umask 141 142 def set_file_privs(filename): 143 file_stat = os.stat(filename) 144 145 file_privs = max_privs 146 # if the original file permissions do not have at least one 147 # executable bit then do not set it anywhere 148 if not file_stat.st_mode & 0111: 149 file_privs &= ~0111 150 151 os.chmod(filename, file_privs) 152 153 # try a bottom-up walk so changes on directory permissions won't cut 154 # our access to the files/directories inside it 155 for root, dirs, files in os.walk(dest, topdown=False): 156 # when setting the privileges we emulate the chmod "X" behaviour 157 # that sets to execute only if it is a directory or any of the 158 # owner/group/other already has execute right 159 for dirname in dirs: 160 os.chmod(os.path.join(root, dirname), max_privs) 161 162 for filename in files: 163 set_file_privs(os.path.join(root, filename)) 164 165 166 # now set privs for the dest itself 167 if os.path.isdir(dest): 168 os.chmod(dest, max_privs) 169 else: 170 set_file_privs(dest) 171 172 173 def get_file(self, source, dest, delete_dest=False, preserve_perm=True, 174 preserve_symlinks=False): 175 """ 176 Copy files from the remote host to a local path. 177 178 Directories will be copied recursively. 179 If a source component is a directory with a trailing slash, 180 the content of the directory will be copied, otherwise, the 181 directory itself and its content will be copied. This 182 behavior is similar to that of the program 'rsync'. 183 184 Args: 185 source: either 186 1) a single file or directory, as a string 187 2) a list of one or more (possibly mixed) 188 files or directories 189 dest: a file or a directory (if source contains a 190 directory or more than one element, you must 191 supply a directory dest) 192 delete_dest: if this is true, the command will also clear 193 out any old files at dest that are not in the 194 source 195 preserve_perm: tells get_file() to try to preserve the sources 196 permissions on files and dirs 197 preserve_symlinks: try to preserve symlinks instead of 198 transforming them into files/dirs on copy 199 200 Raises: 201 AutoservRunError: the scp command failed 202 """ 203 if isinstance(source, basestring): 204 source = [source] 205 dest = os.path.abspath(dest) 206 207 try: 208 remote_source = self._encode_remote_paths(source) 209 local_dest = utils.sh_escape(dest) 210 rsync = self._make_rsync_cmd([remote_source], local_dest, 211 delete_dest, preserve_symlinks) 212 utils.run(rsync) 213 except error.CmdError, e: 214 # scp has no equivalent to --delete, just drop the entire dest dir 215 if delete_dest and os.path.isdir(dest): 216 shutil.rmtree(dest) 217 os.mkdir(dest) 218 219 remote_source = self._make_rsync_compatible_source(source, False) 220 if remote_source: 221 # _make_rsync_compatible_source() already did the escaping 222 remote_source = self._encode_remote_paths(remote_source, 223 escape=False) 224 local_dest = utils.sh_escape(dest) 225 scp = self._make_scp_cmd([remote_source], local_dest) 226 try: 227 utils.run(scp) 228 except error.CmdError, e: 229 raise error.AutoservRunError(e.args[0], e.args[1]) 230 231 if not preserve_perm: 232 # we have no way to tell scp to not try to preserve the 233 # permissions so set them after copy instead. 234 # for rsync we could use "--no-p --chmod=ugo=rwX" but those 235 # options are only in very recent rsync versions 236 self._set_umask_perms(dest) 237 238 239 def send_file(self, source, dest, delete_dest=False, 240 preserve_symlinks=False): 241 """ 242 Copy files from a local path to the remote host. 243 244 Directories will be copied recursively. 245 If a source component is a directory with a trailing slash, 246 the content of the directory will be copied, otherwise, the 247 directory itself and its content will be copied. This 248 behavior is similar to that of the program 'rsync'. 249 250 Args: 251 source: either 252 1) a single file or directory, as a string 253 2) a list of one or more (possibly mixed) 254 files or directories 255 dest: a file or a directory (if source contains a 256 directory or more than one element, you must 257 supply a directory dest) 258 delete_dest: if this is true, the command will also clear 259 out any old files at dest that are not in the 260 source 261 preserve_symlinks: controls if symlinks on the source will be 262 copied as such on the destination or transformed into the 263 referenced file/directory 264 265 Raises: 266 AutoservRunError: the scp command failed 267 """ 268 if isinstance(source, basestring): 269 source = [source] 270 remote_dest = self._encode_remote_paths([dest]) 271 272 try: 273 local_sources = [utils.sh_escape(path) for path in source] 274 rsync = self._make_rsync_cmd(local_sources, remote_dest, 275 delete_dest, preserve_symlinks) 276 utils.run(rsync) 277 except error.CmdError, e: 278 # scp has no equivalent to --delete, just drop the entire dest dir 279 if delete_dest: 280 is_dir = self.run("ls -d %s/" % dest, 281 ignore_status=True).exit_status == 0 282 if is_dir: 283 cmd = "rm -rf %s && mkdir %s" 284 cmd %= (dest, dest) 285 self.run(cmd) 286 287 local_sources = self._make_rsync_compatible_source(source, True) 288 if local_sources: 289 scp = self._make_scp_cmd(local_sources, remote_dest) 290 try: 291 utils.run(scp) 292 except error.CmdError, e: 293 raise error.AutoservRunError(e.args[0], e.args[1]) 294 295 296 def ssh_ping(self, timeout=60): 297 try: 298 self.run("true", timeout=timeout, connect_timeout=timeout) 299 except error.AutoservSSHTimeout: 300 msg = "Host (ssh) verify timed out (timeout = %d)" % timeout 301 raise error.AutoservSSHTimeout(msg) 302 except error.AutoservSshPermissionDeniedError: 303 #let AutoservSshPermissionDeniedError be visible to the callers 304 raise 305 except error.AutoservRunError, e: 306 # convert the generic AutoservRunError into something more 307 # specific for this context 308 raise error.AutoservSshPingHostError(e.description + '\n' + 309 repr(e.result_obj)) 310 311 312 def is_up(self): 313 """ 314 Check if the remote host is up. 315 316 Returns: 317 True if the remote host is up, False otherwise 318 """ 319 try: 320 self.ssh_ping() 321 except error.AutoservError: 322 return False 323 else: 324 return True 325 326 327 def wait_up(self, timeout=None): 328 """ 329 Wait until the remote host is up or the timeout expires. 330 331 In fact, it will wait until an ssh connection to the remote 332 host can be established, and getty is running. 333 334 Args: 335 timeout: time limit in seconds before returning even 336 if the host is not up. 337 338 Returns: 339 True if the host was found to be up, False otherwise 340 """ 341 if timeout: 342 end_time = time.time() + timeout 343 344 while not timeout or time.time() < end_time: 345 if self.is_up(): 346 try: 347 if self.are_wait_up_processes_up(): 348 return True 349 except error.AutoservError: 350 pass 351 time.sleep(1) 352 353 return False 354 355 356 def wait_down(self, timeout=None, warning_timer=None): 357 """ 358 Wait until the remote host is down or the timeout expires. 359 360 In fact, it will wait until an ssh connection to the remote 361 host fails. 362 363 Args: 364 timeout: time limit in seconds before returning even 365 if the host is still up. 366 warning_timer: time limit in seconds that will generate 367 a warning if the host is not down yet. 368 369 Returns: 370 True if the host was found to be down, False otherwise 371 """ 372 current_time = time.time() 373 if timeout: 374 end_time = current_time + timeout 375 376 if warning_timer: 377 warn_time = current_time + warning_timer 378 379 while not timeout or current_time < end_time: 380 if not self.is_up(): 381 return True 382 383 if warning_timer and current_time > warn_time: 384 self.record("WARN", None, "shutdown", 385 "Shutdown took longer than %ds" % warning_timer) 386 # Print the warning only once. 387 warning_timer = None 388 # If a machine is stuck switching runlevels 389 # This may cause the machine to reboot. 390 self.run('kill -HUP 1', ignore_status=True) 391 392 time.sleep(1) 393 current_time = time.time() 394 395 return False 396 397 398 # tunable constants for the verify & repair code 399 AUTOTEST_GB_DISKSPACE_REQUIRED = 20 400 401 402 def verify_connectivity(self): 403 super(AbstractSSHHost, self).verify_connectivity() 404 405 logging.info('Pinging host ' + self.hostname) 406 self.ssh_ping() 407 logging.info("Host (ssh) %s is alive", self.hostname) 408 409 if self.is_shutting_down(): 410 raise error.AutoservHostIsShuttingDownError("Host is shutting down") 411 412 413 def verify_software(self): 414 super(AbstractSSHHost, self).verify_software() 415 try: 416 self.check_diskspace(autotest.Autotest.get_install_dir(self), 417 self.AUTOTEST_GB_DISKSPACE_REQUIRED) 418 except error.AutoservHostError: 419 raise # only want to raise if it's a space issue 420 except autotest.AutodirNotFoundError: 421 # autotest dir may not exist, etc. ignore 422 logging.debug('autodir space check exception, this is probably ' 423 'safe to ignore\n' + traceback.format_exc()) 424