abstract_ssh.py revision 6369cf20d9c4619c0e1a88464b87585b3a2b8b78
1import os, time, types, socket 2from autotest_lib.client.common_lib import error 3from autotest_lib.server import utils 4from autotest_lib.server.hosts import site_host 5 6 7def make_ssh_command(user="root", port=22, opts='', connect_timeout=30): 8 base_command = ("/usr/bin/ssh -a -x %s -o BatchMode=yes " 9 "-o ConnectTimeout=%d -o ServerAliveInterval=300 " 10 "-o GSSAPIAuthentication=no -o GSSAPIKeyExchange=no " 11 "-l %s -p %d") 12 assert isinstance(connect_timeout, (int, long)) 13 assert connect_timeout > 0 # can't disable the timeout 14 return base_command % (opts, connect_timeout, user, port) 15 16 17class AbstractSSHHost(site_host.SiteHost): 18 """ This class represents a generic implementation of most of the 19 framework necessary for controlling a host via ssh. It implements 20 almost all of the abstract Host methods, except for the core 21 Host.run method. """ 22 23 def _initialize(self, hostname, user="root", port=22, password="", 24 *args, **dargs): 25 super(AbstractSSHHost, self)._initialize(hostname=hostname, 26 *args, **dargs) 27 self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0] 28 self.user = user 29 self.port = port 30 self.password = password 31 32 33 def _copy_files(self, sources, dest, delete_dest): 34 """ 35 Copy files from one machine to another. 36 37 This is for internal use by other methods that intend to move 38 files between machines. It expects a list of source files and 39 a destination (a filename if the source is a single file, a 40 destination otherwise). The names must already be 41 pre-processed into the appropriate rsync/scp friendly 42 format (%s@%s:%s). 43 """ 44 45 print '_copy_files: copying %s to %s' % (sources, dest) 46 try: 47 ssh = make_ssh_command(self.user, self.port) 48 if delete_dest: 49 delete_flag = "--delete" 50 else: 51 delete_flag = "" 52 command = "rsync -L %s --rsh='%s' -az %s %s" 53 command %= (delete_flag, ssh, " ".join(sources), dest) 54 utils.run(command) 55 except Exception, e: 56 print "warning: rsync failed with: %s" % e 57 print "attempting to copy with scp instead" 58 try: 59 if delete_dest: 60 dest_path = dest.split(":", 1)[1] 61 is_dir = self.run("ls -d %s/" % dest_path, 62 ignore_status=True).exit_status == 0 63 if is_dir: 64 cmd = "rm -rf %s && mkdir %s" 65 cmd %= (dest_path, dest_path) 66 self.run(cmd) 67 command = "scp -rpq -P %d %s '%s'" 68 command %= (self.port, ' '.join(sources), dest) 69 utils.run(command) 70 except error.CmdError, cmderr: 71 raise error.AutoservRunError(cmderr.args[0], cmderr.args[1]) 72 73 74 def get_file(self, source, dest, delete_dest=False): 75 """ 76 Copy files from the remote host to a local path. 77 78 Directories will be copied recursively. 79 If a source component is a directory with a trailing slash, 80 the content of the directory will be copied, otherwise, the 81 directory itself and its content will be copied. This 82 behavior is similar to that of the program 'rsync'. 83 84 Args: 85 source: either 86 1) a single file or directory, as a string 87 2) a list of one or more (possibly mixed) 88 files or directories 89 dest: a file or a directory (if source contains a 90 directory or more than one element, you must 91 supply a directory dest) 92 delete_dest: if this is true, the command will also clear 93 out any old files at dest that are not in the 94 source 95 96 Raises: 97 AutoservRunError: the scp command failed 98 """ 99 if isinstance(source, basestring): 100 source = [source] 101 102 processed_source = [] 103 for path in source: 104 if path.endswith('/'): 105 format_string = '%s@%s:"%s*"' 106 else: 107 format_string = '%s@%s:"%s"' 108 entry = format_string % (self.user, self.hostname, 109 utils.scp_remote_escape(path)) 110 processed_source.append(entry) 111 112 processed_dest = utils.sh_escape(os.path.abspath(dest)) 113 if os.path.isdir(dest): 114 processed_dest += "/" 115 116 self._copy_files(processed_source, processed_dest, delete_dest) 117 118 119 def send_file(self, source, dest, delete_dest=False): 120 """ 121 Copy files from a local path to the remote host. 122 123 Directories will be copied recursively. 124 If a source component is a directory with a trailing slash, 125 the content of the directory will be copied, otherwise, the 126 directory itself and its content will be copied. This 127 behavior is similar to that of the program 'rsync'. 128 129 Args: 130 source: either 131 1) a single file or directory, as a string 132 2) a list of one or more (possibly mixed) 133 files or directories 134 dest: a file or a directory (if source contains a 135 directory or more than one element, you must 136 supply a directory dest) 137 delete_dest: if this is true, the command will also clear 138 out any old files at dest that are not in the 139 source 140 141 Raises: 142 AutoservRunError: the scp command failed 143 """ 144 if isinstance(source, basestring): 145 source = [source] 146 147 processed_source = [] 148 for path in source: 149 if path.endswith('/'): 150 format_string = '"%s/"*' 151 else: 152 format_string = '"%s"' 153 entry = format_string % (utils.sh_escape(os.path.abspath(path)),) 154 processed_source.append(entry) 155 156 remote_dest = '%s@%s:"%s"' % (self.user, self.hostname, 157 utils.scp_remote_escape(dest)) 158 159 self._copy_files(processed_source, remote_dest, delete_dest) 160 self.run('find "%s" -type d -print0 | xargs -0r chmod o+rx' % dest) 161 self.run('find "%s" -type f -print0 | xargs -0r chmod o+r' % dest) 162 if self.target_file_owner: 163 self.run('chown -R %s %s' % (self.target_file_owner, dest)) 164 165 166 def ssh_ping(self, timeout=60): 167 try: 168 self.run("true", timeout=timeout, connect_timeout=timeout) 169 except error.AutoservSSHTimeout: 170 msg = "ssh ping timed out (timeout = %d)" % timeout 171 raise error.AutoservSSHTimeout(msg) 172 except error.AutoservRunError, e: 173 msg = "command true failed in ssh ping" 174 raise error.AutoservRunError(msg, e.result_obj) 175 176 177 def is_up(self): 178 """ 179 Check if the remote host is up. 180 181 Returns: 182 True if the remote host is up, False otherwise 183 """ 184 try: 185 self.ssh_ping() 186 except error.AutoservError: 187 return False 188 else: 189 return True 190 191 192 def wait_up(self, timeout=None): 193 """ 194 Wait until the remote host is up or the timeout expires. 195 196 In fact, it will wait until an ssh connection to the remote 197 host can be established, and getty is running. 198 199 Args: 200 timeout: time limit in seconds before returning even 201 if the host is not up. 202 203 Returns: 204 True if the host was found to be up, False otherwise 205 """ 206 if timeout: 207 end_time = time.time() + timeout 208 209 while not timeout or time.time() < end_time: 210 if self.is_up(): 211 try: 212 if self.are_wait_up_processes_up(): 213 return True 214 except error.AutoservError: 215 pass 216 time.sleep(1) 217 218 return False 219 220 221 def wait_down(self, timeout=None): 222 """ 223 Wait until the remote host is down or the timeout expires. 224 225 In fact, it will wait until an ssh connection to the remote 226 host fails. 227 228 Args: 229 timeout: time limit in seconds before returning even 230 if the host is not up. 231 232 Returns: 233 True if the host was found to be down, False otherwise 234 """ 235 if timeout: 236 end_time = time.time() + timeout 237 238 while not timeout or time.time() < end_time: 239 if not self.is_up(): 240 return True 241 time.sleep(1) 242 243 return False 244 245 # tunable constants for the verify & repair code 246 AUTOTEST_GB_DISKSPACE_REQUIRED = 20 247 HOURS_TO_WAIT_FOR_RECOVERY = 2.5 248 249 def verify(self): 250 super(AbstractSSHHost, self).verify() 251 252 print 'Pinging host ' + self.hostname 253 self.ssh_ping() 254 255 try: 256 autodir = autotest._get_autodir(self) 257 if autodir: 258 print 'Checking diskspace for %s on %s' % (self.hostname, 259 autodir) 260 self.check_diskspace(autodir, 261 self.AUTOTEST_GB_DISKSPACE_REQUIRED) 262 except error.AutoservHostError: 263 raise # only want to raise if it's a space issue 264 except Exception: 265 pass # autotest dir may not exist, etc. ignore 266 267 268 def repair_filesystem_only(self): 269 super(AbstractSSHHost, self).repair_filesystem_only() 270 self.wait_up(int(self.HOURS_TO_WAIT_FOR_RECOVERY * 3600)) 271 self.reboot() 272 273 274 def repair_full(self): 275 super(AbstractSSHHost, self).repair_full() 276 try: 277 self.repair_filesystem_only() 278 self.verify() 279 except Exception: 280 # the filesystem-only repair failed, try something more drastic 281 print "Filesystem-only repair failed" 282 traceback.print_exc() 283 try: 284 self.machine_install() 285 except NotImplementedError, e: 286 sys.stderr.write(str(e) + "\n\n") 287