abstract_ssh.py revision 89e258d442be542143b1c79fcd5a990bde701f34 
1import os, time, types
2from autotest_lib.client.common_lib import error
3from autotest_lib.server import utils
4from autotest_lib.server.hosts import site_host
5
6
7def make_ssh_command(user="root", port=22, opts='', connect_timeout=30):
8    base_command = ("/usr/bin/ssh -a -x %s -o BatchMode=yes "
9                    "-o ConnectTimeout=%d -o ServerAliveInterval=300 "
10                    "-o GSSAPIAuthentication=no -o GSSAPIKeyExchange=no "
11                    "-l %s -p %d")
12    assert isinstance(connect_timeout, (int, long))
13    assert connect_timeout > 0 # can't disable the timeout
14    return base_command % (opts, connect_timeout, user, port)
15
16
17class AbstractSSHHost(site_host.SiteHost):
18    """ This class represents a generic implementation of most of the
19    framework necessary for controlling a host via ssh. It implements
20    almost all of the abstract Host methods, except for the core
21    Host.run method. """
22
23    def _initialize(self, hostname, user="root", port=22, password="",
24                    *args, **dargs):
25        super(AbstractSSHHost, self)._initialize(hostname=hostname,
26                                                 *args, **dargs)
27
28        self.user = user
29        self.port = port
30        self.password = password
31
32
33    def _copy_files(self, sources, dest, delete_dest):
34        """
35        Copy files from one machine to another.
36
37        This is for internal use by other methods that intend to move
38        files between machines. It expects a list of source files and
39        a destination (a filename if the source is a single file, a
40        destination otherwise). The names must already be
41        pre-processed into the appropriate rsync/scp friendly
42        format (%s@%s:%s).
43        """
44
45        print '_copy_files: copying %s to %s' % (sources, dest)
46        try:
47            ssh = make_ssh_command(self.user, self.port)
48            if delete_dest:
49                delete_flag = "--delete"
50            else:
51                delete_flag = ""
52            command = "rsync -L %s --rsh='%s' -az %s %s"
53            command %= (delete_flag, ssh, " ".join(sources), dest)
54            utils.run(command)
55        except Exception, e:
56            print "warning: rsync failed with: %s" % e
57            print "attempting to copy with scp instead"
58            try:
59                if delete_dest:
60                    dest_path = dest.split(":", 1)[1]
61                    is_dir = self.run("ls -d %s/" % dest_path,
62                                      ignore_status=True).exit_status == 0
63                    if is_dir:
64                        cmd = "rm -rf %s && mkdir %s"
65                        cmd %= (dest_path, dest_path)
66                        self.run(cmd)
67                command = "scp -rpq -P %d %s '%s'"
68                command %= (self.port, ' '.join(sources), dest)
69                utils.run(command)
70            except error.CmdError, cmderr:
71                raise error.AutoservRunError(cmderr.args[0], cmderr.args[1])
72
73
74    def get_file(self, source, dest, delete_dest=False):
75        """
76        Copy files from the remote host to a local path.
77
78        Directories will be copied recursively.
79        If a source component is a directory with a trailing slash,
80        the content of the directory will be copied, otherwise, the
81        directory itself and its content will be copied. This
82        behavior is similar to that of the program 'rsync'.
83
84        Args:
85                source: either
86                        1) a single file or directory, as a string
87                        2) a list of one or more (possibly mixed)
88                                files or directories
89                dest: a file or a directory (if source contains a
90                        directory or more than one element, you must
91                        supply a directory dest)
92                delete_dest: if this is true, the command will also clear
93                             out any old files at dest that are not in the
94                             source
95
96        Raises:
97                AutoservRunError: the scp command failed
98        """
99        if isinstance(source, basestring):
100            source = [source]
101
102        processed_source = []
103        for path in source:
104            if path.endswith('/'):
105                format_string = '%s@%s:"%s*"'
106            else:
107                format_string = '%s@%s:"%s"'
108            entry = format_string % (self.user, self.hostname,
109                                     utils.scp_remote_escape(path))
110            processed_source.append(entry)
111
112        processed_dest = utils.sh_escape(os.path.abspath(dest))
113        if os.path.isdir(dest):
114            processed_dest += "/"
115
116        self._copy_files(processed_source, processed_dest, delete_dest)
117
118
119    def send_file(self, source, dest, delete_dest=False):
120        """
121        Copy files from a local path to the remote host.
122
123        Directories will be copied recursively.
124        If a source component is a directory with a trailing slash,
125        the content of the directory will be copied, otherwise, the
126        directory itself and its content will be copied. This
127        behavior is similar to that of the program 'rsync'.
128
129        Args:
130                source: either
131                        1) a single file or directory, as a string
132                        2) a list of one or more (possibly mixed)
133                                files or directories
134                dest: a file or a directory (if source contains a
135                        directory or more than one element, you must
136                        supply a directory dest)
137                delete_dest: if this is true, the command will also clear
138                             out any old files at dest that are not in the
139                             source
140
141        Raises:
142                AutoservRunError: the scp command failed
143        """
144        if isinstance(source, basestring):
145            source = [source]
146
147        processed_source = []
148        for path in source:
149            if path.endswith('/'):
150                format_string = '"%s/"*'
151            else:
152                format_string = '"%s"'
153            entry = format_string % (utils.sh_escape(os.path.abspath(path)),)
154            processed_source.append(entry)
155
156        remote_dest = '%s@%s:"%s"' % (self.user, self.hostname,
157                                      utils.scp_remote_escape(dest))
158
159        self._copy_files(processed_source, remote_dest, delete_dest)
160        self.run('find "%s" -type d -print0 | xargs -0r chmod o+rx' % dest)
161        self.run('find "%s" -type f -print0 | xargs -0r chmod o+r' % dest)
162        if self.target_file_owner:
163            self.run('chown -R %s %s' % (self.target_file_owner, dest))
164
165
166    def ssh_ping(self, timeout=60):
167        try:
168            self.run("true", timeout=timeout, connect_timeout=timeout)
169        except error.AutoservSSHTimeout:
170            msg = "ssh ping timed out (timeout = %d)" % timeout
171            raise error.AutoservSSHTimeout(msg)
172        except error.AutoservRunError, e:
173            msg = "command true failed in ssh ping"
174            raise error.AutoservRunError(msg, e.result_obj)
175
176
177    def is_up(self):
178        """
179        Check if the remote host is up.
180
181        Returns:
182                True if the remote host is up, False otherwise
183        """
184        try:
185            self.ssh_ping()
186        except error.AutoservError:
187            return False
188        else:
189            return True
190
191
192    def wait_up(self, timeout=None):
193        """
194        Wait until the remote host is up or the timeout expires.
195
196        In fact, it will wait until an ssh connection to the remote
197        host can be established, and getty is running.
198
199        Args:
200                timeout: time limit in seconds before returning even
201                        if the host is not up.
202
203        Returns:
204                True if the host was found to be up, False otherwise
205        """
206        if timeout:
207            end_time = time.time() + timeout
208
209        while not timeout or time.time() < end_time:
210            if self.is_up():
211                try:
212                    if self.are_wait_up_processes_up():
213                        return True
214                except error.AutoservError:
215                    pass
216            time.sleep(1)
217
218        return False
219
220
221    def wait_down(self, timeout=None):
222        """
223        Wait until the remote host is down or the timeout expires.
224
225        In fact, it will wait until an ssh connection to the remote
226        host fails.
227
228        Args:
229                timeout: time limit in seconds before returning even
230                        if the host is not up.
231
232        Returns:
233                True if the host was found to be down, False otherwise
234        """
235        if timeout:
236            end_time = time.time() + timeout
237
238        while not timeout or time.time() < end_time:
239            if not self.is_up():
240                return True
241            time.sleep(1)
242
243        return False
244
245    # tunable constants for the verify & repair code
246    AUTOTEST_GB_DISKSPACE_REQUIRED = 20
247    HOURS_TO_WAIT_FOR_RECOVERY = 2.5
248
249    def verify(self):
250        super(AbstractSSHHost, self).verify()
251
252        print 'Pinging host ' + self.hostname
253        self.ssh_ping()
254
255        try:
256            autodir = autotest._get_autodir(self)
257            if autodir:
258                print 'Checking diskspace for %s on %s' % (self.hostname,
259                                                           autodir)
260                self.check_diskspace(autodir,
261                                     self.AUTOTEST_GB_DISKSPACE_REQUIRED)
262        except error.AutoservHostError:
263            raise           # only want to raise if it's a space issue
264        except Exception:
265            pass            # autotest dir may not exist, etc. ignore
266
267
268    def repair_filesystem_only(self):
269        super(AbstractSSHHost, self).repair_filesystem_only()
270        self.wait_up(int(self.HOURS_TO_WAIT_FOR_RECOVERY * 3600))
271        self.reboot()
272
273
274    def repair_full(self):
275        super(AbstractSSHHost, self).repair_full()
276        try:
277            self.repair_filesystem_only()
278            self.verify()
279        except Exception:
280            # the filesystem-only repair failed, try something more drastic
281            print "Filesystem-only repair failed"
282            traceback.print_exc()
283            try:
284                self.machine_install()
285            except NotImplementedError, e:
286                sys.stderr.write(str(e) + "\n\n")
287