158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius# Copyright (c) 2015 The Chromium OS Authors. All rights reserved. 258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius# Use of this source code is governed by a BSD-style license that can be 358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius# found in the LICENSE file. 458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusimport httplib 658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusimport logging 758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusimport socket 86c0f09b5d4c5b93e5cfa6b6a344343019e00d472Wai-Hong Tamimport tempfile 958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusimport time 1058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusimport xmlrpclib 1158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 1258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusimport common 1358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusfrom autotest_lib.client.bin import utils 14a28f89a1ad06d4b04ea05b85c0de8ddccb3564f4Dane Pollockfrom autotest_lib.client.common_lib import error 1558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusfrom autotest_lib.client.common_lib.cros import retry 1658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 1758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piustry: 1858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius import jsonrpclib 1958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusexcept ImportError: 2058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius jsonrpclib = None 2158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 2258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 2358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Piusclass RpcServerTracker(object): 2458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 2558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius This class keeps track of all the RPC server connections started on a remote 2658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius host. The caller can use either |xmlrpc_connect| or |jsonrpc_connect| to 2758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius start the required type of rpc server on the remote host. 2858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius The host will cleanup all the open RPC server connections on disconnect. 2958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 3058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 3158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius _RPC_PROXY_URL_FORMAT = 'http://localhost:%d' 3210ce09c325cc46c993929af23d2f54807862d005xixuan _RPC_HOST_ADDRESS_FORMAT = 'localhost:%d' 3358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius _RPC_SHUTDOWN_POLLING_PERIOD_SECONDS = 2 3458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius _RPC_SHUTDOWN_TIMEOUT_SECONDS = 10 3558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 3658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius def __init__(self, host): 3758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 3858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param port: The host object associated with this instance of 3958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius RpcServerTracker. 4058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 4158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self._host = host 4258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self._rpc_proxy_map = {} 4358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 4458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 4510ce09c325cc46c993929af23d2f54807862d005xixuan def _setup_port(self, port, command_name, remote_pid=None): 4610ce09c325cc46c993929af23d2f54807862d005xixuan """Sets up a tunnel process and register it to rpc_server_tracker. 4758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 4858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius Chrome OS on the target closes down most external ports for security. 4958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius We could open the port, but doing that would conflict with security 5058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius tests that check that only expected ports are open. So, to get to 5158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius the port on the target we use an ssh tunnel. 5258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 5358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius This method assumes that xmlrpc and jsonrpc never conflict, since 5458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius we can only either have an xmlrpc or a jsonrpc server listening on 5558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius a remote port. As such, it enforces a single proxy->remote port 5658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius policy, i.e if one starts a jsonrpc proxy/server from port A->B, 5758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius and then tries to start an xmlrpc proxy forwarded to the same port, 5858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius the xmlrpc proxy will override the jsonrpc tunnel process, however: 5958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 6058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 1. None of the methods on the xmlrpc proxy will work because 6158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius the server listening on B is jsonrpc. 6258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 6358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 2. The xmlrpc client cannot initiate a termination of the JsonRPC 6458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius server, as the only use case currently is goofy, which is tied to 6558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius the factory image. It is much easier to handle a failed xmlrpc 6658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius call on the client than it is to terminate goofy in this scenario, 6758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius as doing the latter might leave the DUT in a hard to recover state. 6858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 6958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius With the current implementation newer rpc proxy connections will 7058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius terminate the tunnel processes of older rpc connections tunneling 7158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius to the same remote port. If methods are invoked on the client 7258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius after this has happened they will fail with connection closed errors. 7358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 7458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param port: The remote forwarding port. 7558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param command_name: The name of the remote process, to terminate 7610ce09c325cc46c993929af23d2f54807862d005xixuan using pkill. 7710ce09c325cc46c993929af23d2f54807862d005xixuan @param remote_pid: The PID of the remote background process 7810ce09c325cc46c993929af23d2f54807862d005xixuan as a string. 7958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 8010ce09c325cc46c993929af23d2f54807862d005xixuan @return the local port which is used for port forwarding on the ssh 8110ce09c325cc46c993929af23d2f54807862d005xixuan client. 8258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 8358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self.disconnect(port) 8458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius local_port = utils.get_unused_port() 856cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan tunnel_proc = self._host.create_ssh_tunnel(port, local_port) 8658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self._rpc_proxy_map[port] = (command_name, tunnel_proc, remote_pid) 8710ce09c325cc46c993929af23d2f54807862d005xixuan return local_port 8810ce09c325cc46c993929af23d2f54807862d005xixuan 8910ce09c325cc46c993929af23d2f54807862d005xixuan 9010ce09c325cc46c993929af23d2f54807862d005xixuan def _setup_rpc(self, port, command_name, remote_pid=None): 9110ce09c325cc46c993929af23d2f54807862d005xixuan """Construct a URL for an rpc connection using ssh tunnel. 9210ce09c325cc46c993929af23d2f54807862d005xixuan 9310ce09c325cc46c993929af23d2f54807862d005xixuan @param port: The remote forwarding port. 9410ce09c325cc46c993929af23d2f54807862d005xixuan @param command_name: The name of the remote process, to terminate 9510ce09c325cc46c993929af23d2f54807862d005xixuan using pkill. 9610ce09c325cc46c993929af23d2f54807862d005xixuan @param remote_pid: The PID of the remote background process 9710ce09c325cc46c993929af23d2f54807862d005xixuan as a string. 9810ce09c325cc46c993929af23d2f54807862d005xixuan 9910ce09c325cc46c993929af23d2f54807862d005xixuan @return a url that we can use to initiate the rpc connection. 10010ce09c325cc46c993929af23d2f54807862d005xixuan """ 10110ce09c325cc46c993929af23d2f54807862d005xixuan return self._RPC_PROXY_URL_FORMAT % self._setup_port( 10210ce09c325cc46c993929af23d2f54807862d005xixuan port, command_name, remote_pid=remote_pid) 10310ce09c325cc46c993929af23d2f54807862d005xixuan 10410ce09c325cc46c993929af23d2f54807862d005xixuan 10510ce09c325cc46c993929af23d2f54807862d005xixuan def tunnel_connect(self, port): 10610ce09c325cc46c993929af23d2f54807862d005xixuan """Construct a host address using ssh tunnel. 10710ce09c325cc46c993929af23d2f54807862d005xixuan 10810ce09c325cc46c993929af23d2f54807862d005xixuan @param port: The remote forwarding port. 10910ce09c325cc46c993929af23d2f54807862d005xixuan 11010ce09c325cc46c993929af23d2f54807862d005xixuan @return a host address using ssh tunnel. 11110ce09c325cc46c993929af23d2f54807862d005xixuan """ 11210ce09c325cc46c993929af23d2f54807862d005xixuan return self._RPC_HOST_ADDRESS_FORMAT % self._setup_port(port, None) 11358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 11458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 11558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius def xmlrpc_connect(self, command, port, command_name=None, 11658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius ready_test_name=None, timeout_seconds=10, 117ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang logfile=None, request_timeout_seconds=None): 11858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """Connect to an XMLRPC server on the host. 11958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 12058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius The `command` argument should be a simple shell command that 12158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius starts an XMLRPC server on the given `port`. The command 12258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius must not daemonize, and must terminate cleanly on SIGTERM. 12358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius The command is started in the background on the host, and a 12458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius local XMLRPC client for the server is created and returned 12558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius to the caller. 12658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 12758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius Note that the process of creating an XMLRPC client makes no 12858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius attempt to connect to the remote server; the caller is 12958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius responsible for determining whether the server is running 13058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius correctly, and is ready to serve requests. 13158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 13258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius Optionally, the caller can pass ready_test_name, a string 13358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius containing the name of a method to call on the proxy. This 13458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius method should take no parameters and return successfully only 13558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius when the server is ready to process client requests. When 13658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius ready_test_name is set, xmlrpc_connect will block until the 13758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius proxy is ready, and throw a TestError if the server isn't 13858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius ready by timeout_seconds. 13958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 14058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius If a server is already running on the remote port, this 14158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius method will kill it and disconnect the tunnel process 14258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius associated with the connection before establishing a new one, 14358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius by consulting the rpc_proxy_map in disconnect. 14458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 14558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param command Shell command to start the server. 14658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param port Port number on which the server is expected to 14758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius be serving. 14858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param command_name String to use as input to `pkill` to 14958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius terminate the XMLRPC server on the host. 15058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param ready_test_name String containing the name of a 15158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius method defined on the XMLRPC server. 15258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param timeout_seconds Number of seconds to wait 15358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius for the server to become 'ready.' Will throw a 15458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius TestFail error if server is not ready in time. 15558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param logfile Logfile to send output when running 15658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 'command' argument. 157ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param request_timeout_seconds Timeout in seconds for an XMLRPC request. 15858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 15958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 16058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # Clean up any existing state. If the caller is willing 16158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # to believe their server is down, we ought to clean up 16258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # any tunnels we might have sitting around. 16358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self.disconnect(port) 1646cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan remote_pid = None 1656cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan if command is not None: 1666cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan if logfile: 1676cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan remote_cmd = '%s > %s 2>&1' % (command, logfile) 1686cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan else: 1696cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan remote_cmd = command 1706cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan remote_pid = self._host.run_background(remote_cmd) 1716cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan logging.debug('Started XMLRPC server on host %s, pid = %s', 1726cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan self._host.hostname, remote_pid) 17358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 17458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # Tunnel through SSH to be able to reach that remote port. 17558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius rpc_url = self._setup_rpc(port, command_name, remote_pid=remote_pid) 176ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang if request_timeout_seconds is not None: 177ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang proxy = TimeoutXMLRPCServerProxy( 178ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang rpc_url, timeout=request_timeout_seconds, allow_none=True) 179ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang else: 180ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang proxy = xmlrpclib.ServerProxy(rpc_url, allow_none=True) 18158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 18258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if ready_test_name is not None: 18358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # retry.retry logs each attempt; calculate delay_sec to 18458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # keep log spam to a dull roar. 18558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @retry.retry((socket.error, 18658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius xmlrpclib.ProtocolError, 18758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius httplib.BadStatusLine), 18858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius timeout_min=timeout_seconds / 60.0, 18958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius delay_sec=min(max(timeout_seconds / 20.0, 0.1), 1)) 19058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius def ready_test(): 19158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ Call proxy.ready_test_name(). """ 19258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius getattr(proxy, ready_test_name)() 19358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius successful = False 19458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius try: 19558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.info('Waiting %d seconds for XMLRPC server ' 19658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 'to start.', timeout_seconds) 19758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius ready_test() 19858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius successful = True 19958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius finally: 20058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if not successful: 20158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.error('Failed to start XMLRPC server.') 2026c0f09b5d4c5b93e5cfa6b6a344343019e00d472Wai-Hong Tam if logfile: 2036c0f09b5d4c5b93e5cfa6b6a344343019e00d472Wai-Hong Tam with tempfile.NamedTemporaryFile() as temp: 2046c0f09b5d4c5b93e5cfa6b6a344343019e00d472Wai-Hong Tam self._host.get_file(logfile, temp.name) 2056c0f09b5d4c5b93e5cfa6b6a344343019e00d472Wai-Hong Tam logging.error('The log of XML RPC server:\n%s', 2066c0f09b5d4c5b93e5cfa6b6a344343019e00d472Wai-Hong Tam open(temp.name).read()) 20758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self.disconnect(port) 20858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.info('XMLRPC server started successfully.') 20958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius return proxy 21058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 21158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 21258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius def jsonrpc_connect(self, port): 21358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """Creates a jsonrpc proxy connection through an ssh tunnel. 21458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 21558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius This method exists to facilitate communication with goofy (which is 21658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius the default system manager on all factory images) and as such, leaves 21758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius most of the rpc server sanity checking to the caller. Unlike 21858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius xmlrpc_connect, this method does not facilitate the creation of a remote 21958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius jsonrpc server, as the only clients of this code are factory tests, 22058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius for which the goofy system manager is built in to the image and starts 22158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius when the target boots. 22258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 22358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius One can theoretically create multiple jsonrpc proxies all forwarded 22458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius to the same remote port, provided the remote port has an rpc server 22558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius listening. However, in doing so we stand the risk of leaking an 22658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius existing tunnel process, so we always disconnect any older tunnels 22758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius we might have through disconnect. 22858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 22958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param port: port on the remote host that is serving this proxy. 23058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 23158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @return: The client proxy. 23258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 23358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if not jsonrpclib: 23458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.warning('Jsonrpclib could not be imported. Check that ' 23558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 'site-packages contains jsonrpclib.') 23658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius return None 23758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 23858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius proxy = jsonrpclib.jsonrpc.ServerProxy(self._setup_rpc(port, None)) 23958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 24058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.info('Established a jsonrpc connection through port %s.', port) 24158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius return proxy 24258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 24358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 24458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius def disconnect(self, port): 24558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """Disconnect from an RPC server on the host. 24658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 24758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius Terminates the remote RPC server previously started for 24858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius the given `port`. Also closes the local ssh tunnel created 24958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius for the connection to the host. This function does not 25058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius directly alter the state of a previously returned RPC 25158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius client object; however disconnection will cause all 25258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius subsequent calls to methods on the object to fail. 25358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 25458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius This function does nothing if requested to disconnect a port 25558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius that was not previously connected via _setup_rpc. 25658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 25758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius @param port Port number passed to a previous call to 25858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius `_setup_rpc()`. 25958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """ 26058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if port not in self._rpc_proxy_map: 26158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius return 26258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius remote_name, tunnel_proc, remote_pid = self._rpc_proxy_map[port] 26358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if remote_name: 26458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # We use 'pkill' to find our target process rather than 26558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # a PID, because the host may have rebooted since 26658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # connecting, and we don't want to kill an innocent 26758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # process with the same PID. 26858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # 26958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # 'pkill' helpfully exits with status 1 if no target 27058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # process is found, for which run() will throw an 27158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # exception. We don't want that, so we the ignore 27258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius # status. 27358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self._host.run("pkill -f '%s'" % remote_name, ignore_status=True) 27458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if remote_pid: 27558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.info('Waiting for RPC server "%s" shutdown', 27658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius remote_name) 27758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius start_time = time.time() 27858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius while (time.time() - start_time < 27958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self._RPC_SHUTDOWN_TIMEOUT_SECONDS): 28058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius running_processes = self._host.run( 28158e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius "pgrep -f '%s'" % remote_name, 28258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius ignore_status=True).stdout.split() 28358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius if not remote_pid in running_processes: 28458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius logging.info('Shut down RPC server.') 28558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius break 28658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius time.sleep(self._RPC_SHUTDOWN_POLLING_PERIOD_SECONDS) 28758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius else: 28858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius raise error.TestError('Failed to shutdown RPC server %s' % 28958e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius remote_name) 29058e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 2916cf6d2fa8d69d9896eba005310e110fa521b9cc3xixuan self._host.disconnect_ssh_tunnel(tunnel_proc, port) 29258e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius del self._rpc_proxy_map[port] 29358e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 29458e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius 29558e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius def disconnect_all(self): 29658e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius """Disconnect all known RPC proxy ports.""" 29758e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius for port in self._rpc_proxy_map.keys(): 29858e5dd3b600081607d5ecb514d9bf3ebea83c533Roshan Pius self.disconnect(port) 299ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 300ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 301ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiangclass TimeoutXMLRPCServerProxy(xmlrpclib.ServerProxy): 302ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """XMLRPC ServerProxy supporting timeout.""" 303ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang def __init__(self, uri, timeout=20, *args, **kwargs): 304ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """Initializes a TimeoutXMLRPCServerProxy. 305ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 306ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param uri: URI to a XMLRPC server. 307ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param timeout: Timeout in seconds for a XMLRPC request. 308ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param *args: args to xmlrpclib.ServerProxy. 309ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param **kwargs: kwargs to xmlrpclib.ServerProxy. 310ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 311ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """ 312ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang if timeout: 313ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang kwargs['transport'] = TimeoutXMLRPCTransport(timeout=timeout) 314ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang xmlrpclib.ServerProxy.__init__(self, uri, *args, **kwargs) 315ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 316ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 317ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiangclass TimeoutXMLRPCTransport(xmlrpclib.Transport): 318ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """A Transport subclass supporting timeout.""" 319ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang def __init__(self, timeout=20, *args, **kwargs): 320ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """Initializes a TimeoutXMLRPCTransport. 321ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 322ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param timeout: Timeout in seconds for a HTTP request through this transport layer. 323ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param *args: args to xmlrpclib.Transport. 324ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param **kwargs: kwargs to xmlrpclib.Transport. 325ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 326ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """ 327ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang xmlrpclib.Transport.__init__(self, *args, **kwargs) 328ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang self.timeout = timeout 329ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 330ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 331ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang def make_connection(self, host): 332ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """Overwrites make_connection in xmlrpclib.Transport with timeout. 333ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 334ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @param host: Host address to connect. 335ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 336ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang @return: A httplib.HTTPConnection connecting to host with timeout. 337ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang 338ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang """ 339ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang conn = httplib.HTTPConnection(host, timeout=self.timeout) 340ad248a87c20916552933c96067d0eb79e0ae70e3Cheng-Yi Chiang return conn 341