1be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# Use of this source code is governed by a BSD-style license that can be 3be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# found in the LICENSE file. 4be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 5be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik"""Functions that deal with local and device ports.""" 6be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 7be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport contextlib 8be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport fcntl 9be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport httplib 10be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport logging 11be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport os 12be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport socket 13be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikimport traceback 14be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 15ee838d1c4002134ff5af32da272140586c4d31deJohn Recklogger = logging.getLogger(__name__) 16ee838d1c4002134ff5af32da272140586c4d31deJohn Reck 17be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# The net test server is started from port 10201. 18be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik_TEST_SERVER_PORT_FIRST = 10201 19be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik_TEST_SERVER_PORT_LAST = 30000 20be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# A file to record next valid port of test server. 21be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik_TEST_SERVER_PORT_FILE = '/tmp/test_server_port' 22be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik_TEST_SERVER_PORT_LOCKFILE = '/tmp/test_server_port.lock' 23be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 24be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 25be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# The following two methods are used to allocate the port source for various 26be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# types of test servers. Because some net-related tests can be run on shards at 27be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# same time, it's important to have a mechanism to allocate the port 28be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# process-safe. In here, we implement the safe port allocation by leveraging 29be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik# flock. 30be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikdef ResetTestServerPortAllocation(): 31be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """Resets the port allocation to start from TEST_SERVER_PORT_FIRST. 32be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 33be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns: 34be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns True if reset successes. Otherwise returns False. 35be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """ 36be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik try: 37be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik with open(_TEST_SERVER_PORT_FILE, 'w') as fp: 38be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fp.write('%d' % _TEST_SERVER_PORT_FIRST) 39be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return True 40be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik except Exception: # pylint: disable=broad-except 41ee838d1c4002134ff5af32da272140586c4d31deJohn Reck logger.exception('Error while resetting port allocation') 42be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return False 43be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 44be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 45be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikdef AllocateTestServerPort(): 46be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """Allocates a port incrementally. 47be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 48be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns: 49be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns a valid port which should be in between TEST_SERVER_PORT_FIRST and 50be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik TEST_SERVER_PORT_LAST. Returning 0 means no more valid port can be used. 51be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """ 52be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik port = 0 53be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik ports_tried = [] 54be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik try: 55be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fp_lock = open(_TEST_SERVER_PORT_LOCKFILE, 'w') 56be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fcntl.flock(fp_lock, fcntl.LOCK_EX) 57be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik # Get current valid port and calculate next valid port. 58be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if not os.path.exists(_TEST_SERVER_PORT_FILE): 59be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik ResetTestServerPortAllocation() 60be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik with open(_TEST_SERVER_PORT_FILE, 'r+') as fp: 61be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik port = int(fp.read()) 62be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik ports_tried.append(port) 63be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik while not IsHostPortAvailable(port): 64be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik port += 1 65be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik ports_tried.append(port) 66be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if (port > _TEST_SERVER_PORT_LAST or 67be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik port < _TEST_SERVER_PORT_FIRST): 68be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik port = 0 69be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik else: 70be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fp.seek(0, os.SEEK_SET) 71be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fp.write('%d' % (port + 1)) 72be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik except Exception: # pylint: disable=broad-except 73ee838d1c4002134ff5af32da272140586c4d31deJohn Reck logger.exception('Error while allocating port') 74be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik finally: 75be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if fp_lock: 76be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fcntl.flock(fp_lock, fcntl.LOCK_UN) 77be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik fp_lock.close() 78be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if port: 79ee838d1c4002134ff5af32da272140586c4d31deJohn Reck logger.info('Allocate port %d for test server.', port) 80be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik else: 81ee838d1c4002134ff5af32da272140586c4d31deJohn Reck logger.error('Could not allocate port for test server. ' 82ee838d1c4002134ff5af32da272140586c4d31deJohn Reck 'List of ports tried: %s', str(ports_tried)) 83be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return port 84be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 85be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 86be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikdef IsHostPortAvailable(host_port): 87be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """Checks whether the specified host port is available. 88be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 89be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Args: 90be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik host_port: Port on host to check. 91be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 92be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns: 93be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik True if the port on host is available, otherwise returns False. 94be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """ 95be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik s = socket.socket() 96be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik try: 97be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 98be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik s.bind(('', host_port)) 99be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik s.close() 100be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return True 101be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik except socket.error: 102be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return False 103be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 104be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 105be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikdef IsDevicePortUsed(device, device_port, state=''): 106be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """Checks whether the specified device port is used or not. 107be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 108be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Args: 109be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik device: A DeviceUtils instance. 110be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik device_port: Port on device we want to check. 111be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik state: String of the specified state. Default is empty string, which 112be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik means any state. 113be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 114be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns: 115be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik True if the port on device is already used, otherwise returns False. 116be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """ 117be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik base_urls = ('127.0.0.1:%d' % device_port, 'localhost:%d' % device_port) 118be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik netstat_results = device.RunShellCommand( 119be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik ['netstat', '-a'], check_return=True, large_output=True) 120be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik for single_connect in netstat_results: 121be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik # Column 3 is the local address which we want to check with. 122be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik connect_results = single_connect.split() 123be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if connect_results[0] != 'tcp': 124be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik continue 125be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if len(connect_results) < 6: 126be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik raise Exception('Unexpected format while parsing netstat line: ' + 127be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik single_connect) 128be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik is_state_match = connect_results[5] == state if state else True 129be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if connect_results[3] in base_urls and is_state_match: 130be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return True 131be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return False 132be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 133be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 134be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craikdef IsHttpServerConnectable(host, port, tries=3, command='GET', path='/', 135be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik expected_read='', timeout=2): 136be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """Checks whether the specified http server is ready to serve request or not. 137be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 138be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Args: 139be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik host: Host name of the HTTP server. 140be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik port: Port number of the HTTP server. 141be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik tries: How many times we want to test the connection. The default value is 142be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 3. 143be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik command: The http command we use to connect to HTTP server. The default 144be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik command is 'GET'. 145be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik path: The path we use when connecting to HTTP server. The default path is 146be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik '/'. 147be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik expected_read: The content we expect to read from the response. The default 148be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik value is ''. 149be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik timeout: Timeout (in seconds) for each http connection. The default is 2s. 150be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik 151be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Returns: 152be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik Tuple of (connect status, client error). connect status is a boolean value 153be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik to indicate whether the server is connectable. client_error is the error 154be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik message the server returns when connect status is false. 155be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik """ 156be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik assert tries >= 1 157be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik for i in xrange(0, tries): 158be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik client_error = None 159be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik try: 160be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik with contextlib.closing(httplib.HTTPConnection( 161be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik host, port, timeout=timeout)) as http: 162be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik # Output some debug information when we have tried more than 2 times. 163be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik http.set_debuglevel(i >= 2) 164be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik http.request(command, path) 165be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik r = http.getresponse() 166be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik content = r.read() 167be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if r.status == 200 and r.reason == 'OK' and content == expected_read: 168be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return (True, '') 169be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik client_error = ('Bad response: %s %s version %s\n ' % 170be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik (r.status, r.reason, r.version) + 171be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik '\n '.join([': '.join(h) for h in r.getheaders()])) 172be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik except (httplib.HTTPException, socket.error) as e: 173be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik # Probably too quick connecting: try again. 174be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik exception_error_msgs = traceback.format_exception_only(type(e), e) 175be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik if exception_error_msgs: 176be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik client_error = ''.join(exception_error_msgs) 177be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik # Only returns last client_error. 178be1f909aea58dd8b153538c9fa19cb0bf50bdb17Chris Craik return (False, client_error or 'Timeout') 179