15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# found in the LICENSE file. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)"""Functions that deal with local and device ports.""" 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import contextlib 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import fcntl 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import httplib 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import logging 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import os 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import re 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import socket 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import traceback 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from pylib import cmd_helper 17a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from pylib import constants 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The following two methods are used to allocate the port source for various 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# types of test servers. Because some net-related tests can be run on shards at 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# same time, it's important to have a mechanism to allocate the port 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# process-safe. In here, we implement the safe port allocation by leveraging 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# flock. 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ResetTestServerPortAllocation(): 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Resets the port allocation to start from TEST_SERVER_PORT_FIRST. 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns True if reset successes. Otherwise returns False. 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) with open(constants.TEST_SERVER_PORT_FILE, 'w') as fp: 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fp.write('%d' % constants.TEST_SERVER_PORT_FIRST) 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if os.path.exists(constants.TEST_SERVER_PORT_LOCKFILE): 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) os.unlink(constants.TEST_SERVER_PORT_LOCKFILE) 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except Exception as e: 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) logging.error(e) 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def AllocateTestServerPort(): 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Allocates a port incrementally. 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns a valid port which should be in between TEST_SERVER_PORT_FIRST and 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TEST_SERVER_PORT_LAST. Returning 0 means no more valid port can be used. 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port = 0 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ports_tried = [] 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fp_lock = open(constants.TEST_SERVER_PORT_LOCKFILE, 'w') 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fcntl.flock(fp_lock, fcntl.LOCK_EX) 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Get current valid port and calculate next valid port. 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not os.path.exists(constants.TEST_SERVER_PORT_FILE): 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ResetTestServerPortAllocation() 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) with open(constants.TEST_SERVER_PORT_FILE, 'r+') as fp: 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port = int(fp.read()) 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ports_tried.append(port) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while IsHostPortUsed(port): 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port += 1 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ports_tried.append(port) 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (port > constants.TEST_SERVER_PORT_LAST or 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port < constants.TEST_SERVER_PORT_FIRST): 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port = 0 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fp.seek(0, os.SEEK_SET) 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fp.write('%d' % (port + 1)) 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except Exception as e: 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) logging.info(e) 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if fp_lock: 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fcntl.flock(fp_lock, fcntl.LOCK_UN) 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fp_lock.close() 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if port: 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) logging.info('Allocate port %d for test server.', port) 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) logging.error('Could not allocate port for test server. ' 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'List of ports tried: %s', str(ports_tried)) 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return port 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def IsHostPortUsed(host_port): 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Checks whether the specified host port is used or not. 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Uses -n -P to inhibit the conversion of host/port numbers to host/port names. 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) host_port: Port on host we want to check. 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) True if the port on host is already used, otherwise returns False. 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port_info = '(\*)|(127\.0\.0\.1)|(localhost):%d' % host_port 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(jnd): Find a better way to filter the port. Note that connecting to the 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # socket and closing it would leave it in the TIME_WAIT state. Setting 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # SO_LINGER on it and then closing it makes the Python HTTP server crash. 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) re_port = re.compile(port_info, re.MULTILINE) 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if re_port.search(cmd_helper.GetCmdOutput(['lsof', '-nPi:%d' % host_port])): 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 104a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochdef IsDevicePortUsed(device, device_port, state=''): 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Checks whether the specified device port is used or not. 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 108a02191e04bc25c4935f804f2c080ae28663d096dBen Murdoch device: A DeviceUtils instance. 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) device_port: Port on device we want to check. 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) state: String of the specified state. Default is empty string, which 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) means any state. 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) True if the port on device is already used, otherwise returns False. 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base_url = '127.0.0.1:%d' % device_port 117f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) netstat_results = device.RunShellCommand('netstat') 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for single_connect in netstat_results: 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Column 3 is the local address which we want to check with. 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) connect_results = single_connect.split() 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if connect_results[0] != 'tcp': 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(connect_results) < 6: 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise Exception('Unexpected format while parsing netstat line: ' + 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) single_connect) 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) is_state_match = connect_results[5] == state if state else True 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if connect_results[3] == base_url and is_state_match: 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def IsHttpServerConnectable(host, port, tries=3, command='GET', path='/', 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) expected_read='', timeout=2): 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Checks whether the specified http server is ready to serve request or not. 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) host: Host name of the HTTP server. 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) port: Port number of the HTTP server. 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tries: How many times we want to test the connection. The default value is 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3. 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) command: The http command we use to connect to HTTP server. The default 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) command is 'GET'. 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) path: The path we use when connecting to HTTP server. The default path is 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '/'. 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) expected_read: The content we expect to read from the response. The default 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) value is ''. 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timeout: Timeout (in seconds) for each http connection. The default is 2s. 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tuple of (connect status, client error). connect status is a boolean value 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to indicate whether the server is connectable. client_error is the error 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) message the server returns when connect status is false. 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert tries >= 1 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for i in xrange(0, tries): 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_error = None 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) with contextlib.closing(httplib.HTTPConnection( 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) host, port, timeout=timeout)) as http: 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Output some debug information when we have tried more than 2 times. 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) http.set_debuglevel(i >= 2) 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) http.request(command, path) 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) r = http.getresponse() 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) content = r.read() 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if r.status == 200 and r.reason == 'OK' and content == expected_read: 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (True, '') 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_error = ('Bad response: %s %s version %s\n ' % 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (r.status, r.reason, r.version) + 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '\n '.join([': '.join(h) for h in r.getheaders()])) 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except (httplib.HTTPException, socket.error) as e: 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Probably too quick connecting: try again. 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) exception_error_msgs = traceback.format_exception_only(type(e), e) 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if exception_error_msgs: 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_error = ''.join(exception_error_msgs) 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Only returns last client_error. 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (False, client_error or 'Timeout') 177