16d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian#! /usr/bin/python 26d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 36d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian"""A simple heartbeat server. 46d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 56d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth BalasubramanianExecutes *readonly* heartbeats against the given database. 66d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 76d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth BalasubramanianUsage: 86d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian1. heartbeat_server.py 96d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian --port 8080 106d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 116d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian Start to serve heartbeats on port 8080 using the database credentials 126d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian found in the shadow_config. One would perform heartbeats for board:lumpy 136d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian against this server with: 146d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian curl http://localhost:8080/lumpy. 156d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian Or just visiting the url through the browser. 166d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 176d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian Such a server is capable of handling the following urls: 186d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian /lumpy: Return formatted heartbeat packets with timing information for 196d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian each stage, to be viewed in the browser. 206d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian /lumpy?raw: Return raw json heartbeat packets for lumpy 216d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian /lumpy?raw&host_limit=1&job_limit=0: Return a 'raw' heartbeat with the 226d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian first host and not jobs. 236d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 246d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian2. heartbeat_server.py 256d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian --db_host <ip, eg: production db server> 266d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian --db_user <user, eg: chromeosqa-admin> 276d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian --db_password <password, eg: production db password> 286d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 296d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian The same as 1. but use the remote db server specified via 306d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian db_(host,user,password). 316d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian""" 326d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 336d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 346d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianimport argparse 356d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianimport sys 366d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianimport time 376d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianimport urlparse 386d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianfrom BaseHTTPServer import BaseHTTPRequestHandler 396d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianfrom BaseHTTPServer import HTTPServer 406d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 416d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianimport common 426d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianfrom autotest_lib.client.common_lib.global_config import global_config as config 436d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianfrom autotest_lib.frontend import setup_django_environment 446d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 456d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 466d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian# Populated with command line database credentials. 476d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth BalasubramanianDB_SETTINGS = { 486d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 'ENGINE': 'autotest_lib.frontend.db.backends.afe', 496d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian} 506d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 516d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian# Indent level used when formatting json for the browser. 526d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth BalasubramanianJSON_FORMATTING_INDENT = 4 536d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 546d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 556d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramaniandef time_call(func): 566d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian """A simple timer wrapper. 576d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 586d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @param func: The function to wrap. 596d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian """ 606d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian def wrapper(*args, **kwargs): 616d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian """Wrapper returned by time_call decorator.""" 626d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian start = time.time() 636d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian res = func(*args, **kwargs) 646d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return time.time()-start, res 656d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return wrapper 666d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 676d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 686d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianclass BoardHandler(BaseHTTPRequestHandler): 696d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian """Handles heartbeat urls.""" 706d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 716d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # Prefix for all board labels. 726d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian board_prefix = 'board:' 736d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 746d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 756d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @staticmethod 766d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @time_call 776d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian def _get_jobs(board, job_limit=None): 786d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian jobs = models.Job.objects.filter( 796d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian dependency_labels__name=board).exclude( 806d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian hostqueueentry__complete=True).exclude( 816d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian hostqueueentry__active=True) 826d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return jobs[:job_limit] if job_limit is not None else jobs 836d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 846d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 856d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @staticmethod 866d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @time_call 876d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian def _get_hosts(board, host_limit=None): 886d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian hosts = models.Host.objects.filter( 896d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian labels__name__in=[board], leased=False) 906d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return hosts[:host_limit] if host_limit is not None else hosts 916d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 926d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 936d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @staticmethod 946d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian @time_call 956d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian def _create_packet(hosts, jobs): 966d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return { 976d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 'hosts': [h.serialize() for h in hosts], 986d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 'jobs': [j.serialize() for j in jobs] 996d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian } 1006d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1016d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1026d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian def do_GET(self): 1036d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian """GET handler. 1046d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1056d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian Handles urls like: http://localhost:8080/lumpy?raw&host_limit=5 1066d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian and writes the appropriate http response containing the heartbeat. 1076d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian """ 1086d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parsed_path = urlparse.urlparse(self.path, allow_fragments=True) 1096d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian board = '%s%s' % (self.board_prefix, parsed_path.path.rsplit('/')[-1]) 1106d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1116d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian raw = False 1126d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian job_limit = None 1136d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian host_limit = None 1146d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian for query in parsed_path.query.split('&'): 1156d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian split_query = query.split('=') 1166d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian if split_query[0] == 'job_limit': 1176d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian job_limit = int(split_query[1]) 1186d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian elif split_query[0] == 'host_limit': 1196d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian host_limit = int(split_query[1]) 1206d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian elif split_query[0] == 'raw': 1216d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian raw = True 1226d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1236d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian host_time, hosts = self._get_hosts(board, host_limit) 1246d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian job_time, jobs = self._get_jobs(board, job_limit) 1256d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1266d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian serialize_time, heartbeat_packet = self._create_packet(hosts, jobs) 1276d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian self.send_response(200) 1286d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian self.end_headers() 1296d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1306d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # Format browser requests, the heartbeat client will request using ?raw 1316d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # while the browser will perform a plain request like 1326d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # http://localhost:8080/lumpy. The latter needs to be human readable and 1336d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # include more details timing information. 1346d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian json_encoder = django_encoder.DjangoJSONEncoder() 1356d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian if not raw: 1366d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian json_encoder.indent = JSON_FORMATTING_INDENT 1376d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian self.wfile.write('Serialize: %s,\nJob query: %s\nHost query: %s\n' 1386d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 'Hosts: %s\nJobs: %s\n' % 1396d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian (serialize_time, job_time, host_time, 1406d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian len(heartbeat_packet['hosts']), 1416d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian len(heartbeat_packet['jobs']))) 1426d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian self.wfile.write(json_encoder.encode(heartbeat_packet)) 1436d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return 1446d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1456d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1466d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramaniandef _parse_args(args): 1476d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser = argparse.ArgumentParser( 1486d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian description='Start up a simple heartbeat server on localhost.') 1496d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser.add_argument( 1506d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian '--port', default=8080, 1516d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian help='The port to start the heartbeat server.') 1526d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser.add_argument( 1536d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian '--db_host', 1546d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian default=config.get_config_value('AUTOTEST_WEB', 'host'), 1556d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian help='Db server ip address.') 1566d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser.add_argument( 1576d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian '--db_name', 1586d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian default=config.get_config_value('AUTOTEST_WEB', 'database'), 1596d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian help='Name of the db table.') 1606d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser.add_argument( 1616d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian '--db_user', 1626d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian default=config.get_config_value('AUTOTEST_WEB', 'user'), 1636d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian help='User for the db server.') 1646d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser.add_argument( 1656d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian '--db_password', 1666d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian default=config.get_config_value('AUTOTEST_WEB', 'password'), 1676d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian help='Password for the db server.') 1686d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian parser.add_argument( 1696d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian '--db_port', 1706d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian default=config.get_config_value('AUTOTEST_WEB', 'port', default=''), 1716d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian help='Port of the db server.') 1726d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1736d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian return parser.parse_args(args) 1746d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1756d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1766d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanianif __name__ == '__main__': 1776d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian args = _parse_args(sys.argv[1:]) 1786d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian server = HTTPServer(('localhost', args.port), BoardHandler) 1796d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian print ('Starting heartbeat server, query eg: http://localhost:%s/lumpy' % 1806d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian args.port) 1816d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1826d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # We need these lazy imports to allow command line specification of 1836d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian # database credentials. 1846d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian from autotest_lib.frontend import settings 1856d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian DB_SETTINGS['HOST'] = args.db_host 1866d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian DB_SETTINGS['NAME'] = args.db_name 1876d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian DB_SETTINGS['USER'] = args.db_user 1886d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian DB_SETTINGS['PASSWORD'] = args.db_password 1896d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian DB_SETTINGS['PORT'] = args.db_port 1906d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian settings.DATABASES['default'] = DB_SETTINGS 1916d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian from autotest_lib.frontend.afe import models 1926d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian from django.core.serializers import json as django_encoder 1936d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 1946d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian server.serve_forever() 1956d3bd9735dd64ebf98ed3d697d123a830319e360Prashanth Balasubramanian 196