rdb_testing_utils.py revision 0e960285b022fad77f0b087a2007867363bf6ab9
12d8047e8b2d901bec66d483664d8b6322501d245Prashanth B#!/usr/bin/python 22d8047e8b2d901bec66d483664d8b6322501d245Prashanth B#pylint: disable-msg=C0111 32d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 42d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 52d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# Use of this source code is governed by a BSD-style license that can be 62d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# found in the LICENSE file. 72d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 82d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bimport abc 92d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 102d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bimport common 112d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 122d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.database import database_connection 132d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend import setup_django_environment 142d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend.afe import frontend_test_utils 152d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend.afe import models 162d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend.afe import rdb_model_extensions as rdb_models 172d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import monitor_db 182d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import monitor_db_functional_test 190e960285b022fad77f0b087a2007867363bf6ab9Prashanth Bfrom autotest_lib.scheduler import scheduler_lib 202d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import scheduler_models 212d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import rdb_hosts 222d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import rdb_requests 232d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.server.cros import provision 242d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# Set for verbose table creation output. 272d8047e8b2d901bec66d483664d8b6322501d245Prashanth B_DEBUG = False 282d8047e8b2d901bec66d483664d8b6322501d245Prashanth BDEFAULT_ACLS = ['Everyone', 'my_acl'] 292d8047e8b2d901bec66d483664d8b6322501d245Prashanth BDEFAULT_DEPS = ['a', 'b'] 302d8047e8b2d901bec66d483664d8b6322501d245Prashanth BDEFAULT_USER = 'system' 312d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 322d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bdef get_default_job_params(): 332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return {'deps': DEFAULT_DEPS, 'user': DEFAULT_USER, 'acls': DEFAULT_ACLS, 342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 'priority': 0, 'parent_job_id': 0} 352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 372d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bdef get_default_host_params(): 382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return {'deps': DEFAULT_DEPS, 'acls': DEFAULT_ACLS} 392d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 402d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 412d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bclass FakeHost(rdb_hosts.RDBHost): 422d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Fake host to use in unittests.""" 432d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def __init__(self, hostname, host_id, **kwargs): 452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B kwargs.update({'hostname': hostname, 'id': host_id}) 462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B kwargs = rdb_models.AbstractHostModel.provide_default_values( 472d8047e8b2d901bec66d483664d8b6322501d245Prashanth B kwargs) 482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B super(FakeHost, self).__init__(**kwargs) 492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 512d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bdef wire_format_response_map(response_map): 522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B wire_formatted_map = {} 532d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for request, response in response_map.iteritems(): 542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B wire_formatted_map[request] = [reply.wire_format() 552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for reply in response] 562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return wire_formatted_map 572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 592d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bclass DBHelper(object): 602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Utility class for updating the database.""" 612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 622d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def __init__(self): 632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Initialized django so it uses an in memory sqllite database.""" 642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.database = ( 652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B database_connection.TranslatingDatabase.get_test_database( 662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B translators=monitor_db_functional_test._DB_TRANSLATORS)) 672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.database.connect(db_type='django') 682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.database.debug = _DEBUG 692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_labels(cls, **kwargs): 732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Get a label queryset based on the kwargs.""" 742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.Label.objects.filter(**kwargs) 752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_acls(cls, **kwargs): 792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Get an aclgroup queryset based on the kwargs.""" 802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.AclGroup.objects.filter(**kwargs) 812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 822d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 832d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 842d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_host(cls, **kwargs): 852d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Get a host queryset based on the kwargs.""" 862d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.Host.objects.filter(**kwargs) 872d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_label(cls, name, **kwargs): 912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label = cls.get_labels(name=name, **kwargs) 922d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return (models.Label.add_object(name=name, **kwargs) 932d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if not label else label[0]) 942d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 952d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 962d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 972d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_user(cls, name): 982d8047e8b2d901bec66d483664d8b6322501d245Prashanth B user = models.User.objects.filter(login=name) 992d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.User.add_object(login=name) if not user else user[0] 1002d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1012d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1022d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1032d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_labels_to_host(cls, host, label_names=set([])): 1042d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects = set([]) 1052d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for label in label_names: 1062d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects.add(cls.create_label(label)) 1072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host.labels.add(*label_objects) 1082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1092d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1102d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1112d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_acl_group(cls, name): 1122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup = cls.get_acls(name=name) 1132d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return (models.AclGroup.add_object(name=name) 1142d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if not aclgroup else aclgroup[0]) 1152d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1162d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1172d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1182d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_deps_to_job(cls, job, dep_names=set([])): 1192d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects = set([]) 1202d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for label in dep_names: 1212d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects.add(cls.create_label(label)) 1222d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job.dependency_labels.add(*label_objects) 1232d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1242d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_host_to_aclgroup(cls, host, aclgroup_names=set([])): 1272d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for group_name in aclgroup_names: 1282d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup = cls.create_acl_group(group_name) 1292d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup.hosts.add(host) 1302d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1312d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_user_to_aclgroups(cls, username, aclgroup_names=set([])): 1342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B user = cls.create_user(username) 1352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for group_name in aclgroup_names: 1362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup = cls.create_acl_group(group_name) 1372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup.users.add(user) 1382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1392d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1402d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1412d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_host(cls, name, deps=set([]), acls=set([]), status='Ready', 1422d8047e8b2d901bec66d483664d8b6322501d245Prashanth B locked=0, leased=0, protection=0, dirty=0): 1432d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Create a host. 1442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Also adds the appropriate labels to the host, and adds the host to the 1462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B required acl groups. 1472d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param name: The hostname. 1492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param kwargs: 1502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps: The labels on the host that match job deps. 1512d8047e8b2d901bec66d483664d8b6322501d245Prashanth B acls: The aclgroups this host must be a part of. 1522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B status: The status of the host. 1532d8047e8b2d901bec66d483664d8b6322501d245Prashanth B locked: 1 if the host is locked. 1542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B leased: 1 if the host is leased. 1552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B protection: Any protection level, such as Do Not Verify. 1562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B dirty: 1 if the host requires cleanup. 1572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @return: The host object for the new host. 1592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 1602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # TODO: Modify this to use the create host request once 1612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # crbug.com/350995 is fixed. 1622d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host = models.Host.add_object( 1632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hostname=name, status=status, locked=locked, 1642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B leased=leased, protection=protection) 1652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B cls.add_labels_to_host(host, label_names=deps) 1662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B cls.add_host_to_aclgroup(host, aclgroup_names=acls) 1672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Though we can return the host object above, this proves that the host 1692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # actually got saved in the database. For example, this will return none 1702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # if save() wasn't called on the model.Host instance. 1712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return cls.get_host(hostname=name)[0] 1722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_host_to_job(cls, host, job_id): 1762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Add a host to the hqe of a job. 1772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host: An instance of the host model. 1792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param job_id: The job to which we need to add the host. 1802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises ValueError: If the hqe for the job already has a host, 1822d8047e8b2d901bec66d483664d8b6322501d245Prashanth B or if the host argument isn't a Host instance. 1832d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 1842d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe = models.HostQueueEntry.objects.get(job_id=job_id) 1852d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if hqe.host: 1862d8047e8b2d901bec66d483664d8b6322501d245Prashanth B raise ValueError('HQE for job %s already has a host' % job_id) 1872d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe.host = host 1882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe.save() 1892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1922d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_host_to_job(cls, host, job_id): 1932d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Add a host to the hqe of a job. 1942d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1952d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host: An instance of the host model. 1962d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param job_id: The job to which we need to add the host. 1972d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1982d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises ValueError: If the hqe for the job already has a host, 1992d8047e8b2d901bec66d483664d8b6322501d245Prashanth B or if the host argument isn't a Host instance. 2002d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 2012d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe = models.HostQueueEntry.objects.get(job_id=job_id) 2022d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if hqe.host: 2032d8047e8b2d901bec66d483664d8b6322501d245Prashanth B raise ValueError('HQE for job %s already has a host' % job_id) 2042d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe.host = host 2052d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe.save() 2062d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 2092d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def increment_priority(cls, job_id): 2102d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job = models.Job.objects.get(id=job_id) 2112d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job.priority = job.priority + 1 2122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job.save() 2132d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2142d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2152d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bclass AbstractBaseRDBTester(frontend_test_utils.FrontendTestMixin): 2162d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2172d8047e8b2d901bec66d483664d8b6322501d245Prashanth B __meta__ = abc.ABCMeta 2182d8047e8b2d901bec66d483664d8b6322501d245Prashanth B _config_section = 'AUTOTEST_WEB' 2192d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2202d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2212d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @staticmethod 2222d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_request(dep_names, acl_names, priority=0, parent_job_id=0): 2232d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps = [dep.id for dep in DBHelper.get_labels(name__in=dep_names)] 2242d8047e8b2d901bec66d483664d8b6322501d245Prashanth B acls = [acl.id for acl in DBHelper.get_acls(name__in=acl_names)] 2252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return rdb_requests.AcquireHostRequest( 2262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps=deps, acls=acls, host_id=None, priority=priority, 2272d8047e8b2d901bec66d483664d8b6322501d245Prashanth B parent_job_id=parent_job_id)._request 2282d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2292d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2302d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def _release_unused_hosts(self): 2312d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Release all hosts unused by an active hqe. """ 2322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.host_scheduler.tick() 2332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def setUp(self): 2362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.db_helper = DBHelper() 2372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._database = self.db_helper.database 2382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Runs syncdb setting up initial database conditions 2392d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._frontend_common_setup() 2400e960285b022fad77f0b087a2007867363bf6ab9Prashanth B connection_manager = scheduler_lib.ConnectionManager(autocommit=False) 2410e960285b022fad77f0b087a2007867363bf6ab9Prashanth B self.god.stub_with(connection_manager, 'db_connection', self._database) 2420e960285b022fad77f0b087a2007867363bf6ab9Prashanth B self.god.stub_with(monitor_db, '_db_manager', connection_manager) 2432d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.god.stub_with(scheduler_models, '_db', self._database) 2442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._dispatcher = monitor_db.Dispatcher() 2452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.host_scheduler = self._dispatcher._host_scheduler 2462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._release_unused_hosts() 2472d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def tearDown(self): 2502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.god.unstub_all() 2512d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._database.disconnect() 2522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._frontend_common_teardown() 2532d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_job(self, user='autotest_system', 2562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps=set([]), acls=set([]), hostless_job=False, 2572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B priority=0, parent_job_id=None): 2582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Create a job owned by user, with the deps and acls specified. 2592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B This method is a wrapper around frontend_test_utils.create_job, that 2612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B also takes care of creating the appropriate deps for a job, and the 2622d8047e8b2d901bec66d483664d8b6322501d245Prashanth B appropriate acls for the given user. 2632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises ValueError: If no deps are specified for a job, since all jobs 2652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B need at least the metahost. 2662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If no hqe was created for the job. 2672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @return: An instance of the job model associated with the new job. 2692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 2702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # This is a slight hack around the implementation of 2712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # scheduler_models.is_hostless_job, even though a metahost is just 2722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # another label to the rdb. 2732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if not deps: 2742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B raise ValueError('Need at least one dep for metahost') 2752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # TODO: This is a hack around the fact that frontend_test_utils still 2772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # need a metahost, but metahost is treated like any other label. 2782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B metahost = self.db_helper.create_label(list(deps)[0]) 2792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job = self._create_job(metahosts=[metahost.id], priority=priority, 2802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B owner=user, parent_job_id=parent_job_id) 2812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(len(job.hostqueueentry_set.all()) == 1) 2822d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2832d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.db_helper.add_deps_to_job(job, dep_names=list(deps)[1:]) 2842d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.db_helper.add_user_to_aclgroups(user, aclgroup_names=acls) 2852d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.Job.objects.filter(id=job.id)[0] 2862d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2872d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def assert_host_db_status(self, host_id): 2892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Assert host state right after acquisition. 2902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Call this method to check the status of any host leased by the 2922d8047e8b2d901bec66d483664d8b6322501d245Prashanth B rdb before it has been assigned to an hqe. It must be leased and 2932d8047e8b2d901bec66d483664d8b6322501d245Prashanth B ready at this point in time. 2942d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2952d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host_id: Id of the host to check. 2962d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2972d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If the host is either not leased or Ready. 2982d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 2992d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host = models.Host.objects.get(id=host_id) 3002d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(host.leased) 3012d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(host.status == 'Ready') 3022d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3032d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3042d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def check_hosts(self, host_iter): 3052d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Sanity check all hosts in the host_gen. 3062d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host_iter: A generator/iterator of RDBClientHostWrappers. 3082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B eg: The generator returned by rdb_lib.acquire_hosts. If a request 3092d8047e8b2d901bec66d483664d8b6322501d245Prashanth B was not satisfied this iterator can contain None. 3102d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3112d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If any of the sanity checks fail. 3122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 3132d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for host in host_iter: 3142d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if host: 3152d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_host_db_status(host.id) 3162d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(host.leased == 1) 3172d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3182d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3192d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_suite(self, user='autotest_system', num=2, priority=0, 3202d8047e8b2d901bec66d483664d8b6322501d245Prashanth B board='z', build='x'): 3212d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Create num jobs with the same parent_job_id, board, build, priority. 3222d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3232d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @return: A dictionary with the parent job object keyed as 'parent_job' 3242d8047e8b2d901bec66d483664d8b6322501d245Prashanth B and all other jobs keyed at an index from 0-num. 3252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 3262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B jobs = {} 3272d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Create a hostless parent job without an hqe or deps. Since the 3282d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # hostless job does nothing, we need to hand craft cros-version. 3292d8047e8b2d901bec66d483664d8b6322501d245Prashanth B parent_job = self._create_job(owner=user, priority=priority) 3302d8047e8b2d901bec66d483664d8b6322501d245Prashanth B jobs['parent_job'] = parent_job 3312d8047e8b2d901bec66d483664d8b6322501d245Prashanth B build = '%s:%s' % (provision.CROS_VERSION_PREFIX, build) 3322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for job_index in range(0, num): 3332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B jobs[job_index] = self.create_job(user=user, priority=priority, 3342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps=set([board, build]), 3352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B parent_job_id=parent_job.id) 3362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return jobs 3372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3392d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def check_host_assignment(self, job_id, host_id): 3402d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Check is a job<->host assignment is valid. 3412d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3422d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Uses the deps of a job and the aclgroups the owner of the job is 3432d8047e8b2d901bec66d483664d8b6322501d245Prashanth B in to see if the given host can be used to run the given job. Also 3442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B checks that the host-job assignment has Not been made, but that the 3452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host is no longer in the available hosts pool. 3462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3472d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Use this method to check host assignements made by the rdb, Before 3482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B they're handed off to the scheduler, since the scheduler. 3492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param job_id: The id of the job to use in the compatibility check. 3512d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host_id: The id of the host to check for compatibility. 3522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3532d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If the job and the host are incompatible. 3542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 3552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job = models.Job.objects.get(id=job_id) 3562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host = models.Host.objects.get(id=host_id) 3572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe = job.hostqueueentry_set.all()[0] 3582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Confirm that the host has not been assigned, either to another hqe 3602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # or the this one. 3612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B all_hqes = models.HostQueueEntry.objects.filter( 3622d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host_id=host_id, complete=0) 3632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(len(all_hqes) <= 1) 3642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(hqe.host_id == None) 3652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_host_db_status(host_id) 3662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Assert that all deps of the job are satisfied. 3682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job_deps = set([d.name for d in job.dependency_labels.all()]) 3692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host_labels = set([l.name for l in host.labels.all()]) 3702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(job_deps.intersection(host_labels) == job_deps) 3712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Assert that the owner of the job is in at least one of the 3732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # groups that owns the host. 3742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job_owner_aclgroups = set([job_acl.name for job_acl 3752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B in job.user().aclgroup_set.all()]) 3762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host_aclgroups = set([host_acl.name for host_acl 3772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B in host.aclgroup_set.all()]) 3782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(job_owner_aclgroups.intersection(host_aclgroups)) 3792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 381