12d8047e8b2d901bec66d483664d8b6322501d245Prashanth B#!/usr/bin/python 22d8047e8b2d901bec66d483664d8b6322501d245Prashanth B#pylint: disable-msg=C0111 32d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 42d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 52d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# Use of this source code is governed by a BSD-style license that can be 62d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# found in the LICENSE file. 72d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 82d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bimport abc 94ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth Bimport os 102d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 112d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bimport common 122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 132d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.database import database_connection 142d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend import setup_django_environment 152d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend.afe import frontend_test_utils 162d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend.afe import models 172d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.frontend.afe import rdb_model_extensions as rdb_models 182d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import monitor_db 19f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth Bfrom autotest_lib.scheduler import query_managers 200e960285b022fad77f0b087a2007867363bf6ab9Prashanth Bfrom autotest_lib.scheduler import scheduler_lib 212d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import scheduler_models 222d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import rdb_hosts 232d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.scheduler import rdb_requests 242d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bfrom autotest_lib.server.cros import provision 252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 272d8047e8b2d901bec66d483664d8b6322501d245Prashanth B# Set for verbose table creation output. 282d8047e8b2d901bec66d483664d8b6322501d245Prashanth B_DEBUG = False 292d8047e8b2d901bec66d483664d8b6322501d245Prashanth BDEFAULT_ACLS = ['Everyone', 'my_acl'] 302d8047e8b2d901bec66d483664d8b6322501d245Prashanth BDEFAULT_DEPS = ['a', 'b'] 312d8047e8b2d901bec66d483664d8b6322501d245Prashanth BDEFAULT_USER = 'system' 322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 33f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B 342d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bdef get_default_job_params(): 352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return {'deps': DEFAULT_DEPS, 'user': DEFAULT_USER, 'acls': DEFAULT_ACLS, 362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 'priority': 0, 'parent_job_id': 0} 372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 392d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bdef get_default_host_params(): 402d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return {'deps': DEFAULT_DEPS, 'acls': DEFAULT_ACLS} 412d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 422d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 432d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bclass FakeHost(rdb_hosts.RDBHost): 442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Fake host to use in unittests.""" 452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def __init__(self, hostname, host_id, **kwargs): 472d8047e8b2d901bec66d483664d8b6322501d245Prashanth B kwargs.update({'hostname': hostname, 'id': host_id}) 482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B kwargs = rdb_models.AbstractHostModel.provide_default_values( 492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B kwargs) 502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B super(FakeHost, self).__init__(**kwargs) 512d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 532d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bdef wire_format_response_map(response_map): 542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B wire_formatted_map = {} 552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for request, response in response_map.iteritems(): 562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B wire_formatted_map[request] = [reply.wire_format() 572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for reply in response] 582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return wire_formatted_map 592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 612d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bclass DBHelper(object): 622d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Utility class for updating the database.""" 632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def __init__(self): 654ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """Initialized django so it uses an in memory SQLite database.""" 662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.database = ( 672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B database_connection.TranslatingDatabase.get_test_database( 684ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B translators=scheduler_lib._DB_TRANSLATORS)) 692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.database.connect(db_type='django') 702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.database.debug = _DEBUG 712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_labels(cls, **kwargs): 752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Get a label queryset based on the kwargs.""" 762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.Label.objects.filter(**kwargs) 772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_acls(cls, **kwargs): 812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Get an aclgroup queryset based on the kwargs.""" 822d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.AclGroup.objects.filter(**kwargs) 832d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 842d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 852d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 862d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_host(cls, **kwargs): 872d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Get a host queryset based on the kwargs.""" 882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.Host.objects.filter(**kwargs) 892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 92f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B def get_hqes(cls, **kwargs): 93f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B return models.HostQueueEntry.objects.filter(**kwargs) 94f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B 95f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B 96f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B @classmethod 974ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def get_tasks(cls, **kwargs): 984ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B return models.SpecialTask.objects.filter(**kwargs) 994ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 1004ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 1014ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @classmethod 10222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def get_shard(cls, **kwargs): 10322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian return models.Shard.objects.filter(**kwargs) 10422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 10522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 10622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @classmethod 1072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_label(cls, name, **kwargs): 1082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label = cls.get_labels(name=name, **kwargs) 1092d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return (models.Label.add_object(name=name, **kwargs) 1102d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if not label else label[0]) 1112d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1132d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1142d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_user(cls, name): 1152d8047e8b2d901bec66d483664d8b6322501d245Prashanth B user = models.User.objects.filter(login=name) 1162d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.User.add_object(login=name) if not user else user[0] 1172d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1182d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1192d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 120f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B def create_special_task(cls, job_id=None, host_id=None, 121f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B task=models.SpecialTask.Task.VERIFY, 122f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B user='autotest-system'): 123f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B if job_id: 124f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B queue_entry = cls.get_hqes(job_id=job_id)[0] 125f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B host_id = queue_entry.host.id 126f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B else: 127f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B queue_entry = None 128f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B host = models.Host.objects.get(id=host_id) 129f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B owner = cls.create_user(user) 130f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B if not host: 131f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B raise ValueError('Require a host to create special tasks.') 132f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B return models.SpecialTask.objects.create( 133f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B host=host, queue_entry=queue_entry, task=task, 134f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B requested_by_id=owner.id) 135f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B 136f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B 137f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B @classmethod 13822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def create_shard(cls, shard_hostname): 13922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Create a shard with the given hostname if one doesn't already exist. 14022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 14122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @param shard_hostname: The hostname of the shard. 14222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """ 14322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian shard = cls.get_shard(hostname=shard_hostname) 14422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian return (models.Shard.objects.create(hostname=shard_hostname) 14522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian if not shard else shard[0]) 14622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 14722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 14822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @classmethod 1492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_labels_to_host(cls, host, label_names=set([])): 1502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects = set([]) 1512d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for label in label_names: 1522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects.add(cls.create_label(label)) 1532d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host.labels.add(*label_objects) 1542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_acl_group(cls, name): 1582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup = cls.get_acls(name=name) 1592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return (models.AclGroup.add_object(name=name) 1602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if not aclgroup else aclgroup[0]) 1612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1622d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_deps_to_job(cls, job, dep_names=set([])): 1652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects = set([]) 1662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for label in dep_names: 1672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B label_objects.add(cls.create_label(label)) 1682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job.dependency_labels.add(*label_objects) 1692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 17222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def assign_job_to_shard(cls, job_id, shard_hostname): 17322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Assign a job to a shard. 17422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 17522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @param job: A job object without a shard. 17622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @param shard_hostname: The hostname of a shard to assign the job. 17722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 17822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @raises ValueError: If the job already has a shard. 17922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """ 18022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job_filter = models.Job.objects.filter(id=job_id, shard__isnull=True) 18122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian if len(job_filter) != 1: 18222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian raise ValueError('Failed to assign job %s to shard %s' % 18322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job_filter, shard_hostname) 18422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job_filter.update(shard=cls.create_shard(shard_hostname)) 18522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 18622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 18722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian @classmethod 1882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_host_to_aclgroup(cls, host, aclgroup_names=set([])): 1892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for group_name in aclgroup_names: 1902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup = cls.create_acl_group(group_name) 1912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup.hosts.add(host) 1922d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1932d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 1942d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 1952d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def add_user_to_aclgroups(cls, username, aclgroup_names=set([])): 1962d8047e8b2d901bec66d483664d8b6322501d245Prashanth B user = cls.create_user(username) 1972d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for group_name in aclgroup_names: 1982d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup = cls.create_acl_group(group_name) 1992d8047e8b2d901bec66d483664d8b6322501d245Prashanth B aclgroup.users.add(user) 2002d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2012d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2022d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 2032d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_host(cls, name, deps=set([]), acls=set([]), status='Ready', 2046818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori locked=0, lock_reason='', leased=0, protection=0, dirty=0): 2052d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Create a host. 2062d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Also adds the appropriate labels to the host, and adds the host to the 2082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B required acl groups. 2092d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2102d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param name: The hostname. 2112d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param kwargs: 2122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps: The labels on the host that match job deps. 2132d8047e8b2d901bec66d483664d8b6322501d245Prashanth B acls: The aclgroups this host must be a part of. 2142d8047e8b2d901bec66d483664d8b6322501d245Prashanth B status: The status of the host. 2152d8047e8b2d901bec66d483664d8b6322501d245Prashanth B locked: 1 if the host is locked. 2166818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori lock_reason: non-empty string if the host is locked. 2172d8047e8b2d901bec66d483664d8b6322501d245Prashanth B leased: 1 if the host is leased. 2182d8047e8b2d901bec66d483664d8b6322501d245Prashanth B protection: Any protection level, such as Do Not Verify. 2192d8047e8b2d901bec66d483664d8b6322501d245Prashanth B dirty: 1 if the host requires cleanup. 2202d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2212d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @return: The host object for the new host. 2222d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 2232d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # TODO: Modify this to use the create host request once 2242d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # crbug.com/350995 is fixed. 2252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host = models.Host.add_object( 2262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hostname=name, status=status, locked=locked, 2276818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori lock_reason=lock_reason, leased=leased, 2286818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori protection=protection) 2292d8047e8b2d901bec66d483664d8b6322501d245Prashanth B cls.add_labels_to_host(host, label_names=deps) 2302d8047e8b2d901bec66d483664d8b6322501d245Prashanth B cls.add_host_to_aclgroup(host, aclgroup_names=acls) 2312d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Though we can return the host object above, this proves that the host 2332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # actually got saved in the database. For example, this will return none 2342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # if save() wasn't called on the model.Host instance. 2352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return cls.get_host(hostname=name)[0] 2362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 2394ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def update_hqe(cls, hqe_id, **kwargs): 2404ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """Update the hqe with the given kwargs. 2414ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2424ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @param hqe_id: The id of the hqe to update. 2434ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """ 2444ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B models.HostQueueEntry.objects.filter(id=hqe_id).update(**kwargs) 2454ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2464ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2474ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @classmethod 2484ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def update_special_task(cls, task_id, **kwargs): 2494ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """Update special tasks with the given kwargs. 2504ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2514ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @param task_id: The if of the task to update. 2524ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """ 2534ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B models.SpecialTask.objects.filter(id=task_id).update(**kwargs) 2544ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2554ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2564ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @classmethod 2574ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def add_host_to_job(cls, host, job_id, activate=0): 2582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Add a host to the hqe of a job. 2592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host: An instance of the host model. 2612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param job_id: The job to which we need to add the host. 2624ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @param activate: If true, flip the active bit on the hqe. 2632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises ValueError: If the hqe for the job already has a host, 2652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B or if the host argument isn't a Host instance. 2662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 2672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe = models.HostQueueEntry.objects.get(job_id=job_id) 2682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if hqe.host: 2692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B raise ValueError('HQE for job %s already has a host' % job_id) 2702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe.host = host 2712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe.save() 2724ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B if activate: 2734ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B cls.update_hqe(hqe.id, active=True) 2742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @classmethod 2772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def increment_priority(cls, job_id): 2782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job = models.Job.objects.get(id=job_id) 2792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job.priority = job.priority + 1 2802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job.save() 2812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2822d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 2834ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth Bclass FileDatabaseHelper(object): 2844ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """A helper class to setup a SQLite database backed by a file. 2854ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2864ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B Note that initializing a file database takes significantly longer than an 2874ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B in-memory database and should only be used for functional tests. 2884ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """ 2894ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2904ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B DB_FILE = os.path.join(common.autotest_dir, 'host_scheduler_db') 2914ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2924ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def initialize_database_for_testing(self, db_file_path=None): 2934ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """Initialize a SQLite database for testing. 2944ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 2954ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B To force monitor_db and the host_scheduler to use the same SQLite file 2964ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B database, call this method before initializing the database through 2974ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B frontend_test_utils. The host_scheduler is setup to look for the 2984ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B host_scheduler_db when invoked with --testing. 2994ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 3004ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B @param db_file_path: The name of the file to use to create 3014ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B a SQLite database. Since this database is shared across different 3024ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B processes using a file is closer to the real world. 3034ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """ 3044ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B if not db_file_path: 3054ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B db_file_path = self.DB_FILE 3064ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B # TODO: Move the translating database elsewhere. Monitor_db circular 3074ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B # imports host_scheduler. 3084ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B from autotest_lib.frontend import setup_test_environment 3094ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B from django.conf import settings 3104ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B self.old_django_db_name = settings.DATABASES['default']['NAME'] 3114ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B settings.DATABASES['default']['NAME'] = db_file_path 3124ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B self.db_file_path = db_file_path 3134ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B _db_manager = scheduler_lib.ConnectionManager(autocommit=False) 3144ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B _db_manager.db_connection = ( 3154ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B database_connection.TranslatingDatabase.get_test_database( 3164ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B translators=scheduler_lib._DB_TRANSLATORS)) 3174ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 3184ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 3194ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def teardown_file_database(self): 3204ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B """Teardown django database settings.""" 3214ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B # TODO: Move the translating database elsewhere. Monitor_db circular 3224ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B # imports host_scheduler. 3234ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B from django.conf import settings 3244ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B settings.DATABASES['default']['NAME'] = self.old_django_db_name 3254ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B try: 3264ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B os.remove(self.db_file_path) 3274ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B except (OSError, AttributeError): 3284ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B pass 3294ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 3304ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B 3312d8047e8b2d901bec66d483664d8b6322501d245Prashanth Bclass AbstractBaseRDBTester(frontend_test_utils.FrontendTestMixin): 3322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B __meta__ = abc.ABCMeta 3342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B _config_section = 'AUTOTEST_WEB' 3352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @staticmethod 3382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def get_request(dep_names, acl_names, priority=0, parent_job_id=0): 3392d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps = [dep.id for dep in DBHelper.get_labels(name__in=dep_names)] 3402d8047e8b2d901bec66d483664d8b6322501d245Prashanth B acls = [acl.id for acl in DBHelper.get_acls(name__in=acl_names)] 3412d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return rdb_requests.AcquireHostRequest( 3422d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps=deps, acls=acls, host_id=None, priority=priority, 3432d8047e8b2d901bec66d483664d8b6322501d245Prashanth B parent_job_id=parent_job_id)._request 3442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def _release_unused_hosts(self): 3472d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Release all hosts unused by an active hqe. """ 3482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.host_scheduler.tick() 3492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3514ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B def setUp(self, inline_host_acquisition=True, setup_tables=True): 352f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B """Common setup module for tests that need a jobs/host database. 353f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B 354f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B @param inline_host_acquisition: If True, the dispatcher tries to acquire 355f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B hosts inline with the rest of the tick. 356f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B """ 3572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.db_helper = DBHelper() 3582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._database = self.db_helper.database 3592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Runs syncdb setting up initial database conditions 3604ec9867f46deb969c154bebf2e64729d56c3a1d3Prashanth B self._frontend_common_setup(setup_tables=setup_tables) 3610e960285b022fad77f0b087a2007867363bf6ab9Prashanth B connection_manager = scheduler_lib.ConnectionManager(autocommit=False) 3620e960285b022fad77f0b087a2007867363bf6ab9Prashanth B self.god.stub_with(connection_manager, 'db_connection', self._database) 3630e960285b022fad77f0b087a2007867363bf6ab9Prashanth B self.god.stub_with(monitor_db, '_db_manager', connection_manager) 3642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.god.stub_with(scheduler_models, '_db', self._database) 365f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B self.god.stub_with(monitor_db, '_inline_host_acquisition', 366f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B inline_host_acquisition) 3672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._dispatcher = monitor_db.Dispatcher() 3682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.host_scheduler = self._dispatcher._host_scheduler 369f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B self.host_query_manager = query_managers.AFEHostQueryManager() 370f66d51b5caa96995b91e7c155ff4378cdef4baafPrashanth B self.job_query_manager = self._dispatcher._job_query_manager 3712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._release_unused_hosts() 3722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def tearDown(self): 3752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.god.unstub_all() 3762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._database.disconnect() 3772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self._frontend_common_teardown() 3782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_job(self, user='autotest_system', 3812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps=set([]), acls=set([]), hostless_job=False, 38222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian priority=0, parent_job_id=None, shard_hostname=None): 3832d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Create a job owned by user, with the deps and acls specified. 3842d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3852d8047e8b2d901bec66d483664d8b6322501d245Prashanth B This method is a wrapper around frontend_test_utils.create_job, that 3862d8047e8b2d901bec66d483664d8b6322501d245Prashanth B also takes care of creating the appropriate deps for a job, and the 3872d8047e8b2d901bec66d483664d8b6322501d245Prashanth B appropriate acls for the given user. 3882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises ValueError: If no deps are specified for a job, since all jobs 3902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B need at least the metahost. 3912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If no hqe was created for the job. 3922d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 3932d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @return: An instance of the job model associated with the new job. 3942d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 3952d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # This is a slight hack around the implementation of 3962d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # scheduler_models.is_hostless_job, even though a metahost is just 3972d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # another label to the rdb. 3982d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if not deps: 3992d8047e8b2d901bec66d483664d8b6322501d245Prashanth B raise ValueError('Need at least one dep for metahost') 4002d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4012d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # TODO: This is a hack around the fact that frontend_test_utils still 4022d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # need a metahost, but metahost is treated like any other label. 4032d8047e8b2d901bec66d483664d8b6322501d245Prashanth B metahost = self.db_helper.create_label(list(deps)[0]) 4042d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job = self._create_job(metahosts=[metahost.id], priority=priority, 4052d8047e8b2d901bec66d483664d8b6322501d245Prashanth B owner=user, parent_job_id=parent_job_id) 4062d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(len(job.hostqueueentry_set.all()) == 1) 4072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.db_helper.add_deps_to_job(job, dep_names=list(deps)[1:]) 4092d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.db_helper.add_user_to_aclgroups(user, aclgroup_names=acls) 41022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian if shard_hostname: 41122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian self.db_helper.assign_job_to_shard(job.id, shard_hostname) 4122d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return models.Job.objects.filter(id=job.id)[0] 4132d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4142d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4152d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def assert_host_db_status(self, host_id): 4162d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Assert host state right after acquisition. 4172d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4182d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Call this method to check the status of any host leased by the 4192d8047e8b2d901bec66d483664d8b6322501d245Prashanth B rdb before it has been assigned to an hqe. It must be leased and 4202d8047e8b2d901bec66d483664d8b6322501d245Prashanth B ready at this point in time. 4212d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4222d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host_id: Id of the host to check. 4232d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4242d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If the host is either not leased or Ready. 4252d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 4262d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host = models.Host.objects.get(id=host_id) 4272d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(host.leased) 4282d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(host.status == 'Ready') 4292d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4302d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4312d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def check_hosts(self, host_iter): 4322d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Sanity check all hosts in the host_gen. 4332d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4342d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host_iter: A generator/iterator of RDBClientHostWrappers. 4352d8047e8b2d901bec66d483664d8b6322501d245Prashanth B eg: The generator returned by rdb_lib.acquire_hosts. If a request 4362d8047e8b2d901bec66d483664d8b6322501d245Prashanth B was not satisfied this iterator can contain None. 4372d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4382d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If any of the sanity checks fail. 4392d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 4402d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for host in host_iter: 4412d8047e8b2d901bec66d483664d8b6322501d245Prashanth B if host: 4422d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_host_db_status(host.id) 4432d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(host.leased == 1) 4442d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4452d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4462d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def create_suite(self, user='autotest_system', num=2, priority=0, 44752a239316b829106b540e57a0100496fed1fe5aaFang Deng board='z', build='x', acls=set()): 4482d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Create num jobs with the same parent_job_id, board, build, priority. 4492d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4502d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @return: A dictionary with the parent job object keyed as 'parent_job' 4512d8047e8b2d901bec66d483664d8b6322501d245Prashanth B and all other jobs keyed at an index from 0-num. 4522d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 4532d8047e8b2d901bec66d483664d8b6322501d245Prashanth B jobs = {} 4542d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Create a hostless parent job without an hqe or deps. Since the 4552d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # hostless job does nothing, we need to hand craft cros-version. 4562d8047e8b2d901bec66d483664d8b6322501d245Prashanth B parent_job = self._create_job(owner=user, priority=priority) 4572d8047e8b2d901bec66d483664d8b6322501d245Prashanth B jobs['parent_job'] = parent_job 4582d8047e8b2d901bec66d483664d8b6322501d245Prashanth B build = '%s:%s' % (provision.CROS_VERSION_PREFIX, build) 4592d8047e8b2d901bec66d483664d8b6322501d245Prashanth B for job_index in range(0, num): 4602d8047e8b2d901bec66d483664d8b6322501d245Prashanth B jobs[job_index] = self.create_job(user=user, priority=priority, 4612d8047e8b2d901bec66d483664d8b6322501d245Prashanth B deps=set([board, build]), 46252a239316b829106b540e57a0100496fed1fe5aaFang Deng acls=acls, 4632d8047e8b2d901bec66d483664d8b6322501d245Prashanth B parent_job_id=parent_job.id) 4642d8047e8b2d901bec66d483664d8b6322501d245Prashanth B return jobs 4652d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4662d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4672d8047e8b2d901bec66d483664d8b6322501d245Prashanth B def check_host_assignment(self, job_id, host_id): 4682d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """Check is a job<->host assignment is valid. 4692d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4702d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Uses the deps of a job and the aclgroups the owner of the job is 4712d8047e8b2d901bec66d483664d8b6322501d245Prashanth B in to see if the given host can be used to run the given job. Also 4722d8047e8b2d901bec66d483664d8b6322501d245Prashanth B checks that the host-job assignment has Not been made, but that the 4732d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host is no longer in the available hosts pool. 4742d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4752d8047e8b2d901bec66d483664d8b6322501d245Prashanth B Use this method to check host assignements made by the rdb, Before 4762d8047e8b2d901bec66d483664d8b6322501d245Prashanth B they're handed off to the scheduler, since the scheduler. 4772d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4782d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param job_id: The id of the job to use in the compatibility check. 4792d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @param host_id: The id of the host to check for compatibility. 4802d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4812d8047e8b2d901bec66d483664d8b6322501d245Prashanth B @raises AssertionError: If the job and the host are incompatible. 4822d8047e8b2d901bec66d483664d8b6322501d245Prashanth B """ 4832d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job = models.Job.objects.get(id=job_id) 4842d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host = models.Host.objects.get(id=host_id) 4852d8047e8b2d901bec66d483664d8b6322501d245Prashanth B hqe = job.hostqueueentry_set.all()[0] 4862d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4872d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Confirm that the host has not been assigned, either to another hqe 4882d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # or the this one. 4892d8047e8b2d901bec66d483664d8b6322501d245Prashanth B all_hqes = models.HostQueueEntry.objects.filter( 4902d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host_id=host_id, complete=0) 4912d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(len(all_hqes) <= 1) 4922d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(hqe.host_id == None) 4932d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_host_db_status(host_id) 4942d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 4952d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Assert that all deps of the job are satisfied. 4962d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job_deps = set([d.name for d in job.dependency_labels.all()]) 4972d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host_labels = set([l.name for l in host.labels.all()]) 4982d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(job_deps.intersection(host_labels) == job_deps) 4992d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 5002d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # Assert that the owner of the job is in at least one of the 5012d8047e8b2d901bec66d483664d8b6322501d245Prashanth B # groups that owns the host. 5022d8047e8b2d901bec66d483664d8b6322501d245Prashanth B job_owner_aclgroups = set([job_acl.name for job_acl 5032d8047e8b2d901bec66d483664d8b6322501d245Prashanth B in job.user().aclgroup_set.all()]) 5042d8047e8b2d901bec66d483664d8b6322501d245Prashanth B host_aclgroups = set([host_acl.name for host_acl 5052d8047e8b2d901bec66d483664d8b6322501d245Prashanth B in host.aclgroup_set.all()]) 5062d8047e8b2d901bec66d483664d8b6322501d245Prashanth B self.assert_(job_owner_aclgroups.intersection(host_aclgroups)) 5072d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 5082d8047e8b2d901bec66d483664d8b6322501d245Prashanth B 509