1b28def33556c59638a885cdc5710c9594c8fcd76Richard Barnette# pylint: disable=missing-docstring 20b9cfc93f17ec81535de54e3bae4f5f0e8d8595bAviv Keshet 32b79fd7627875033b2650c302e385712976aea7fFang Dengimport logging 4fb2a7fa621ddc91634dde6c56a47a1c8df2610efshowardfrom datetime import datetime 5fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshetimport django.core 6fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshettry: 7fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshet from django.db import models as dbmodels, connection 8fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshetexcept django.core.exceptions.ImproperlyConfigured: 9fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshet raise ImportError('Django database not yet configured. Import either ' 10fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshet 'setup_django_environment or ' 11fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshet 'setup_django_lite_environment from ' 12fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshet 'autotest_lib.frontend before any imports that ' 13fa1990038c71dd4abed91a145d7a186dd076a70fAviv Keshet 'depend on django models.') 1435a70227e69e03d0492fe47608a6370dc339f287jamesrenfrom xml.sax import saxutils 15cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showardimport common 16dd855244f44b65d0508345c6fef74846652c8c26jamesrenfrom autotest_lib.frontend.afe import model_logic, model_attributes 17489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth Bfrom autotest_lib.frontend.afe import rdb_model_extensions 18cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showardfrom autotest_lib.frontend import settings, thread_local 19a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelichfrom autotest_lib.client.common_lib import enum, error, host_protections 20a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelichfrom autotest_lib.client.common_lib import global_config 21eaa408e59e932ce0540aa5d50bf966af5080f640showardfrom autotest_lib.client.common_lib import host_queue_entry_states 22a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelichfrom autotest_lib.client.common_lib import control_data, priorities, decorators 236edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanianfrom autotest_lib.client.common_lib import site_utils 24b72f4fbcf1583da27f09f4abb9d8162530bf4559Gabe Blackfrom autotest_lib.client.common_lib.cros.graphite import autotest_es 250c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryufrom autotest_lib.server import utils as server_utils 26e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 270fc3830f17d644bab74bfe38556299f5e58bc0fashoward# job options and user preferences 28dd855244f44b65d0508345c6fef74846652c8c26jamesrenDEFAULT_REBOOT_BEFORE = model_attributes.RebootBefore.IF_DIRTY 2907e09aff0baf871b33e5479e337e5e3e0523b729Dan ShiDEFAULT_REBOOT_AFTER = model_attributes.RebootBefore.NEVER 30e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 3189f84dbadf071ba430244356b57af395c79486e4showard 32e8819cdf80ca0e0602d22551a50f970aa68e108dmblighclass AclAccessViolation(Exception): 330afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """\ 340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Raised when an operation is attempted with proper permissions as 350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski dictated by ACLs. 360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 37e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 38e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 39205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showardclass AtomicGroup(model_logic.ModelWithInvalid, dbmodels.Model): 4089f84dbadf071ba430244356b57af395c79486e4showard """\ 4189f84dbadf071ba430244356b57af395c79486e4showard An atomic group defines a collection of hosts which must only be scheduled 4289f84dbadf071ba430244356b57af395c79486e4showard all at once. Any host with a label having an atomic group will only be 4389f84dbadf071ba430244356b57af395c79486e4showard scheduled for a job at the same time as other hosts sharing that label. 4489f84dbadf071ba430244356b57af395c79486e4showard 4589f84dbadf071ba430244356b57af395c79486e4showard Required: 467db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey name: A name for this atomic group, e.g. 'rack23' or 'funky_net'. 4789f84dbadf071ba430244356b57af395c79486e4showard max_number_of_machines: The maximum number of machines that will be 4889f84dbadf071ba430244356b57af395c79486e4showard scheduled at once when scheduling jobs to this atomic group. 4989f84dbadf071ba430244356b57af395c79486e4showard The job.synch_count is considered the minimum. 5089f84dbadf071ba430244356b57af395c79486e4showard 5189f84dbadf071ba430244356b57af395c79486e4showard Optional: 5289f84dbadf071ba430244356b57af395c79486e4showard description: Arbitrary text description of this group's purpose. 5389f84dbadf071ba430244356b57af395c79486e4showard """ 54a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward name = dbmodels.CharField(max_length=255, unique=True) 5589f84dbadf071ba430244356b57af395c79486e4showard description = dbmodels.TextField(blank=True) 56e9450c934d860f154175abfee19389bf1a114305showard # This magic value is the default to simplify the scheduler logic. 57e9450c934d860f154175abfee19389bf1a114305showard # It must be "large". The common use of atomic groups is to want all 58e9450c934d860f154175abfee19389bf1a114305showard # machines in the group to be used, limits on which subset used are 59e9450c934d860f154175abfee19389bf1a114305showard # often chosen via dependency labels. 607db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey # TODO(dennisjeffrey): Revisit this so we don't have to assume that 617db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey # "infinity" is around 3.3 million. 62e9450c934d860f154175abfee19389bf1a114305showard INFINITE_MACHINES = 333333333 63e9450c934d860f154175abfee19389bf1a114305showard max_number_of_machines = dbmodels.IntegerField(default=INFINITE_MACHINES) 64205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard invalid = dbmodels.BooleanField(default=False, 65a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward editable=settings.FULL_ADMIN) 6689f84dbadf071ba430244356b57af395c79486e4showard 6789f84dbadf071ba430244356b57af395c79486e4showard name_field = 'name' 68e36562323bec14d14b7c583db94788a725514ff2jamesren objects = model_logic.ModelWithInvalidManager() 69205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard valid_objects = model_logic.ValidObjectsManager() 70205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard 71205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard 7229f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard def enqueue_job(self, job, is_template=False): 737db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Enqueue a job on an associated atomic group of hosts. 747db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 757db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param job: A job to enqueue. 767db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param is_template: Whether the status should be "Template". 777db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 7829f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard queue_entry = HostQueueEntry.create(atomic_group=self, job=job, 7929f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard is_template=is_template) 80c92da83b2dd5b7992a58081c6005b3dfb3dc5f3ashoward queue_entry.save() 81c92da83b2dd5b7992a58081c6005b3dfb3dc5f3ashoward 82c92da83b2dd5b7992a58081c6005b3dfb3dc5f3ashoward 83205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard def clean_object(self): 84205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard self.label_set.clear() 8589f84dbadf071ba430244356b57af395c79486e4showard 8689f84dbadf071ba430244356b57af395c79486e4showard 8789f84dbadf071ba430244356b57af395c79486e4showard class Meta: 887db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class AtomicGroup.""" 89eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_atomic_groups' 9089f84dbadf071ba430244356b57af395c79486e4showard 91205fd60f9c9d2f64ec2773f295de1cf5cfd3bc77showard 92a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 93a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.name) 9489f84dbadf071ba430244356b57af395c79486e4showard 9589f84dbadf071ba430244356b57af395c79486e4showard 967c7852819d0611b4d0e8e69b3011b79e4016a770showardclass Label(model_logic.ModelWithInvalid, dbmodels.Model): 970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """\ 980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Required: 9989f84dbadf071ba430244356b57af395c79486e4showard name: label name 100e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Optional: 10289f84dbadf071ba430244356b57af395c79486e4showard kernel_config: URL/path to kernel config for jobs run on this label. 10389f84dbadf071ba430244356b57af395c79486e4showard platform: If True, this is a platform label (defaults to False). 10489f84dbadf071ba430244356b57af395c79486e4showard only_if_needed: If True, a Host with this label can only be used if that 10589f84dbadf071ba430244356b57af395c79486e4showard label is requested by the job/test (either as the meta_host or 10689f84dbadf071ba430244356b57af395c79486e4showard in the job_dependencies). 10789f84dbadf071ba430244356b57af395c79486e4showard atomic_group: The atomic group associated with this label. 1080afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 109a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward name = dbmodels.CharField(max_length=255, unique=True) 110a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward kernel_config = dbmodels.CharField(max_length=255, blank=True) 1110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski platform = dbmodels.BooleanField(default=False) 1120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski invalid = dbmodels.BooleanField(default=False, 1130afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski editable=settings.FULL_ADMIN) 114b1e5187f9aa303c4fc914f07312286d302b46a0eshoward only_if_needed = dbmodels.BooleanField(default=False) 115e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name_field = 'name' 117e36562323bec14d14b7c583db94788a725514ff2jamesren objects = model_logic.ModelWithInvalidManager() 1180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski valid_objects = model_logic.ValidObjectsManager() 11989f84dbadf071ba430244356b57af395c79486e4showard atomic_group = dbmodels.ForeignKey(AtomicGroup, null=True, blank=True) 12089f84dbadf071ba430244356b57af395c79486e4showard 1215244cbb54ca3d426b35561c4853c356f3f51f5fdmbligh 1220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def clean_object(self): 1230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.host_set.clear() 12401a5167f13d9788c9f359ebba31358e329c98ebcshoward self.test_set.clear() 125e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 126e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 127204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li def enqueue_job(self, job, is_template=False): 1287db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Enqueue a job on any host of this label. 1297db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 1307db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param job: A job to enqueue. 1317db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param is_template: Whether the status should be "Template". 1327db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 13329f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard queue_entry = HostQueueEntry.create(meta_host=self, job=job, 134204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li is_template=is_template) 1350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski queue_entry.save() 136e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 137e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 138ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 1407db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class Label.""" 141eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_labels' 142e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 143ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 144a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 145a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.name) 146e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 147e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 14892c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülichclass Shard(dbmodels.Model, model_logic.ModelExtensions): 14992c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich 150de2b9a93674cb762230a83b1c41cc373073e4fdbJakob Juelich hostname = dbmodels.CharField(max_length=255, unique=True) 151de2b9a93674cb762230a83b1c41cc373073e4fdbJakob Juelich 152de2b9a93674cb762230a83b1c41cc373073e4fdbJakob Juelich name_field = 'hostname' 153de2b9a93674cb762230a83b1c41cc373073e4fdbJakob Juelich 15492c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich labels = dbmodels.ManyToManyField(Label, blank=True, 15592c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich db_table='afe_shards_labels') 15692c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich 15792c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich class Meta: 15892c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich """Metadata for class ParameterizedJob.""" 15992c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich db_table = 'afe_shards' 16092c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich 16192c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich 1626edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian def rpc_hostname(self): 1636edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian """Get the rpc hostname of the shard. 1646edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian 1656edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian @return: Just the shard hostname for all non-testing environments. 1666edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian The address of the default gateway for vm testing environments. 1676edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian """ 1686edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # TODO: Figure out a better solution for testing. Since no 2 shards 1696edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # can run on the same host, if the shard hostname is localhost we 1706edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # conclude that it must be a vm in a test cluster. In such situations 1716edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # a name of localhost:<port> is necessary to achieve the correct 1726edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # afe links/redirection from the frontend (this happens through the 1736edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # host), but for rpcs that are performed *on* the shard, they need to 1746edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian # use the address of the gateway. 1758f8cdb425db94473f50579e2123f61831fac7322MK Ryu # In the virtual machine testing environment (i.e., puppylab), each 1768f8cdb425db94473f50579e2123f61831fac7322MK Ryu # shard VM has a hostname like localhost:<port>. In the real cluster 1778f8cdb425db94473f50579e2123f61831fac7322MK Ryu # environment, a shard node does not have 'localhost' for its hostname. 1788f8cdb425db94473f50579e2123f61831fac7322MK Ryu # The following hostname substitution is needed only for the VM 1798f8cdb425db94473f50579e2123f61831fac7322MK Ryu # in puppylab. 1808f8cdb425db94473f50579e2123f61831fac7322MK Ryu # The 'hostname' should not be replaced in the case of real cluster. 1818f8cdb425db94473f50579e2123f61831fac7322MK Ryu if site_utils.is_puppylab_vm(self.hostname): 1828f8cdb425db94473f50579e2123f61831fac7322MK Ryu hostname = self.hostname.split(':')[0] 1836edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian return self.hostname.replace( 1846edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian hostname, site_utils.DEFAULT_VM_GATEWAY) 1856edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian return self.hostname 1866edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian 1876edaaf9f5d10cda21d1c7f2401eb1d0bef2192faPrashanth Balasubramanian 18876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesrenclass Drone(dbmodels.Model, model_logic.ModelExtensions): 18976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 19076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren A scheduler drone 19176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 19276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren hostname: the drone's hostname 19376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 19476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren hostname = dbmodels.CharField(max_length=255, unique=True) 19576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 19676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren name_field = 'hostname' 19776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren objects = model_logic.ExtendedManager() 19876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 19976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 20076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def save(self, *args, **kwargs): 20176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren if not User.current_user().is_superuser(): 20276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren raise Exception('Only superusers may edit drones') 20376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren super(Drone, self).save(*args, **kwargs) 20476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 20576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 20676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def delete(self): 20776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren if not User.current_user().is_superuser(): 20876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren raise Exception('Only superusers may delete drones') 20976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren super(Drone, self).delete() 21076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 21176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 21276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren class Meta: 2137db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class Drone.""" 21476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren db_table = 'afe_drones' 21576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 21676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def __unicode__(self): 21776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return unicode(self.hostname) 21876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 21976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 22076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesrenclass DroneSet(dbmodels.Model, model_logic.ModelExtensions): 22176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 22276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren A set of scheduler drones 22376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 22476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren These will be used by the scheduler to decide what drones a job is allowed 22576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren to run on. 22676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 22776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren name: the drone set's name 22876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren drones: the drones that are part of the set 22976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 23076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren DRONE_SETS_ENABLED = global_config.global_config.get_config_value( 23176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 'SCHEDULER', 'drone_sets_enabled', type=bool, default=False) 23276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren DEFAULT_DRONE_SET_NAME = global_config.global_config.get_config_value( 23376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 'SCHEDULER', 'default_drone_set_name', default=None) 23476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 23576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren name = dbmodels.CharField(max_length=255, unique=True) 23676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren drones = dbmodels.ManyToManyField(Drone, db_table='afe_drone_sets_drones') 23776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 23876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren name_field = 'name' 23976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren objects = model_logic.ExtendedManager() 24076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 24176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 24276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def save(self, *args, **kwargs): 24376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren if not User.current_user().is_superuser(): 24476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren raise Exception('Only superusers may edit drone sets') 24576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren super(DroneSet, self).save(*args, **kwargs) 24676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 24776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 24876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def delete(self): 24976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren if not User.current_user().is_superuser(): 25076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren raise Exception('Only superusers may delete drone sets') 25176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren super(DroneSet, self).delete() 25276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 25376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 25476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren @classmethod 25576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def drone_sets_enabled(cls): 2567db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns whether drone sets are enabled. 2577db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 2587db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 2597db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 26076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return cls.DRONE_SETS_ENABLED 26176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 26276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 26376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren @classmethod 26476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def default_drone_set_name(cls): 2657db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns the default drone set name. 2667db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 2677db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 2687db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 26976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return cls.DEFAULT_DRONE_SET_NAME 27076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 27176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 27276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren @classmethod 27376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def get_default(cls): 2747db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Gets the default drone set name, compatible with Job.add_object. 2757db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 2767db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 2777db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 27876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return cls.smart_get(cls.DEFAULT_DRONE_SET_NAME) 27976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 28076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 28176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren @classmethod 28276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def resolve_name(cls, drone_set_name): 28376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 28476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren Returns the name of one of these, if not None, in order of preference: 28576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 1) the drone set given, 28676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 2) the current user's default drone set, or 28776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 3) the global default drone set 28876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 28976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren or returns None if drone sets are disabled 2907db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 2917db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 2927db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param drone_set_name: A drone set name. 29376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 29476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren if not cls.drone_sets_enabled(): 29576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return None 29676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 29776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren user = User.current_user() 29876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren user_drone_set_name = user.drone_set and user.drone_set.name 29976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 30076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return drone_set_name or user_drone_set_name or cls.get_default().name 30176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 30276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 30376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def get_drone_hostnames(self): 30476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 30576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren Gets the hostnames of all drones in this drone set 30676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren """ 30776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return set(self.drones.all().values_list('hostname', flat=True)) 30876fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 30976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 31076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren class Meta: 3117db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class DroneSet.""" 31276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren db_table = 'afe_drone_sets' 31376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 31476fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren def __unicode__(self): 31576fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren return unicode(self.name) 31676fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 31776fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 318fb2a7fa621ddc91634dde6c56a47a1c8df2610efshowardclass User(dbmodels.Model, model_logic.ModelExtensions): 319fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward """\ 320fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward Required: 321fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward login :user login name 322fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 323fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward Optional: 324fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward access_level: 0=User (default), 1=Admin, 100=Root 325fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward """ 326fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward ACCESS_ROOT = 100 327fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward ACCESS_ADMIN = 1 328fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward ACCESS_USER = 0 329fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 33064a9595406f2884fb3ece241190b10aa054439a9showard AUTOTEST_SYSTEM = 'autotest_system' 33164a9595406f2884fb3ece241190b10aa054439a9showard 332a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward login = dbmodels.CharField(max_length=255, unique=True) 333fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True) 334fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 3350fc3830f17d644bab74bfe38556299f5e58bc0fashoward # user preferences 336dd855244f44b65d0508345c6fef74846652c8c26jamesren reboot_before = dbmodels.SmallIntegerField( 337dd855244f44b65d0508345c6fef74846652c8c26jamesren choices=model_attributes.RebootBefore.choices(), blank=True, 338dd855244f44b65d0508345c6fef74846652c8c26jamesren default=DEFAULT_REBOOT_BEFORE) 339dd855244f44b65d0508345c6fef74846652c8c26jamesren reboot_after = dbmodels.SmallIntegerField( 340dd855244f44b65d0508345c6fef74846652c8c26jamesren choices=model_attributes.RebootAfter.choices(), blank=True, 341dd855244f44b65d0508345c6fef74846652c8c26jamesren default=DEFAULT_REBOOT_AFTER) 34276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren drone_set = dbmodels.ForeignKey(DroneSet, null=True, blank=True) 34397db5ba6cdee21d706dff3d1a85c354c8bfae6f9showard show_experimental = dbmodels.BooleanField(default=False) 3440fc3830f17d644bab74bfe38556299f5e58bc0fashoward 345fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward name_field = 'login' 346fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward objects = model_logic.ExtendedManager() 347fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 348fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 349a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def save(self, *args, **kwargs): 350fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward # is this a new object being saved for the first time? 351fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward first_time = (self.id is None) 352fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward user = thread_local.get_user() 3530fc3830f17d644bab74bfe38556299f5e58bc0fashoward if user and not user.is_superuser() and user.login != self.login: 3540fc3830f17d644bab74bfe38556299f5e58bc0fashoward raise AclAccessViolation("You cannot modify user " + self.login) 355a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward super(User, self).save(*args, **kwargs) 356fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward if first_time: 357fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward everyone = AclGroup.objects.get(name='Everyone') 358fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward everyone.users.add(self) 359fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 360fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 361fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward def is_superuser(self): 3627db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns whether the user has superuser access.""" 363fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward return self.access_level >= self.ACCESS_ROOT 364fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 365fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 36664a9595406f2884fb3ece241190b10aa054439a9showard @classmethod 36764a9595406f2884fb3ece241190b10aa054439a9showard def current_user(cls): 3687db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns the current user. 3697db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 3707db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 3717db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 37264a9595406f2884fb3ece241190b10aa054439a9showard user = thread_local.get_user() 37364a9595406f2884fb3ece241190b10aa054439a9showard if user is None: 374cfcdd80cb19eb28f593a79f9ba2e56496c3f9f3ashoward user, _ = cls.objects.get_or_create(login=cls.AUTOTEST_SYSTEM) 37564a9595406f2884fb3ece241190b10aa054439a9showard user.access_level = cls.ACCESS_ROOT 37664a9595406f2884fb3ece241190b10aa054439a9showard user.save() 37764a9595406f2884fb3ece241190b10aa054439a9showard return user 37864a9595406f2884fb3ece241190b10aa054439a9showard 37964a9595406f2884fb3ece241190b10aa054439a9showard 380af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian @classmethod 381af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian def get_record(cls, data): 382af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian """Check the database for an identical record. 383af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 384af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian Check for a record with matching id and login. If one exists, 385af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian return it. If one does not exist there is a possibility that 386af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian the following cases have happened: 387af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 1. Same id, different login 388af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian We received: "1 chromeos-test" 389af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian And we have: "1 debug-user" 390af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian In this case we need to delete "1 debug_user" and insert 391af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian "1 chromeos-test". 392af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 393af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 2. Same login, different id: 394af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian We received: "1 chromeos-test" 395af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian And we have: "2 chromeos-test" 396af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian In this case we need to delete "2 chromeos-test" and insert 397af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian "1 chromeos-test". 398af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 399af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian As long as this method deletes bad records and raises the 400af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian DoesNotExist exception the caller will handle creating the 401af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian new record. 402af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 403af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian @raises: DoesNotExist, if a record with the matching login and id 404af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian does not exist. 405af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian """ 406af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 407af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # Both the id and login should be uniqe but there are cases when 408af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # we might already have a user with the same login/id because 409af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # current_user will proactively create a user record if it doesn't 410af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # exist. Since we want to avoid conflict between the master and 411af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # shard, just delete any existing user records that don't match 412af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # what we're about to deserialize from the master. 413af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian try: 414af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian return cls.objects.get(login=data['login'], id=data['id']) 415af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian except cls.DoesNotExist: 416af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian cls.delete_matching_record(login=data['login']) 417af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian cls.delete_matching_record(id=data['id']) 418af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian raise 419af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 420af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 421fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward class Meta: 4227db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class User.""" 423eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_users' 424fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 425a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 426a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.login) 427fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 428fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward 429489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth Bclass Host(model_logic.ModelWithInvalid, rdb_model_extensions.AbstractHostModel, 430f8b19046c4496f570c776b65288382108a633ab4showard model_logic.ModelWithAttributes): 4310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """\ 4320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Required: 4330afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski hostname 4340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski optional: 43621baa459ea14f96e06212f1f35fcddab9442b3fcshoward locked: if true, host is locked and will not be queued 4370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Internal: 439489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B From AbstractHostModel: 440489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B status: string describing status of host 441489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B invalid: true if the host has been deleted 442489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B protection: indicates what can be done to this host during repair 443489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B lock_time: DateTime at which the host was locked 444489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B dirty: true if the host has been used without being rebooted 445489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B Local: 446489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B locked_by: user that locked the host, or null if the host is unlocked 4470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4480afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4493bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich SERIALIZATION_LINKS_TO_FOLLOW = set(['aclgroup_set', 4503bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 'hostattribute_set', 4513bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 'labels', 4523bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 'shard']) 4535949b4af7a872aeb58e7ad29090812d648725ed5Prashanth Balasubramanian SERIALIZATION_LOCAL_LINKS_TO_UPDATE = set(['invalid']) 4543bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 455f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 456f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich def custom_deserialize_relation(self, link, data): 457116ff0fc2674082fe475afd64ce4dec998ed71b7Jakob Juelich assert link == 'shard', 'Link %s should not be deserialized' % link 458f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich self.shard = Shard.deserialize(data) 459f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 460f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 461489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B # Note: Only specify foreign keys here, specify all native host columns in 462489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B # rdb_model_extensions instead. 463489b91d72cd225e902081dbd3f9e47448fe867f6Prashanth B Protection = host_protections.Protection 464eab66ce582bfe05076ff096c3a044d8f0497bbcashoward labels = dbmodels.ManyToManyField(Label, blank=True, 465eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table='afe_hosts_labels') 466fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward locked_by = dbmodels.ForeignKey(User, null=True, blank=True, editable=False) 4670afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name_field = 'hostname' 468e36562323bec14d14b7c583db94788a725514ff2jamesren objects = model_logic.ModelWithInvalidManager() 4690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski valid_objects = model_logic.ValidObjectsManager() 470cc9fc70587d37775673e47b3dcb4d6ded0c6dcb4beeps leased_objects = model_logic.LeasedHostManager() 4710afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 472de2b9a93674cb762230a83b1c41cc373073e4fdbJakob Juelich shard = dbmodels.ForeignKey(Shard, blank=True, null=True) 4732bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 4742bab8f45adedeacbf2d62d37b90255581adc3c7dshoward def __init__(self, *args, **kwargs): 4752bab8f45adedeacbf2d62d37b90255581adc3c7dshoward super(Host, self).__init__(*args, **kwargs) 4762bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self._record_attributes(['status']) 4772bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 4782bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 479b8471e33ed22512001ec4cec8c33fdf01f32eb62showard @staticmethod 480b8471e33ed22512001ec4cec8c33fdf01f32eb62showard def create_one_time_host(hostname): 4817db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Creates a one-time host. 4827db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 4837db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param hostname: The name for the host. 4847db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 485b8471e33ed22512001ec4cec8c33fdf01f32eb62showard query = Host.objects.filter(hostname=hostname) 486b8471e33ed22512001ec4cec8c33fdf01f32eb62showard if query.count() == 0: 487b8471e33ed22512001ec4cec8c33fdf01f32eb62showard host = Host(hostname=hostname, invalid=True) 488a8411aff779fc965537d8470268107b5a87e0346showard host.do_validate() 489b8471e33ed22512001ec4cec8c33fdf01f32eb62showard else: 490b8471e33ed22512001ec4cec8c33fdf01f32eb62showard host = query[0] 491b8471e33ed22512001ec4cec8c33fdf01f32eb62showard if not host.invalid: 492b8471e33ed22512001ec4cec8c33fdf01f32eb62showard raise model_logic.ValidationError({ 493b5b7b5dc37012db192e9e47cd8e6fe45faf48dd2mbligh 'hostname' : '%s already exists in the autotest DB. ' 494b5b7b5dc37012db192e9e47cd8e6fe45faf48dd2mbligh 'Select it rather than entering it as a one time ' 495b5b7b5dc37012db192e9e47cd8e6fe45faf48dd2mbligh 'host.' % hostname 496b8471e33ed22512001ec4cec8c33fdf01f32eb62showard }) 4971ab512bb148ff7706cde7d74eee9de08c47118e0showard host.protection = host_protections.Protection.DO_NOT_REPAIR 498946a7af939c07a64a627234acb094688125c4683showard host.locked = False 499b8471e33ed22512001ec4cec8c33fdf01f32eb62showard host.save() 5002924b0ac9e0ca35e2cd45a23b60ecfc204360c44showard host.clean_object() 501b8471e33ed22512001ec4cec8c33fdf01f32eb62showard return host 5020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5031ff7b2e88ae7a382f85ab76e786a471134e8a6a0showard 50459cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich @classmethod 5051b52574752be108a743d3b33561c34324f8538e7Jakob Juelich def assign_to_shard(cls, shard, known_ids): 50659cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich """Assigns hosts to a shard. 50759cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 508638a6cf6935329fd726bade7a87a45a180218f29MK Ryu For all labels that have been assigned to a shard, all hosts that 509638a6cf6935329fd726bade7a87a45a180218f29MK Ryu have at least one of the shard's labels are assigned to the shard. 5101b52574752be108a743d3b33561c34324f8538e7Jakob Juelich Hosts that are assigned to the shard but aren't already present on the 5111b52574752be108a743d3b33561c34324f8538e7Jakob Juelich shard are returned. 51259cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 513638a6cf6935329fd726bade7a87a45a180218f29MK Ryu Board to shard mapping is many-to-one. Many different boards can be 514638a6cf6935329fd726bade7a87a45a180218f29MK Ryu hosted in a shard. However, DUTs of a single board cannot be distributed 515638a6cf6935329fd726bade7a87a45a180218f29MK Ryu into more than one shard. 516638a6cf6935329fd726bade7a87a45a180218f29MK Ryu 51759cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich @param shard: The shard object to assign labels/hosts for. 5181b52574752be108a743d3b33561c34324f8538e7Jakob Juelich @param known_ids: List of all host-ids the shard already knows. 5191b52574752be108a743d3b33561c34324f8538e7Jakob Juelich This is used to figure out which hosts should be sent 5201b52574752be108a743d3b33561c34324f8538e7Jakob Juelich to the shard. If shard_ids were used instead, hosts 5211b52574752be108a743d3b33561c34324f8538e7Jakob Juelich would only be transferred once, even if the client 5221b52574752be108a743d3b33561c34324f8538e7Jakob Juelich failed persisting them. 5231b52574752be108a743d3b33561c34324f8538e7Jakob Juelich The number of hosts usually lies in O(100), so the 5241b52574752be108a743d3b33561c34324f8538e7Jakob Juelich overhead is acceptable. 5251b52574752be108a743d3b33561c34324f8538e7Jakob Juelich 52659cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich @returns the hosts objects that should be sent to the shard. 52759cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich """ 52859cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 52959cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # Disclaimer: concurrent heartbeats should theoretically not occur in 53059cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # the current setup. As they may be introduced in the near future, 53159cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # this comment will be left here. 53259cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 53359cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # Sending stuff twice is acceptable, but forgetting something isn't. 53459cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # Detecting duplicates on the client is easy, but here it's harder. The 53559cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # following options were considered: 53659cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # - SELECT ... WHERE and then UPDATE ... WHERE: Update might update more 53759cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # than select returned, as concurrently more hosts might have been 53859cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # inserted 53959cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # - UPDATE and then SELECT WHERE shard=shard: select always returns all 54059cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # hosts for the shard, this is overhead 54159cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # - SELECT and then UPDATE only selected without requerying afterwards: 54259cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # returns the old state of the records. 543638a6cf6935329fd726bade7a87a45a180218f29MK Ryu host_ids = set(Host.objects.filter( 544638a6cf6935329fd726bade7a87a45a180218f29MK Ryu labels__in=shard.labels.all(), 54559cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich leased=False 5461b52574752be108a743d3b33561c34324f8538e7Jakob Juelich ).exclude( 5471b52574752be108a743d3b33561c34324f8538e7Jakob Juelich id__in=known_ids, 54859cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich ).values_list('pk', flat=True)) 54959cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 55059cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich if host_ids: 55159cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich Host.objects.filter(pk__in=host_ids).update(shard=shard) 55259cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich return list(Host.objects.filter(pk__in=host_ids).all()) 55359cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich return [] 55459cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 555afd97de523a8c8c4e6657fa2db6214fda68d8086showard def resurrect_object(self, old_object): 556afd97de523a8c8c4e6657fa2db6214fda68d8086showard super(Host, self).resurrect_object(old_object) 557afd97de523a8c8c4e6657fa2db6214fda68d8086showard # invalid hosts can be in use by the scheduler (as one-time hosts), so 558afd97de523a8c8c4e6657fa2db6214fda68d8086showard # don't change the status 559afd97de523a8c8c4e6657fa2db6214fda68d8086showard self.status = old_object.status 560afd97de523a8c8c4e6657fa2db6214fda68d8086showard 561afd97de523a8c8c4e6657fa2db6214fda68d8086showard 5620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def clean_object(self): 5630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.aclgroup_set.clear() 5640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.labels.clear() 5650afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 5660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 567a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi def record_state(self, type_str, state, value, other_metadata=None): 568a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi """Record metadata in elasticsearch. 569a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi 570a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi @param type_str: sets the _type field in elasticsearch db. 571a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi @param state: string representing what state we are recording, 572a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi e.g. 'locked' 573a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi @param value: value of the state, e.g. True 574a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi @param other_metadata: Other metadata to store in metaDB. 575a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi """ 576a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi metadata = { 577a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi state: value, 578a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi 'hostname': self.hostname, 579a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi } 580a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi if other_metadata: 581a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi metadata = dict(metadata.items() + other_metadata.items()) 5826818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori autotest_es.post(use_http=True, type_str=type_str, metadata=metadata) 583a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi 584a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi 585a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def save(self, *args, **kwargs): 5860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # extra spaces in the hostname can be a sneaky source of errors 5870afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.hostname = self.hostname.strip() 5880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # is this a new object being saved for the first time? 5890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski first_time = (self.id is None) 5903dd47c247147ca6920b222d43a8eb6ee54209c8fshoward if not first_time: 5913dd47c247147ca6920b222d43a8eb6ee54209c8fshoward AclGroup.check_for_acl_violation_hosts([self]) 592a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi # If locked is changed, send its status and user made the change to 593a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi # metaDB. Locks are important in host history because if a device is 594a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi # locked then we don't really care what state it is in. 595fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward if self.locked and not self.locked_by: 59664a9595406f2884fb3ece241190b10aa054439a9showard self.locked_by = User.current_user() 597d53e1492e481c3e9d07ba790577914ba20d7631aMK Ryu if not self.lock_time: 598d53e1492e481c3e9d07ba790577914ba20d7631aMK Ryu self.lock_time = datetime.now() 599a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi self.record_state('lock_history', 'locked', self.locked, 6006818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori {'changed_by': self.locked_by.login, 6016818633834ad52c3de153235639ea9299a6e9a6dMatthew Sartori 'lock_reason': self.lock_reason}) 60221baa459ea14f96e06212f1f35fcddab9442b3fcshoward self.dirty = True 603fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward elif not self.locked and self.locked_by: 604a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi self.record_state('lock_history', 'locked', self.locked, 605a0acfbcb08a96fbe13eeb68203e119f71f1ce56cDan Shi {'changed_by': self.locked_by.login}) 606fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward self.locked_by = None 607fb2a7fa621ddc91634dde6c56a47a1c8df2610efshoward self.lock_time = None 608a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward super(Host, self).save(*args, **kwargs) 6090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if first_time: 6100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski everyone = AclGroup.objects.get(name='Everyone') 6110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski everyone.hosts.add(self) 6122bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self._check_for_updated_attributes() 6132bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 6140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 615b8471e33ed22512001ec4cec8c33fdf01f32eb62showard def delete(self): 6163dd47c247147ca6920b222d43a8eb6ee54209c8fshoward AclGroup.check_for_acl_violation_hosts([self]) 617b8471e33ed22512001ec4cec8c33fdf01f32eb62showard for queue_entry in self.hostqueueentry_set.all(): 618b8471e33ed22512001ec4cec8c33fdf01f32eb62showard queue_entry.deleted = True 61964a9595406f2884fb3ece241190b10aa054439a9showard queue_entry.abort() 620b8471e33ed22512001ec4cec8c33fdf01f32eb62showard super(Host, self).delete() 621b8471e33ed22512001ec4cec8c33fdf01f32eb62showard 6220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 6232bab8f45adedeacbf2d62d37b90255581adc3c7dshoward def on_attribute_changed(self, attribute, old_value): 6242bab8f45adedeacbf2d62d37b90255581adc3c7dshoward assert attribute == 'status' 625f1175bb2edbe3215f4bfa117c6b93d2e090d1d07showard logging.info(self.hostname + ' -> ' + self.status) 6262bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 6272bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 628204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li def enqueue_job(self, job, is_template=False): 6297db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Enqueue a job on this host. 6307db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 6317db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param job: A job to enqueue. 6327db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param is_template: Whther the status should be "Template". 6337db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 63429f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard queue_entry = HostQueueEntry.create(host=self, job=job, 635204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li is_template=is_template) 6360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # allow recovery of dead hosts from the frontend 6370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not self.active_queue_entry() and self.is_dead(): 6380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.status = Host.Status.READY 6390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski self.save() 6400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski queue_entry.save() 6410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 64208f981bddc5f1a199d789d177cbf583dee9f6c17showard block = IneligibleHostQueue(job=job, host=self) 64308f981bddc5f1a199d789d177cbf583dee9f6c17showard block.save() 64408f981bddc5f1a199d789d177cbf583dee9f6c17showard 6450afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 6460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def platform(self): 6477db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """The platform of the host.""" 6480afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # TODO(showard): slighly hacky? 6490afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski platforms = self.labels.filter(platform=True) 6500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if len(platforms) == 0: 6510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 6520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return platforms[0] 6530afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski platform.short_description = 'Platform' 6540afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 6550afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 656cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard @classmethod 657cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard def check_no_platform(cls, hosts): 6587db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Verify the specified hosts have no associated platforms. 6597db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 6607db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 6617db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param hosts: The hosts to verify. 6627db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @raises model_logic.ValidationError if any hosts already have a 6637db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey platform. 6647db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 665cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard Host.objects.populate_relationships(hosts, Label, 'label_list') 666cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard errors = [] 667cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard for host in hosts: 668cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard platforms = [label.name for label in host.label_list 669cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard if label.platform] 670cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard if platforms: 671cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard # do a join, just in case this host has multiple platforms, 672cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard # we'll be able to see it 673cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard errors.append('Host %s already has a platform: %s' % ( 674cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard host.hostname, ', '.join(platforms))) 675cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard if errors: 676cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard raise model_logic.ValidationError({'labels': '; '.join(errors)}) 677cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard 678cafd16e33f658f63fcbe09b0f232bbeb354ae3c2showard 67940e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao @classmethod 680b5b8b4f981036971c619b26956c8847140eabd35Dan Shi def check_board_labels_allowed(cls, hosts, new_labels=[]): 681b5b8b4f981036971c619b26956c8847140eabd35Dan Shi """Verify the specified hosts have valid board labels and the given 682b5b8b4f981036971c619b26956c8847140eabd35Dan Shi new board labels can be added. 68340e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao 68440e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao @param cls: Implicit class object. 68540e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao @param hosts: The hosts to verify. 686b5b8b4f981036971c619b26956c8847140eabd35Dan Shi @param new_labels: A list of labels to be added to the hosts. 687b5b8b4f981036971c619b26956c8847140eabd35Dan Shi 688b5b8b4f981036971c619b26956c8847140eabd35Dan Shi @raises model_logic.ValidationError if any host has invalid board labels 689b5b8b4f981036971c619b26956c8847140eabd35Dan Shi or the given board labels cannot be added to the hsots. 69040e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao """ 69140e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao Host.objects.populate_relationships(hosts, Label, 'label_list') 69240e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao errors = [] 69340e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao for host in hosts: 69440e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao boards = [label.name for label in host.label_list 69540e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao if label.name.startswith('board:')] 696b5b8b4f981036971c619b26956c8847140eabd35Dan Shi if not server_utils.board_labels_allowed(boards + new_labels): 69740e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao # do a join, just in case this host has multiple boards, 69840e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao # we'll be able to see it 699b5b8b4f981036971c619b26956c8847140eabd35Dan Shi errors.append('Host %s already has board labels: %s' % ( 70040e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao host.hostname, ', '.join(boards))) 70140e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao if errors: 70240e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao raise model_logic.ValidationError({'labels': '; '.join(errors)}) 70340e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao 70440e182b10320d3a8596a7a4eb87d2ec6b981bc71Shuqian Zhao 7050afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def is_dead(self): 7067db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns whether the host is dead (has status repair failed).""" 7070afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return self.status == Host.Status.REPAIR_FAILED 7080afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 7090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 7100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def active_queue_entry(self): 7117db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns the active queue entry for this host, or None if none.""" 7120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski active = list(self.hostqueueentry_set.filter(active=True)) 7130afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not active: 7140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 7150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski assert len(active) == 1, ('More than one active entry for ' 7160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 'host ' + self.hostname) 7170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return active[0] 7180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 7190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 720f8b19046c4496f570c776b65288382108a633ab4showard def _get_attribute_model_and_args(self, attribute): 721f8b19046c4496f570c776b65288382108a633ab4showard return HostAttribute, dict(host=self, attribute=attribute) 7220957a848a17b726836aaf9b5532bbfac691d983dshoward 7230957a848a17b726836aaf9b5532bbfac691d983dshoward 724ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @classmethod 725ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng def get_attribute_model(cls): 726ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """Return the attribute model. 727ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 728ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Override method in parent class. See ModelExtensions for details. 729ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @returns: The attribute model of Host. 730ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """ 731ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng return HostAttribute 732ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 733ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 7340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 7357db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for the Host class.""" 736eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_hosts' 7370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 738ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 739a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 740a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.hostname) 741e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 742e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 743acf359283e28219cc42ff3fe82b8a53f6a2d90f3MK Ryuclass HostAttribute(dbmodels.Model, model_logic.ModelExtensions): 7440957a848a17b726836aaf9b5532bbfac691d983dshoward """Arbitrary keyvals associated with hosts.""" 74586248502517d42d6c036a66a57e3a24d4a6dc3b8Fang Deng 74686248502517d42d6c036a66a57e3a24d4a6dc3b8Fang Deng SERIALIZATION_LINKS_TO_KEEP = set(['host']) 747ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng SERIALIZATION_LOCAL_LINKS_TO_UPDATE = set(['value']) 7480957a848a17b726836aaf9b5532bbfac691d983dshoward host = dbmodels.ForeignKey(Host) 749a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward attribute = dbmodels.CharField(max_length=90) 750a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward value = dbmodels.CharField(max_length=300) 7510957a848a17b726836aaf9b5532bbfac691d983dshoward 7520957a848a17b726836aaf9b5532bbfac691d983dshoward objects = model_logic.ExtendedManager() 7530957a848a17b726836aaf9b5532bbfac691d983dshoward 7540957a848a17b726836aaf9b5532bbfac691d983dshoward class Meta: 7557db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for the HostAttribute class.""" 756eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_host_attributes' 7570957a848a17b726836aaf9b5532bbfac691d983dshoward 7580957a848a17b726836aaf9b5532bbfac691d983dshoward 759ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @classmethod 760ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng def get_record(cls, data): 761ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """Check the database for an identical record. 762ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 763ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Use host_id and attribute to search for a existing record. 764ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 765ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @raises: DoesNotExist, if no record found 766ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @raises: MultipleObjectsReturned if multiple records found. 767ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """ 768ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng # TODO(fdeng): We should use host_id and attribute together as 769ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng # a primary key in the db. 770ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng return cls.objects.get(host_id=data['host_id'], 771ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng attribute=data['attribute']) 772ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 773ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 774ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @classmethod 775ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng def deserialize(cls, data): 776ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """Override deserialize in parent class. 777ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 778ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Do not deserialize id as id is not kept consistent on master and shards. 779ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 780ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @param data: A dictionary of data to deserialize. 781ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 782ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @returns: A HostAttribute object. 783ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """ 784ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng if data: 785ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng data.pop('id') 786ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng return super(HostAttribute, cls).deserialize(data) 787ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 788ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 7897c7852819d0611b4d0e8e69b3011b79e4016a770showardclass Test(dbmodels.Model, model_logic.ModelExtensions): 7900afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """\ 7910afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Required: 792909c7a661e6739c110690e70e2f4421ffc4e5433showard author: author name 793909c7a661e6739c110690e70e2f4421ffc4e5433showard description: description of the test 7940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name: test name 795909c7a661e6739c110690e70e2f4421ffc4e5433showard time: short, medium, long 796909c7a661e6739c110690e70e2f4421ffc4e5433showard test_class: This describes the class for your the test belongs in. 797909c7a661e6739c110690e70e2f4421ffc4e5433showard test_category: This describes the category for your tests 7980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski test_type: Client or Server 7990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski path: path to pass to run_test() 800909c7a661e6739c110690e70e2f4421ffc4e5433showard sync_count: is a number >=1 (1 being the default). If it's 1, then it's an 801909c7a661e6739c110690e70e2f4421ffc4e5433showard async job. If it's >1 it's sync job for that number of machines 8022bab8f45adedeacbf2d62d37b90255581adc3c7dshoward i.e. if sync_count = 2 it is a sync job that requires two 8032bab8f45adedeacbf2d62d37b90255581adc3c7dshoward machines. 8040afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Optional: 805909c7a661e6739c110690e70e2f4421ffc4e5433showard dependencies: What the test requires to run. Comma deliminated list 806989f25dcbb6361218f0f84d1c8404761b4c39d96showard dependency_labels: many-to-many relationship with labels corresponding to 807989f25dcbb6361218f0f84d1c8404761b4c39d96showard test dependencies. 808909c7a661e6739c110690e70e2f4421ffc4e5433showard experimental: If this is set to True production servers will ignore the test 809909c7a661e6739c110690e70e2f4421ffc4e5433showard run_verify: Whether or not the scheduler should run the verify stage 81007e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi run_reset: Whether or not the scheduler should run the reset stage 8119af96d3b0c1d3fc7e137a3535835391b9019b810Aviv Keshet test_retry: Number of times to retry test if the test did not complete 8129af96d3b0c1d3fc7e137a3535835391b9019b810Aviv Keshet successfully. (optional, default: 0) 8130afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 814909c7a661e6739c110690e70e2f4421ffc4e5433showard TestTime = enum.Enum('SHORT', 'MEDIUM', 'LONG', start_value=1) 8150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 816a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward name = dbmodels.CharField(max_length=255, unique=True) 817a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward author = dbmodels.CharField(max_length=255) 818a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward test_class = dbmodels.CharField(max_length=255) 819a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward test_category = dbmodels.CharField(max_length=255) 820a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward dependencies = dbmodels.CharField(max_length=255, blank=True) 8210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski description = dbmodels.TextField(blank=True) 822909c7a661e6739c110690e70e2f4421ffc4e5433showard experimental = dbmodels.BooleanField(default=True) 82307e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi run_verify = dbmodels.BooleanField(default=False) 824909c7a661e6739c110690e70e2f4421ffc4e5433showard test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(), 825909c7a661e6739c110690e70e2f4421ffc4e5433showard default=TestTime.MEDIUM) 8263dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet test_type = dbmodels.SmallIntegerField( 8273dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet choices=control_data.CONTROL_TYPE.choices()) 828909c7a661e6739c110690e70e2f4421ffc4e5433showard sync_count = dbmodels.IntegerField(default=1) 829a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward path = dbmodels.CharField(max_length=255, unique=True) 8309af96d3b0c1d3fc7e137a3535835391b9019b810Aviv Keshet test_retry = dbmodels.IntegerField(blank=True, default=0) 83107e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi run_reset = dbmodels.BooleanField(default=True) 8320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 833eab66ce582bfe05076ff096c3a044d8f0497bbcashoward dependency_labels = ( 834eab66ce582bfe05076ff096c3a044d8f0497bbcashoward dbmodels.ManyToManyField(Label, blank=True, 835eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table='afe_autotests_dependency_labels')) 8360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name_field = 'name' 8370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski objects = model_logic.ExtendedManager() 8380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 8390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 84035a70227e69e03d0492fe47608a6370dc339f287jamesren def admin_description(self): 8417db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns a string representing the admin description.""" 84235a70227e69e03d0492fe47608a6370dc339f287jamesren escaped_description = saxutils.escape(self.description) 84335a70227e69e03d0492fe47608a6370dc339f287jamesren return '<span style="white-space:pre">%s</span>' % escaped_description 84435a70227e69e03d0492fe47608a6370dc339f287jamesren admin_description.allow_tags = True 845cae88c6d6f2bd3c8b3ee46628e7230b3fcc444bcjamesren admin_description.short_description = 'Description' 84635a70227e69e03d0492fe47608a6370dc339f287jamesren 84735a70227e69e03d0492fe47608a6370dc339f287jamesren 8480afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 8497db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class Test.""" 850eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_autotests' 8510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 852a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 853a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.name) 854e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 855e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 8564a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesrenclass TestParameter(dbmodels.Model): 8574a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren """ 8584a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren A declared parameter of a test 8594a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren """ 8604a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren test = dbmodels.ForeignKey(Test) 8614a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren name = dbmodels.CharField(max_length=255) 8624a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren 8634a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren class Meta: 8647db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class TestParameter.""" 8654a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren db_table = 'afe_test_parameters' 8664a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren unique_together = ('test', 'name') 8674a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren 8684a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren def __unicode__(self): 8690a99391317b5caca3d373d0bba931bcb62302566Eric Li return u'%s (%s)' % (self.name, self.test.name) 8704a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren 8714a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren 8722b9a88bcd336233fa483490892338e29f0a5fa67showardclass Profiler(dbmodels.Model, model_logic.ModelExtensions): 8732b9a88bcd336233fa483490892338e29f0a5fa67showard """\ 8742b9a88bcd336233fa483490892338e29f0a5fa67showard Required: 8752b9a88bcd336233fa483490892338e29f0a5fa67showard name: profiler name 8762b9a88bcd336233fa483490892338e29f0a5fa67showard test_type: Client or Server 8772b9a88bcd336233fa483490892338e29f0a5fa67showard 8782b9a88bcd336233fa483490892338e29f0a5fa67showard Optional: 8792b9a88bcd336233fa483490892338e29f0a5fa67showard description: arbirary text description 8802b9a88bcd336233fa483490892338e29f0a5fa67showard """ 881a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward name = dbmodels.CharField(max_length=255, unique=True) 8822b9a88bcd336233fa483490892338e29f0a5fa67showard description = dbmodels.TextField(blank=True) 8832b9a88bcd336233fa483490892338e29f0a5fa67showard 8842b9a88bcd336233fa483490892338e29f0a5fa67showard name_field = 'name' 8852b9a88bcd336233fa483490892338e29f0a5fa67showard objects = model_logic.ExtendedManager() 8862b9a88bcd336233fa483490892338e29f0a5fa67showard 8872b9a88bcd336233fa483490892338e29f0a5fa67showard 8882b9a88bcd336233fa483490892338e29f0a5fa67showard class Meta: 8897db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class Profiler.""" 890eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_profilers' 8912b9a88bcd336233fa483490892338e29f0a5fa67showard 892a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 893a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.name) 8942b9a88bcd336233fa483490892338e29f0a5fa67showard 8952b9a88bcd336233fa483490892338e29f0a5fa67showard 8967c7852819d0611b4d0e8e69b3011b79e4016a770showardclass AclGroup(dbmodels.Model, model_logic.ModelExtensions): 8970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """\ 8980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Required: 8990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name: name of ACL group 900e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 9010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Optional: 9020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski description: arbitrary description of group 9030afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 9043bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 9053bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich SERIALIZATION_LINKS_TO_FOLLOW = set(['users']) 9063bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 907a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward name = dbmodels.CharField(max_length=255, unique=True) 908a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward description = dbmodels.CharField(max_length=255, blank=True) 909eab66ce582bfe05076ff096c3a044d8f0497bbcashoward users = dbmodels.ManyToManyField(User, blank=False, 910eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table='afe_acl_groups_users') 911eab66ce582bfe05076ff096c3a044d8f0497bbcashoward hosts = dbmodels.ManyToManyField(Host, blank=True, 912eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table='afe_acl_groups_hosts') 913e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 9140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name_field = 'name' 9150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski objects = model_logic.ExtendedManager() 916eb3be4d9b4c1bb292723ea1d52612aec5967ca0eshoward 91708f981bddc5f1a199d789d177cbf583dee9f6c17showard @staticmethod 9183dd47c247147ca6920b222d43a8eb6ee54209c8fshoward def check_for_acl_violation_hosts(hosts): 9197db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Verify the current user has access to the specified hosts. 9207db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 9217db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param hosts: The hosts to verify against. 9227db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @raises AclAccessViolation if the current user doesn't have access 9237db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey to a host. 9247db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 92564a9595406f2884fb3ece241190b10aa054439a9showard user = User.current_user() 9263dd47c247147ca6920b222d43a8eb6ee54209c8fshoward if user.is_superuser(): 9279dbdcda5104991cbf344ea5cba1aa58e1af444f3showard return 9283dd47c247147ca6920b222d43a8eb6ee54209c8fshoward accessible_host_ids = set( 929d9ac445a60d6d11537f566503164344e09527917showard host.id for host in Host.objects.filter(aclgroup__users=user)) 9303dd47c247147ca6920b222d43a8eb6ee54209c8fshoward for host in hosts: 9313dd47c247147ca6920b222d43a8eb6ee54209c8fshoward # Check if the user has access to this host, 9327db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey # but only if it is not a metahost or a one-time-host. 93398ead171ab61b324414dd0316478c5ce72bbd4b1showard no_access = (isinstance(host, Host) 93498ead171ab61b324414dd0316478c5ce72bbd4b1showard and not host.invalid 93598ead171ab61b324414dd0316478c5ce72bbd4b1showard and int(host.id) not in accessible_host_ids) 93698ead171ab61b324414dd0316478c5ce72bbd4b1showard if no_access: 937eaa408e59e932ce0540aa5d50bf966af5080f640showard raise AclAccessViolation("%s does not have access to %s" % 938eaa408e59e932ce0540aa5d50bf966af5080f640showard (str(user), str(host))) 9393dd47c247147ca6920b222d43a8eb6ee54209c8fshoward 9409dbdcda5104991cbf344ea5cba1aa58e1af444f3showard 9419dbdcda5104991cbf344ea5cba1aa58e1af444f3showard @staticmethod 942dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward def check_abort_permissions(queue_entries): 9437db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Look for queue entries that aren't abortable by the current user. 9447db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 9457db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey An entry is not abortable if: 9467db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey * the job isn't owned by this user, and 947dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward * the machine isn't ACL-accessible, or 948dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward * the machine is in the "Everyone" ACL 9497db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 9507db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param queue_entries: The queue entries to check. 9517db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @raises AclAccessViolation if a queue entry is not abortable by the 9527db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey current user. 953dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward """ 95464a9595406f2884fb3ece241190b10aa054439a9showard user = User.current_user() 9559dbdcda5104991cbf344ea5cba1aa58e1af444f3showard if user.is_superuser(): 9569dbdcda5104991cbf344ea5cba1aa58e1af444f3showard return 957dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward not_owned = queue_entries.exclude(job__owner=user.login) 958dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward # I do this using ID sets instead of just Django filters because 959a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward # filtering on M2M dbmodels is broken in Django 0.96. It's better in 960a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward # 1.0. 961a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward # TODO: Use Django filters, now that we're using 1.0. 962dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward accessible_ids = set( 963dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward entry.id for entry 964d9ac445a60d6d11537f566503164344e09527917showard in not_owned.filter(host__aclgroup__users__login=user.login)) 965dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward public_ids = set(entry.id for entry 966d9ac445a60d6d11537f566503164344e09527917showard in not_owned.filter(host__aclgroup__name='Everyone')) 967dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward cannot_abort = [entry for entry in not_owned.select_related() 968dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward if entry.id not in accessible_ids 969dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward or entry.id in public_ids] 970dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward if len(cannot_abort) == 0: 971dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward return 972dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward entry_names = ', '.join('%s-%s/%s' % (entry.job.id, entry.job.owner, 9733f15eedfedab4b1b9eaff67713e411eada0c84dfshoward entry.host_or_metahost_name()) 974dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward for entry in cannot_abort) 975dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward raise AclAccessViolation('You cannot abort the following job entries: ' 976dc8175153fc559d1e7ae605b15ceef66e25e7cdeshoward + entry_names) 9779dbdcda5104991cbf344ea5cba1aa58e1af444f3showard 9789dbdcda5104991cbf344ea5cba1aa58e1af444f3showard 9793dd47c247147ca6920b222d43a8eb6ee54209c8fshoward def check_for_acl_violation_acl_group(self): 9807db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Verifies the current user has acces to this ACL group. 9817db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 9827db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @raises AclAccessViolation if the current user doesn't have access to 9837db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey this ACL group. 9847db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 98564a9595406f2884fb3ece241190b10aa054439a9showard user = User.current_user() 9863dd47c247147ca6920b222d43a8eb6ee54209c8fshoward if user.is_superuser(): 9878cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward return 9888cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward if self.name == 'Everyone': 9898cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward raise AclAccessViolation("You cannot modify 'Everyone'!") 9903dd47c247147ca6920b222d43a8eb6ee54209c8fshoward if not user in self.users.all(): 9913dd47c247147ca6920b222d43a8eb6ee54209c8fshoward raise AclAccessViolation("You do not have access to %s" 9923dd47c247147ca6920b222d43a8eb6ee54209c8fshoward % self.name) 9933dd47c247147ca6920b222d43a8eb6ee54209c8fshoward 9943dd47c247147ca6920b222d43a8eb6ee54209c8fshoward @staticmethod 99508f981bddc5f1a199d789d177cbf583dee9f6c17showard def on_host_membership_change(): 9967db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Invoked when host membership changes.""" 99708f981bddc5f1a199d789d177cbf583dee9f6c17showard everyone = AclGroup.objects.get(name='Everyone') 99808f981bddc5f1a199d789d177cbf583dee9f6c17showard 9993dd47c247147ca6920b222d43a8eb6ee54209c8fshoward # find hosts that aren't in any ACL group and add them to Everyone 100008f981bddc5f1a199d789d177cbf583dee9f6c17showard # TODO(showard): this is a bit of a hack, since the fact that this query 100108f981bddc5f1a199d789d177cbf583dee9f6c17showard # works is kind of a coincidence of Django internals. This trick 100208f981bddc5f1a199d789d177cbf583dee9f6c17showard # doesn't work in general (on all foreign key relationships). I'll 100308f981bddc5f1a199d789d177cbf583dee9f6c17showard # replace it with a better technique when the need arises. 1004d9ac445a60d6d11537f566503164344e09527917showard orphaned_hosts = Host.valid_objects.filter(aclgroup__id__isnull=True) 100508f981bddc5f1a199d789d177cbf583dee9f6c17showard everyone.hosts.add(*orphaned_hosts.distinct()) 100608f981bddc5f1a199d789d177cbf583dee9f6c17showard 100708f981bddc5f1a199d789d177cbf583dee9f6c17showard # find hosts in both Everyone and another ACL group, and remove them 100808f981bddc5f1a199d789d177cbf583dee9f6c17showard # from Everyone 1009a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward hosts_in_everyone = Host.valid_objects.filter(aclgroup__name='Everyone') 1010a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward acled_hosts = set() 1011a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward for host in hosts_in_everyone: 1012a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward # Has an ACL group other than Everyone 1013a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward if host.aclgroup_set.count() > 1: 1014a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward acled_hosts.add(host) 1015a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward everyone.hosts.remove(*acled_hosts) 101608f981bddc5f1a199d789d177cbf583dee9f6c17showard 101708f981bddc5f1a199d789d177cbf583dee9f6c17showard 101808f981bddc5f1a199d789d177cbf583dee9f6c17showard def delete(self): 10193dd47c247147ca6920b222d43a8eb6ee54209c8fshoward if (self.name == 'Everyone'): 10203dd47c247147ca6920b222d43a8eb6ee54209c8fshoward raise AclAccessViolation("You cannot delete 'Everyone'!") 10213dd47c247147ca6920b222d43a8eb6ee54209c8fshoward self.check_for_acl_violation_acl_group() 102208f981bddc5f1a199d789d177cbf583dee9f6c17showard super(AclGroup, self).delete() 102308f981bddc5f1a199d789d177cbf583dee9f6c17showard self.on_host_membership_change() 102408f981bddc5f1a199d789d177cbf583dee9f6c17showard 102508f981bddc5f1a199d789d177cbf583dee9f6c17showard 102604f2cd81b1776c12085d9314ec27c3a7f755a843showard def add_current_user_if_empty(self): 10277db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Adds the current user if the set of users is empty.""" 102804f2cd81b1776c12085d9314ec27c3a7f755a843showard if not self.users.count(): 102964a9595406f2884fb3ece241190b10aa054439a9showard self.users.add(User.current_user()) 103004f2cd81b1776c12085d9314ec27c3a7f755a843showard 103104f2cd81b1776c12085d9314ec27c3a7f755a843showard 10328cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward def perform_after_save(self, change): 10337db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Called after a save. 10347db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 10357db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param change: Whether there was a change. 10367db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 10378cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward if not change: 103864a9595406f2884fb3ece241190b10aa054439a9showard self.users.add(User.current_user()) 10398cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward self.add_current_user_if_empty() 10408cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward self.on_host_membership_change() 10418cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward 10428cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward 10438cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward def save(self, *args, **kwargs): 1044116ff0fc2674082fe475afd64ce4dec998ed71b7Jakob Juelich change = bool(self.id) 1045116ff0fc2674082fe475afd64ce4dec998ed71b7Jakob Juelich if change: 10468cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward # Check the original object for an ACL violation 1047116ff0fc2674082fe475afd64ce4dec998ed71b7Jakob Juelich AclGroup.objects.get(id=self.id).check_for_acl_violation_acl_group() 10488cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward super(AclGroup, self).save(*args, **kwargs) 1049116ff0fc2674082fe475afd64ce4dec998ed71b7Jakob Juelich self.perform_after_save(change) 10508cbaf1e2984c7c5f7df877ac6a759ccef13aa63ashoward 1051eb3be4d9b4c1bb292723ea1d52612aec5967ca0eshoward 10520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 10537db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class AclGroup.""" 1054eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_acl_groups' 1055e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1056a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 1057a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return unicode(self.name) 1058e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1059e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 10604a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesrenclass ParameterizedJob(dbmodels.Model): 10614a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren """ 10627db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey Auxiliary configuration for a parameterized job. 1063f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1064fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette This class is obsolete, and ought to be dead. Due to a series of 1065fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette unfortunate events, it can't be deleted: 1066fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette * In `class Job` we're required to keep a reference to this class 1067fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette for the sake of the scheduler unit tests. 1068fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette * The existence of the reference in `Job` means that certain 1069fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette methods here will get called from the `get_jobs` RPC. 1070fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette So, the definitions below seem to be the minimum stub we can support 1071fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette unless/until we change the database schema. 1072fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette """ 1073f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1074f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette @classmethod 1075f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette def smart_get(cls, id_or_name, *args, **kwargs): 1076f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette """For compatibility with Job.add_object. 1077f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1078f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette @param cls: Implicit class object. 1079f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette @param id_or_name: The ID or name to get. 1080f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette @param args: Non-keyword arguments. 1081f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette @param kwargs: Keyword arguments. 1082f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette """ 1083f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette return cls.objects.get(pk=id_or_name) 1084f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1085f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1086f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette def job(self): 1087f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette """Returns the job if it exists, or else None.""" 1088f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette jobs = self.job_set.all() 1089f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette assert jobs.count() <= 1 1090f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette return jobs and jobs[0] or None 1091f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1092f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1093f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette class Meta: 1094f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette """Metadata for class ParameterizedJob.""" 1095f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette db_table = 'afe_parameterized_jobs' 1096f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 1097f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette def __unicode__(self): 1098f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette return u'%s (parameterized) - %s' % (self.test.name, self.job()) 10994a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren 1100f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 11017c7852819d0611b4d0e8e69b3011b79e4016a770showardclass JobManager(model_logic.ExtendedManager): 11020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 'Custom manager to provide efficient status counts querying.' 11030afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def get_status_counts(self, job_ids): 11047db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns a dict mapping the given job IDs to their status count dicts. 11057db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 11067db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param job_ids: A list of job IDs. 11070afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 11080afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not job_ids: 11090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return {} 11100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids) 11110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cursor = connection.cursor() 11120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cursor.execute(""" 1113d3dc199703bfb8784a2f8f072d0514532c86c0a9showard SELECT job_id, status, aborted, complete, COUNT(*) 1114eab66ce582bfe05076ff096c3a044d8f0497bbcashoward FROM afe_host_queue_entries 11150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski WHERE job_id IN %s 1116d3dc199703bfb8784a2f8f072d0514532c86c0a9showard GROUP BY job_id, status, aborted, complete 11170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ % id_list) 111825aaf3f6c159bc00233d9180e4d724e53cbfc53cshoward all_job_counts = dict((job_id, {}) for job_id in job_ids) 1119d3dc199703bfb8784a2f8f072d0514532c86c0a9showard for job_id, status, aborted, complete, count in cursor.fetchall(): 112025aaf3f6c159bc00233d9180e4d724e53cbfc53cshoward job_dict = all_job_counts[job_id] 1121d3dc199703bfb8784a2f8f072d0514532c86c0a9showard full_status = HostQueueEntry.compute_full_status(status, aborted, 1122d3dc199703bfb8784a2f8f072d0514532c86c0a9showard complete) 1123b6d1662e18d756483d5fd81f4057cae4ef62152cshoward job_dict.setdefault(full_status, 0) 112425aaf3f6c159bc00233d9180e4d724e53cbfc53cshoward job_dict[full_status] += count 11250afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return all_job_counts 1126e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1127e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 11287c7852819d0611b4d0e8e69b3011b79e4016a770showardclass Job(dbmodels.Model, model_logic.ModelExtensions): 11290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """\ 11300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski owner: username of job owner 11310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski name: job name (does not have to be unique) 11327d658cf6bade565c1098fd7b47075e96e7b542caAlex Miller priority: Integer priority value. Higher is more important. 11330afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski control_file: contents of control file 11340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski control_type: Client or Server 11350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski created_on: date of job creation 11360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski submitted_on: date of job submission 11372bab8f45adedeacbf2d62d37b90255581adc3c7dshoward synch_count: how many hosts should be used per autoserv execution 1138909c7a661e6739c110690e70e2f4421ffc4e5433showard run_verify: Whether or not to run the verify phase 113907e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi run_reset: Whether or not to run the reset phase 114094dc003dcce29c296a071add2f397013045c240cSimran Basi timeout: DEPRECATED - hours from queuing time until job times out 114194dc003dcce29c296a071add2f397013045c240cSimran Basi timeout_mins: minutes from job queuing time until the job times out 114234217022229b755bc1ee52f83665acba76bd5044Simran Basi max_runtime_hrs: DEPRECATED - hours from job starting time until job 114334217022229b755bc1ee52f83665acba76bd5044Simran Basi times out 114434217022229b755bc1ee52f83665acba76bd5044Simran Basi max_runtime_mins: minutes from job starting time until job times out 1145542e840486b02b5025d26da16f98fed97898a601showard email_list: list of people to email on completion delimited by any of: 1146542e840486b02b5025d26da16f98fed97898a601showard white space, ',', ':', ';' 1147989f25dcbb6361218f0f84d1c8404761b4c39d96showard dependency_labels: many-to-many relationship with labels corresponding to 1148989f25dcbb6361218f0f84d1c8404761b4c39d96showard job dependencies 114921baa459ea14f96e06212f1f35fcddab9442b3fcshoward reboot_before: Never, If dirty, or Always 115021baa459ea14f96e06212f1f35fcddab9442b3fcshoward reboot_after: Never, If all tests passed, or Always 1151a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard parse_failed_repair: if True, a failed repair launched by this job will have 1152a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard its results parsed as part of the job. 115376fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren drone_set: The set of drones to run this job on 11540b9cfc93f17ec81535de54e3bae4f5f0e8d8595bAviv Keshet parent_job: Parent job (optional) 1155cd1ff9b72bd73c7308145750f07908b26c5d08daAviv Keshet test_retry: Number of times to retry test if the test did not complete 1156cd1ff9b72bd73c7308145750f07908b26c5d08daAviv Keshet successfully. (optional, default: 0) 1157c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi require_ssp: Require server-side packaging unless require_ssp is set to 1158c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi False. (optional, default: None) 11590afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 11603bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 11613bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich # TODO: Investigate, if jobkeyval_set is really needed. 11623bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich # dynamic_suite will write them into an attached file for the drone, but 11633bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich # it doesn't seem like they are actually used. If they aren't used, remove 11643bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich # jobkeyval_set here. 11653bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich SERIALIZATION_LINKS_TO_FOLLOW = set(['dependency_labels', 11663bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 'hostqueueentry_set', 11673bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 'jobkeyval_set', 11683bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 'shard']) 11693bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 11702b79fd7627875033b2650c302e385712976aea7fFang Deng # SQL for selecting jobs that should be sent to shard. 11712b79fd7627875033b2650c302e385712976aea7fFang Deng # We use raw sql as django filters were not optimized. 11722b79fd7627875033b2650c302e385712976aea7fFang Deng # The following jobs are excluded by the SQL. 11732b79fd7627875033b2650c302e385712976aea7fFang Deng # - Non-aborted jobs known to shard as specified in |known_ids|. 11742b79fd7627875033b2650c302e385712976aea7fFang Deng # Note for jobs aborted on master, even if already known to shard, 11752b79fd7627875033b2650c302e385712976aea7fFang Deng # will be sent to shard again so that shard can abort them. 11762b79fd7627875033b2650c302e385712976aea7fFang Deng # - Completed jobs 11772b79fd7627875033b2650c302e385712976aea7fFang Deng # - Active jobs 11782b79fd7627875033b2650c302e385712976aea7fFang Deng # - Jobs without host_queue_entries 11792b79fd7627875033b2650c302e385712976aea7fFang Deng NON_ABORTED_KNOWN_JOBS = '(t2.aborted = 0 AND t1.id IN (%(known_ids)s))' 11802b79fd7627875033b2650c302e385712976aea7fFang Deng 11812b79fd7627875033b2650c302e385712976aea7fFang Deng SQL_SHARD_JOBS = ( 11825c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu 'SELECT DISTINCT(t1.id) FROM afe_jobs t1 ' 11832b79fd7627875033b2650c302e385712976aea7fFang Deng 'INNER JOIN afe_host_queue_entries t2 ON ' 11845c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu ' (t1.id = t2.job_id AND t2.complete != 1 AND t2.active != 1 ' 11852b79fd7627875033b2650c302e385712976aea7fFang Deng ' %(check_known_jobs)s) ' 11865c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu 'LEFT OUTER JOIN afe_jobs_dependency_labels t3 ON (t1.id = t3.job_id) ' 11876b89f1c14c65c2a3a130cc1a063143b38ebc3b94Dan Shi 'JOIN afe_shards_labels t4 ' 11886b89f1c14c65c2a3a130cc1a063143b38ebc3b94Dan Shi ' ON (t4.label_id = t3.label_id OR t4.label_id = t2.meta_host) ' 11896b89f1c14c65c2a3a130cc1a063143b38ebc3b94Dan Shi 'WHERE t4.shard_id = %(shard_id)s' 11902b79fd7627875033b2650c302e385712976aea7fFang Deng ) 11912b79fd7627875033b2650c302e385712976aea7fFang Deng 11925c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu # Jobs can be created with assigned hosts and have no dependency 11935c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu # labels nor meta_host. 11942b79fd7627875033b2650c302e385712976aea7fFang Deng # We are looking for: 11955c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu # - a job whose hqe's meta_host is null 11962b79fd7627875033b2650c302e385712976aea7fFang Deng # - a job whose hqe has a host 11972b79fd7627875033b2650c302e385712976aea7fFang Deng # - one of the host's labels matches the shard's label. 11982b79fd7627875033b2650c302e385712976aea7fFang Deng # Non-aborted known jobs, completed jobs, active jobs, jobs 11995c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu # without hqe are exluded as we do with SQL_SHARD_JOBS. 12005c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu SQL_SHARD_JOBS_WITH_HOSTS = ( 12015c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu 'SELECT DISTINCT(t1.id) FROM afe_jobs t1 ' 12022b79fd7627875033b2650c302e385712976aea7fFang Deng 'INNER JOIN afe_host_queue_entries t2 ON ' 12032b79fd7627875033b2650c302e385712976aea7fFang Deng ' (t1.id = t2.job_id AND t2.complete != 1 AND t2.active != 1 ' 12042b79fd7627875033b2650c302e385712976aea7fFang Deng ' AND t2.meta_host IS NULL AND t2.host_id IS NOT NULL ' 12052b79fd7627875033b2650c302e385712976aea7fFang Deng ' %(check_known_jobs)s) ' 12062b79fd7627875033b2650c302e385712976aea7fFang Deng 'LEFT OUTER JOIN afe_hosts_labels t3 ON (t2.host_id = t3.host_id) ' 12072b79fd7627875033b2650c302e385712976aea7fFang Deng 'WHERE (t3.label_id IN ' 12082b79fd7627875033b2650c302e385712976aea7fFang Deng ' (SELECT label_id FROM afe_shards_labels ' 12092b79fd7627875033b2650c302e385712976aea7fFang Deng ' WHERE shard_id = %(shard_id)s))' 12102b79fd7627875033b2650c302e385712976aea7fFang Deng ) 12115c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu 12122b79fd7627875033b2650c302e385712976aea7fFang Deng # Even if we had filters about complete, active and aborted 12135c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu # bits in the above two SQLs, there is a chance that 12142b79fd7627875033b2650c302e385712976aea7fFang Deng # the result may still contain a job with an hqe with 'complete=1' 12152b79fd7627875033b2650c302e385712976aea7fFang Deng # or 'active=1' or 'aborted=0 and afe_job.id in known jobs.' 12162b79fd7627875033b2650c302e385712976aea7fFang Deng # This happens when a job has two (or more) hqes and at least 12172b79fd7627875033b2650c302e385712976aea7fFang Deng # one hqe has different bits than others. 12182b79fd7627875033b2650c302e385712976aea7fFang Deng # We use a second sql to ensure we exclude all un-desired jobs. 12192b79fd7627875033b2650c302e385712976aea7fFang Deng SQL_JOBS_TO_EXCLUDE =( 12202b79fd7627875033b2650c302e385712976aea7fFang Deng 'SELECT t1.id FROM afe_jobs t1 ' 12212b79fd7627875033b2650c302e385712976aea7fFang Deng 'INNER JOIN afe_host_queue_entries t2 ON ' 12222b79fd7627875033b2650c302e385712976aea7fFang Deng ' (t1.id = t2.job_id) ' 12232b79fd7627875033b2650c302e385712976aea7fFang Deng 'WHERE (t1.id in (%(candidates)s) ' 12242b79fd7627875033b2650c302e385712976aea7fFang Deng ' AND (t2.complete=1 OR t2.active=1 ' 12252b79fd7627875033b2650c302e385712976aea7fFang Deng ' %(check_known_jobs)s))' 12262b79fd7627875033b2650c302e385712976aea7fFang Deng ) 12273bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 1228f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich def _deserialize_relation(self, link, data): 1229f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich if link in ['hostqueueentry_set', 'jobkeyval_set']: 1230f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich for obj in data: 1231f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich obj['job_id'] = self.id 1232f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1233f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich super(Job, self)._deserialize_relation(link, data) 1234f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1235f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1236f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich def custom_deserialize_relation(self, link, data): 1237116ff0fc2674082fe475afd64ce4dec998ed71b7Jakob Juelich assert link == 'shard', 'Link %s should not be deserialized' % link 1238f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich self.shard = Shard.deserialize(data) 1239f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1240f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1241a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich def sanity_check_update_from_shard(self, shard, updated_serialized): 124202e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # If the job got aborted on the master after the client fetched it 124302e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # no shard_id will be set. The shard might still push updates though, 124402e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # as the job might complete before the abort bit syncs to the shard. 124502e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # Alternative considered: The master scheduler could be changed to not 124602e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # set aborted jobs to completed that are sharded out. But that would 124702e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # require database queries and seemed more complicated to implement. 124802e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # This seems safe to do, as there won't be updates pushed from the wrong 124902e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # shards should be powered off and wiped hen they are removed from the 125002e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich # master. 125102e6129187328bd3edd96dae8062e6a19ab7936cJakob Juelich if self.shard_id and self.shard_id != shard.id: 1252a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich raise error.UnallowedRecordsSentToMaster( 1253a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 'Job id=%s is assigned to shard (%s). Cannot update it with %s ' 1254a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 'from shard %s.' % (self.id, self.shard_id, updated_serialized, 1255a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich shard.id)) 1256a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 1257a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 125894dc003dcce29c296a071add2f397013045c240cSimran Basi # TIMEOUT is deprecated. 1259b1e5187f9aa303c4fc914f07312286d302b46a0eshoward DEFAULT_TIMEOUT = global_config.global_config.get_config_value( 126094dc003dcce29c296a071add2f397013045c240cSimran Basi 'AUTOTEST_WEB', 'job_timeout_default', default=24) 126194dc003dcce29c296a071add2f397013045c240cSimran Basi DEFAULT_TIMEOUT_MINS = global_config.global_config.get_config_value( 126294dc003dcce29c296a071add2f397013045c240cSimran Basi 'AUTOTEST_WEB', 'job_timeout_mins_default', default=24*60) 126334217022229b755bc1ee52f83665acba76bd5044Simran Basi # MAX_RUNTIME_HRS is deprecated. Will be removed after switch to mins is 126434217022229b755bc1ee52f83665acba76bd5044Simran Basi # completed. 126512f3e3212795a539d95973f893ac570e669e3a22showard DEFAULT_MAX_RUNTIME_HRS = global_config.global_config.get_config_value( 126612f3e3212795a539d95973f893ac570e669e3a22showard 'AUTOTEST_WEB', 'job_max_runtime_hrs_default', default=72) 126734217022229b755bc1ee52f83665acba76bd5044Simran Basi DEFAULT_MAX_RUNTIME_MINS = global_config.global_config.get_config_value( 126834217022229b755bc1ee52f83665acba76bd5044Simran Basi 'AUTOTEST_WEB', 'job_max_runtime_mins_default', default=72*60) 1269a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard DEFAULT_PARSE_FAILED_REPAIR = global_config.global_config.get_config_value( 1270a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard 'AUTOTEST_WEB', 'parse_failed_repair_default', type=bool, 1271a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard default=False) 1272b1e5187f9aa303c4fc914f07312286d302b46a0eshoward 1273a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward owner = dbmodels.CharField(max_length=255) 1274a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward name = dbmodels.CharField(max_length=255) 12757d658cf6bade565c1098fd7b47075e96e7b542caAlex Miller priority = dbmodels.SmallIntegerField(default=priorities.Priority.DEFAULT) 12764a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren control_file = dbmodels.TextField(null=True, blank=True) 12773dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet control_type = dbmodels.SmallIntegerField( 12783dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet choices=control_data.CONTROL_TYPE.choices(), 12793dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet blank=True, # to allow 0 12803dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet default=control_data.CONTROL_TYPE.CLIENT) 128168c7aa02743042196b1f4ce3d41cbe03d1152ec0showard created_on = dbmodels.DateTimeField() 12828d89b64d7edb2ea7d7f2d9c59ded2e2f09ed3ea8Simran Basi synch_count = dbmodels.IntegerField(blank=True, default=0) 1283b1e5187f9aa303c4fc914f07312286d302b46a0eshoward timeout = dbmodels.IntegerField(default=DEFAULT_TIMEOUT) 128407e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi run_verify = dbmodels.BooleanField(default=False) 1285a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward email_list = dbmodels.CharField(max_length=250, blank=True) 1286eab66ce582bfe05076ff096c3a044d8f0497bbcashoward dependency_labels = ( 1287eab66ce582bfe05076ff096c3a044d8f0497bbcashoward dbmodels.ManyToManyField(Label, blank=True, 1288eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table='afe_jobs_dependency_labels')) 1289dd855244f44b65d0508345c6fef74846652c8c26jamesren reboot_before = dbmodels.SmallIntegerField( 1290dd855244f44b65d0508345c6fef74846652c8c26jamesren choices=model_attributes.RebootBefore.choices(), blank=True, 1291dd855244f44b65d0508345c6fef74846652c8c26jamesren default=DEFAULT_REBOOT_BEFORE) 1292dd855244f44b65d0508345c6fef74846652c8c26jamesren reboot_after = dbmodels.SmallIntegerField( 1293dd855244f44b65d0508345c6fef74846652c8c26jamesren choices=model_attributes.RebootAfter.choices(), blank=True, 1294dd855244f44b65d0508345c6fef74846652c8c26jamesren default=DEFAULT_REBOOT_AFTER) 1295a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard parse_failed_repair = dbmodels.BooleanField( 1296a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard default=DEFAULT_PARSE_FAILED_REPAIR) 129734217022229b755bc1ee52f83665acba76bd5044Simran Basi # max_runtime_hrs is deprecated. Will be removed after switch to mins is 129834217022229b755bc1ee52f83665acba76bd5044Simran Basi # completed. 129912f3e3212795a539d95973f893ac570e669e3a22showard max_runtime_hrs = dbmodels.IntegerField(default=DEFAULT_MAX_RUNTIME_HRS) 130034217022229b755bc1ee52f83665acba76bd5044Simran Basi max_runtime_mins = dbmodels.IntegerField(default=DEFAULT_MAX_RUNTIME_MINS) 130176fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren drone_set = dbmodels.ForeignKey(DroneSet, null=True, blank=True) 13020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1303fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # TODO(jrbarnette) We have to keep `parameterized_job` around or it 1304fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # breaks the scheduler_models unit tests (and fixing the unit tests 1305fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # will break the scheduler, so don't do that). 1306fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # 1307fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # The ultimate fix is to delete the column from the database table 1308fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # at which point, you _must_ delete this. Until you're ready to do 1309fddab4a620978bdb74c72299325fbe441e36977aRichard Barnette # that, DON'T MUCK WITH IT. 13104a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren parameterized_job = dbmodels.ForeignKey(ParameterizedJob, null=True, 13114a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren blank=True) 13124a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren 13130b9cfc93f17ec81535de54e3bae4f5f0e8d8595bAviv Keshet parent_job = dbmodels.ForeignKey('self', blank=True, null=True) 13140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1315cd1ff9b72bd73c7308145750f07908b26c5d08daAviv Keshet test_retry = dbmodels.IntegerField(blank=True, default=0) 1316cd1ff9b72bd73c7308145750f07908b26c5d08daAviv Keshet 131707e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi run_reset = dbmodels.BooleanField(default=True) 131807e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi 131994dc003dcce29c296a071add2f397013045c240cSimran Basi timeout_mins = dbmodels.IntegerField(default=DEFAULT_TIMEOUT_MINS) 132094dc003dcce29c296a071add2f397013045c240cSimran Basi 13218421d5905ab0aed8689c2eea6be8d9c4042ce618Jakob Juelich # If this is None on the master, a slave should be found. 13228421d5905ab0aed8689c2eea6be8d9c4042ce618Jakob Juelich # If this is None on a slave, it should be synced back to the master 132392c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich shard = dbmodels.ForeignKey(Shard, blank=True, null=True) 132492c06334ea5e9baab4289ad70135e3f17ed9f872Jakob Jülich 1325c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi # If this is None, server-side packaging will be used for server side test, 1326c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi # unless it's disabled in global config AUTOSERV/enable_ssp_container. 1327c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi require_ssp = dbmodels.NullBooleanField(default=None, blank=True, null=True) 1328c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi 13290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # custom manager 13300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski objects = JobManager() 13310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 13320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1333ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller @decorators.cached_property 1334ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller def labels(self): 1335ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller """All the labels of this job""" 1336ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller # We need to convert dependency_labels to a list, because all() gives us 1337ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller # back an iterator, and storing/caching an iterator means we'd only be 1338ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller # able to read from it once. 1339ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller return list(self.dependency_labels.all()) 1340ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller 1341ec21225c11b0983b563c24bb71828f5bb6bbcaeeAlex Miller 13420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def is_server_job(self): 13437db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns whether this job is of type server.""" 13443dd8beb386f7298ffe84d7410d00cce26973e170Aviv Keshet return self.control_type == control_data.CONTROL_TYPE.SERVER 13450afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 13460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 13470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski @classmethod 1348a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard def create(cls, owner, options, hosts): 13497db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Creates a job. 13507db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 13517db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey The job is created by taking some information (the listed args) and 13527db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey filling in the rest of the necessary information. 13537db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 13547db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 13557db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param owner: The owner for the job. 13567db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param options: An options object. 13577db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param hosts: The hosts to use. 13580afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 13593dd47c247147ca6920b222d43a8eb6ee54209c8fshoward AclGroup.check_for_acl_violation_hosts(hosts) 13604d23375ff9b95e95e9f1d8443d20122f7270c1f4showard 13614a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren control_file = options.get('control_file') 1362f616e8fb22fcefa94ba18a6f2f8223fde642790fRichard Barnette 13634d23375ff9b95e95e9f1d8443d20122f7270c1f4showard user = User.current_user() 13644d23375ff9b95e95e9f1d8443d20122f7270c1f4showard if options.get('reboot_before') is None: 13654d23375ff9b95e95e9f1d8443d20122f7270c1f4showard options['reboot_before'] = user.get_reboot_before_display() 13664d23375ff9b95e95e9f1d8443d20122f7270c1f4showard if options.get('reboot_after') is None: 13674d23375ff9b95e95e9f1d8443d20122f7270c1f4showard options['reboot_after'] = user.get_reboot_after_display() 13684d23375ff9b95e95e9f1d8443d20122f7270c1f4showard 136976fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren drone_set = DroneSet.resolve_name(options.get('drone_set')) 137076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren 137194dc003dcce29c296a071add2f397013045c240cSimran Basi if options.get('timeout_mins') is None and options.get('timeout'): 137294dc003dcce29c296a071add2f397013045c240cSimran Basi options['timeout_mins'] = options['timeout'] * 60 137394dc003dcce29c296a071add2f397013045c240cSimran Basi 13740afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski job = cls.add_object( 1375a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard owner=owner, 1376a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard name=options['name'], 1377a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard priority=options['priority'], 13784a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren control_file=control_file, 1379a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard control_type=options['control_type'], 1380a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard synch_count=options.get('synch_count'), 138194dc003dcce29c296a071add2f397013045c240cSimran Basi # timeout needs to be deleted in the future. 1382a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard timeout=options.get('timeout'), 138394dc003dcce29c296a071add2f397013045c240cSimran Basi timeout_mins=options.get('timeout_mins'), 138434217022229b755bc1ee52f83665acba76bd5044Simran Basi max_runtime_mins=options.get('max_runtime_mins'), 1385a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard run_verify=options.get('run_verify'), 1386a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard email_list=options.get('email_list'), 1387a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard reboot_before=options.get('reboot_before'), 1388a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard reboot_after=options.get('reboot_after'), 1389a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard parse_failed_repair=options.get('parse_failed_repair'), 139076fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren created_on=datetime.now(), 13914a41e0145c2ede3605d19cdfacb779ca6aca1cccjamesren drone_set=drone_set, 1392cd1ff9b72bd73c7308145750f07908b26c5d08daAviv Keshet parent_job=options.get('parent_job_id'), 139307e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi test_retry=options.get('test_retry'), 1394c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi run_reset=options.get('run_reset'), 1395c9e1714424621786322d0e6ea48ac167bf35ce16Dan Shi require_ssp=options.get('require_ssp')) 13960afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1397a1e74b3e9d68792fae0c926f89b6de1736b1fe21showard job.dependency_labels = options['dependencies'] 1398c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 1399d8b6e17c8edb2845949e39c75337e4389a630c63jamesren if options.get('keyvals'): 1400c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard for key, value in options['keyvals'].iteritems(): 1401c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard JobKeyval.objects.create(job=job, key=key, value=value) 1402c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 14030afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return job 14040afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 14050afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 140659cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich @classmethod 14071b52574752be108a743d3b33561c34324f8538e7Jakob Juelich def assign_to_shard(cls, shard, known_ids): 140859cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich """Assigns unassigned jobs to a shard. 140959cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 14101b52574752be108a743d3b33561c34324f8538e7Jakob Juelich For all labels that have been assigned to this shard, all jobs that 14111b52574752be108a743d3b33561c34324f8538e7Jakob Juelich have this label, are assigned to this shard. 14121b52574752be108a743d3b33561c34324f8538e7Jakob Juelich 14131b52574752be108a743d3b33561c34324f8538e7Jakob Juelich Jobs that are assigned to the shard but aren't already present on the 14141b52574752be108a743d3b33561c34324f8538e7Jakob Juelich shard are returned. 14151b52574752be108a743d3b33561c34324f8538e7Jakob Juelich 141659cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich @param shard: The shard to assign jobs to. 14171b52574752be108a743d3b33561c34324f8538e7Jakob Juelich @param known_ids: List of all ids of incomplete jobs, the shard already 14181b52574752be108a743d3b33561c34324f8538e7Jakob Juelich knows about. 14191b52574752be108a743d3b33561c34324f8538e7Jakob Juelich This is used to figure out which jobs should be sent 14201b52574752be108a743d3b33561c34324f8538e7Jakob Juelich to the shard. If shard_ids were used instead, jobs 14211b52574752be108a743d3b33561c34324f8538e7Jakob Juelich would only be transferred once, even if the client 14221b52574752be108a743d3b33561c34324f8538e7Jakob Juelich failed persisting them. 14231b52574752be108a743d3b33561c34324f8538e7Jakob Juelich The number of unfinished jobs usually lies in O(1000). 14241b52574752be108a743d3b33561c34324f8538e7Jakob Juelich Assuming one id takes 8 chars in the json, this means 14251b52574752be108a743d3b33561c34324f8538e7Jakob Juelich overhead that lies in the lower kilobyte range. 14261b52574752be108a743d3b33561c34324f8538e7Jakob Juelich A not in query with 5000 id's takes about 30ms. 14271b52574752be108a743d3b33561c34324f8538e7Jakob Juelich 142859cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich @returns The job objects that should be sent to the shard. 142959cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich """ 143059cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # Disclaimer: Concurrent heartbeats should not occur in today's setup. 143159cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # If this changes or they are triggered manually, this applies: 143259cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # Jobs may be returned more than once by concurrent calls of this 143359cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich # function, as there is a race condition between SELECT and UPDATE. 14342b79fd7627875033b2650c302e385712976aea7fFang Deng job_ids = set([]) 14352b79fd7627875033b2650c302e385712976aea7fFang Deng check_known_jobs_exclude = '' 14362b79fd7627875033b2650c302e385712976aea7fFang Deng check_known_jobs_include = '' 14372b79fd7627875033b2650c302e385712976aea7fFang Deng 14382b79fd7627875033b2650c302e385712976aea7fFang Deng if known_ids: 14392b79fd7627875033b2650c302e385712976aea7fFang Deng check_known_jobs = ( 14402b79fd7627875033b2650c302e385712976aea7fFang Deng cls.NON_ABORTED_KNOWN_JOBS % 14412b79fd7627875033b2650c302e385712976aea7fFang Deng {'known_ids': ','.join([str(i) for i in known_ids])}) 14422b79fd7627875033b2650c302e385712976aea7fFang Deng check_known_jobs_exclude = 'AND NOT ' + check_known_jobs 14432b79fd7627875033b2650c302e385712976aea7fFang Deng check_known_jobs_include = 'OR ' + check_known_jobs 14442b79fd7627875033b2650c302e385712976aea7fFang Deng 14455c7ef30e9d5ad3fd49a57b3ca08520875b2cce6dMK Ryu for sql in [cls.SQL_SHARD_JOBS, cls.SQL_SHARD_JOBS_WITH_HOSTS]: 14462e3e6059f397ba791a2f9c836ed38bb0e3dfe34bMK Ryu query = Job.objects.raw(sql % { 14472e3e6059f397ba791a2f9c836ed38bb0e3dfe34bMK Ryu 'check_known_jobs': check_known_jobs_exclude, 14482e3e6059f397ba791a2f9c836ed38bb0e3dfe34bMK Ryu 'shard_id': shard.id}) 14492b79fd7627875033b2650c302e385712976aea7fFang Deng job_ids |= set([j.id for j in query]) 14502b79fd7627875033b2650c302e385712976aea7fFang Deng 14512b79fd7627875033b2650c302e385712976aea7fFang Deng if job_ids: 14522b79fd7627875033b2650c302e385712976aea7fFang Deng query = Job.objects.raw( 14532b79fd7627875033b2650c302e385712976aea7fFang Deng cls.SQL_JOBS_TO_EXCLUDE % 14542b79fd7627875033b2650c302e385712976aea7fFang Deng {'check_known_jobs': check_known_jobs_include, 14552b79fd7627875033b2650c302e385712976aea7fFang Deng 'candidates': ','.join([str(i) for i in job_ids])}) 14562b79fd7627875033b2650c302e385712976aea7fFang Deng job_ids -= set([j.id for j in query]) 14572b79fd7627875033b2650c302e385712976aea7fFang Deng 145859cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich if job_ids: 145959cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich Job.objects.filter(pk__in=job_ids).update(shard=shard) 146059cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich return list(Job.objects.filter(pk__in=job_ids).all()) 146159cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich return [] 146259cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 146359cfe5497d7f3be2bc4ade144d649a131e88448eJakob Juelich 1464204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li def queue(self, hosts, is_template=False): 14657db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Enqueue a job on the given hosts. 14667db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 14677db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param hosts: The hosts to use. 14687db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param is_template: Whether the status should be "Template". 14697db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 1470a9545c0ab3d8f3e36efadaefdcf37393708666d9showard if not hosts: 1471204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li # hostless job 1472204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li entry = HostQueueEntry.create(job=self, is_template=is_template) 1473204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li entry.save() 1474a9545c0ab3d8f3e36efadaefdcf37393708666d9showard return 1475a9545c0ab3d8f3e36efadaefdcf37393708666d9showard 14760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for host in hosts: 1477204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li host.enqueue_job(self, is_template=is_template) 147829f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 147929f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 14800afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def user(self): 14817db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Gets the user of this job, or None if it doesn't exist.""" 14820afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 14830afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return User.objects.get(login=self.owner) 14840afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except self.DoesNotExist: 14850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 14860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 14870afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 148864a9595406f2884fb3ece241190b10aa054439a9showard def abort(self): 14897db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Aborts this job.""" 14909886397ceb7c752db78a6acd9737992db891015bshoward for queue_entry in self.hostqueueentry_set.all(): 149164a9595406f2884fb3ece241190b10aa054439a9showard queue_entry.abort() 14929886397ceb7c752db78a6acd9737992db891015bshoward 14939886397ceb7c752db78a6acd9737992db891015bshoward 1494d11956572cb7a5c8e9c588c9a6b4a0892de00384showard def tag(self): 14957db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns a string tag for this job.""" 14960c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return server_utils.get_job_tag(self.id, self.owner) 1497d11956572cb7a5c8e9c588c9a6b4a0892de00384showard 1498d11956572cb7a5c8e9c588c9a6b4a0892de00384showard 1499c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard def keyval_dict(self): 15007db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns all keyvals for this job as a dictionary.""" 1501c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard return dict((keyval.key, keyval.value) 1502c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard for keyval in self.jobkeyval_set.all()) 1503c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 1504c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 1505ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @classmethod 1506ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng def get_attribute_model(cls): 1507ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """Return the attribute model. 1508ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1509ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Override method in parent class. This class is called when 1510ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng deserializing the one-to-many relationship betwen Job and JobKeyval. 1511ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng On deserialization, we will try to clear any existing job keyvals 1512ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng associated with a job to avoid any inconsistency. 1513ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Though Job doesn't implement ModelWithAttribute, we still treat 1514ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng it as an attribute model for this purpose. 1515ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1516ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @returns: The attribute model of Job. 1517ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """ 1518ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng return JobKeyval 1519ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1520ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 15210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 15227db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class Job.""" 1523eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_jobs' 15240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1525a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 1526a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return u'%s (%s-%s)' % (self.name, self.id, self.owner) 1527e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1528e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1529c1a98d1e146080bd3e4f034cb13d740dfb1535f4showardclass JobKeyval(dbmodels.Model, model_logic.ModelExtensions): 1530c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard """Keyvals associated with jobs""" 1531ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1532ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng SERIALIZATION_LINKS_TO_KEEP = set(['job']) 1533ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng SERIALIZATION_LOCAL_LINKS_TO_UPDATE = set(['value']) 1534ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1535c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard job = dbmodels.ForeignKey(Job) 1536c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard key = dbmodels.CharField(max_length=90) 1537c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard value = dbmodels.CharField(max_length=300) 1538c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 1539c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard objects = model_logic.ExtendedManager() 1540c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 1541ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1542ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @classmethod 1543ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng def get_record(cls, data): 1544ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """Check the database for an identical record. 1545ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1546ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Use job_id and key to search for a existing record. 1547ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1548ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @raises: DoesNotExist, if no record found 1549ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @raises: MultipleObjectsReturned if multiple records found. 1550ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """ 1551ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng # TODO(fdeng): We should use job_id and key together as 1552ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng # a primary key in the db. 1553ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng return cls.objects.get(job_id=data['job_id'], key=data['key']) 1554ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1555ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1556ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @classmethod 1557ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng def deserialize(cls, data): 1558ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """Override deserialize in parent class. 1559ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1560ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng Do not deserialize id as id is not kept consistent on master and shards. 1561ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1562ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @param data: A dictionary of data to deserialize. 1563ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1564ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng @returns: A JobKeyval object. 1565ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng """ 1566ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng if data: 1567ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng data.pop('id') 1568ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng return super(JobKeyval, cls).deserialize(data) 1569ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1570ff36159d40fc7e7b5124f4950b6a18103aebd922Fang Deng 1571c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard class Meta: 15727db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class JobKeyval.""" 1573c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard db_table = 'afe_job_keyvals' 1574c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 1575c1a98d1e146080bd3e4f034cb13d740dfb1535f4showard 15767c7852819d0611b4d0e8e69b3011b79e4016a770showardclass IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions): 15777db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Represents an ineligible host queue.""" 15780afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski job = dbmodels.ForeignKey(Job) 15790afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host = dbmodels.ForeignKey(Host) 1580e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 15810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski objects = model_logic.ExtendedManager() 1582eb3be4d9b4c1bb292723ea1d52612aec5967ca0eshoward 15830afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 15847db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class IneligibleHostQueue.""" 1585eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_ineligible_host_queues' 1586e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1587e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 15887c7852819d0611b4d0e8e69b3011b79e4016a770showardclass HostQueueEntry(dbmodels.Model, model_logic.ModelExtensions): 15897db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Represents a host queue entry.""" 15903bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 15913bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich SERIALIZATION_LINKS_TO_FOLLOW = set(['meta_host']) 15928c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian SERIALIZATION_LINKS_TO_KEEP = set(['host']) 1593f865d33cc3b0584f10b2f6f4de414a6d7f9e3000Jakob Juelich SERIALIZATION_LOCAL_LINKS_TO_UPDATE = set(['aborted']) 15943bb7c8095d7c66d955462673dd3df5565c1b4a96Jakob Juelich 1595f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1596f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich def custom_deserialize_relation(self, link, data): 1597f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich assert link == 'meta_host' 1598f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich self.meta_host = Label.deserialize(data) 1599f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1600f88fa938050d0e2b662a2b32ef7e931e01a4d8feJakob Juelich 1601a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich def sanity_check_update_from_shard(self, shard, updated_serialized, 1602a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich job_ids_sent): 1603a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich if self.job_id not in job_ids_sent: 1604a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich raise error.UnallowedRecordsSentToMaster( 1605a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 'Sent HostQueueEntry without corresponding ' 1606a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 'job entry: %s' % updated_serialized) 1607a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 1608a94efe60bc94c9aa10ecfe40bddf97518985c7c2Jakob Juelich 1609eaa408e59e932ce0540aa5d50bf966af5080f640showard Status = host_queue_entry_states.Status 1610eaa408e59e932ce0540aa5d50bf966af5080f640showard ACTIVE_STATUSES = host_queue_entry_states.ACTIVE_STATUSES 1611eab66ce582bfe05076ff096c3a044d8f0497bbcashoward COMPLETE_STATUSES = host_queue_entry_states.COMPLETE_STATUSES 1612303d2669b78eb5a2ebd6c41dce80ca18d06a19cfLaurence Goodby PRE_JOB_STATUSES = host_queue_entry_states.PRE_JOB_STATUSES 1613303d2669b78eb5a2ebd6c41dce80ca18d06a19cfLaurence Goodby IDLE_PRE_JOB_STATUSES = host_queue_entry_states.IDLE_PRE_JOB_STATUSES 1614a3ab0d56117c0d55f768d5817284c2c2a0b0305dshoward 16150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski job = dbmodels.ForeignKey(Job) 16160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski host = dbmodels.ForeignKey(Host, blank=True, null=True) 1617a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward status = dbmodels.CharField(max_length=255) 16180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski meta_host = dbmodels.ForeignKey(Label, blank=True, null=True, 16190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski db_column='meta_host') 16200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski active = dbmodels.BooleanField(default=False) 16210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski complete = dbmodels.BooleanField(default=False) 1622b8471e33ed22512001ec4cec8c33fdf01f32eb62showard deleted = dbmodels.BooleanField(default=False) 1623a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward execution_subdir = dbmodels.CharField(max_length=255, blank=True, 1624a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward default='') 162589f84dbadf071ba430244356b57af395c79486e4showard # If atomic_group is set, this is a virtual HostQueueEntry that will 162689f84dbadf071ba430244356b57af395c79486e4showard # be expanded into many actual hosts within the group at schedule time. 162789f84dbadf071ba430244356b57af395c79486e4showard atomic_group = dbmodels.ForeignKey(AtomicGroup, blank=True, null=True) 1628d3dc199703bfb8784a2f8f072d0514532c86c0a9showard aborted = dbmodels.BooleanField(default=False) 1629d3771cc3c770639a816a6a3065aa5a1ef53a24a1showard started_on = dbmodels.DateTimeField(null=True, blank=True) 163051599038f08395067097dc265127cfbcf77c427dFang Deng finished_on = dbmodels.DateTimeField(null=True, blank=True) 1631e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 16320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski objects = model_logic.ExtendedManager() 1633eb3be4d9b4c1bb292723ea1d52612aec5967ca0eshoward 1634e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 16352bab8f45adedeacbf2d62d37b90255581adc3c7dshoward def __init__(self, *args, **kwargs): 16362bab8f45adedeacbf2d62d37b90255581adc3c7dshoward super(HostQueueEntry, self).__init__(*args, **kwargs) 16372bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self._record_attributes(['status']) 16382bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 16392bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 164029f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard @classmethod 1641204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li def create(cls, job, host=None, meta_host=None, 164229f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard is_template=False): 16437db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Creates a new host queue entry. 16447db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 16457db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 16467db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param job: The associated job. 16477db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param host: The associated host. 16487db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param meta_host: The associated meta host. 16497db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param is_template: Whether the status should be "Template". 16507db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 165129f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard if is_template: 165229f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard status = cls.Status.TEMPLATE 165329f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard else: 165429f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard status = cls.Status.QUEUED 165529f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 1656204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li return cls(job=job, host=host, meta_host=meta_host, status=status) 165729f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 165829f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 1659a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def save(self, *args, **kwargs): 16602bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self._set_active_and_complete() 1661a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward super(HostQueueEntry, self).save(*args, **kwargs) 16622bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self._check_for_updated_attributes() 16632bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 16642bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 1665c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard def execution_path(self): 1666c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard """ 1667c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard Path to this entry's results (relative to the base results directory). 1668c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard """ 16690c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return server_utils.get_hqe_exec_path(self.job.tag(), 16700c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu self.execution_subdir) 1671c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard 1672c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard 16733f15eedfedab4b1b9eaff67713e411eada0c84dfshoward def host_or_metahost_name(self): 16747db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns the first non-None name found in priority order. 16757db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 1676204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li The priority order checked is: (1) host name; (2) meta host name 16777db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 16783f15eedfedab4b1b9eaff67713e411eada0c84dfshoward if self.host: 16793f15eedfedab4b1b9eaff67713e411eada0c84dfshoward return self.host.hostname 16807890e797d3549b3de18b79448817ffc80a79ee42showard else: 1681204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li assert self.meta_host 1682204eac11dbaf4b5659ebefdb50a536c821e9bb3eAllen Li return self.meta_host.name 16833f15eedfedab4b1b9eaff67713e411eada0c84dfshoward 16843f15eedfedab4b1b9eaff67713e411eada0c84dfshoward 16852bab8f45adedeacbf2d62d37b90255581adc3c7dshoward def _set_active_and_complete(self): 1686d3dc199703bfb8784a2f8f072d0514532c86c0a9showard if self.status in self.ACTIVE_STATUSES: 16872bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self.active, self.complete = True, False 16882bab8f45adedeacbf2d62d37b90255581adc3c7dshoward elif self.status in self.COMPLETE_STATUSES: 16892bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self.active, self.complete = False, True 16902bab8f45adedeacbf2d62d37b90255581adc3c7dshoward else: 16912bab8f45adedeacbf2d62d37b90255581adc3c7dshoward self.active, self.complete = False, False 16922bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 16932bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 16942bab8f45adedeacbf2d62d37b90255581adc3c7dshoward def on_attribute_changed(self, attribute, old_value): 16952bab8f45adedeacbf2d62d37b90255581adc3c7dshoward assert attribute == 'status' 16967db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey logging.info('%s/%d (%d) -> %s', self.host, self.job.id, self.id, 16977db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey self.status) 16982bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 16992bab8f45adedeacbf2d62d37b90255581adc3c7dshoward 17000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski def is_meta_host_entry(self): 17010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 'True if this is a entry has a meta_host instead of a host.' 17020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return self.host is None and self.meta_host is not None 1703e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1704a3ab0d56117c0d55f768d5817284c2c2a0b0305dshoward 1705c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # This code is shared between rpc_interface and models.HostQueueEntry. 1706c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # Sadly due to circular imports between the 2 (crbug.com/230100) making it 1707c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # a class method was the best way to refactor it. Attempting to put it in 1708c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # rpc_utils or a new utils module failed as that would require us to import 1709c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # models.py but to call it from here we would have to import the utils.py 1710c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # thus creating a cycle. 1711c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi @classmethod 1712c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi def abort_host_queue_entries(cls, host_queue_entries): 1713c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi """Aborts a collection of host_queue_entries. 1714c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1715c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi Abort these host queue entry and all host queue entries of jobs created 1716c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi by them. 1717c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1718c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi @param host_queue_entries: List of host queue entries we want to abort. 1719c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi """ 1720c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # This isn't completely immune to race conditions since it's not atomic, 1721c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # but it should be safe given the scheduler's behavior. 1722c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1723c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # TODO(milleral): crbug.com/230100 1724c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # The |abort_host_queue_entries| rpc does nearly exactly this, 1725c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # however, trying to re-use the code generates some horrible 1726c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # circular import error. I'd be nice to refactor things around 1727c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # sometime so the code could be reused. 1728c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1729c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # Fixpoint algorithm to find the whole tree of HQEs to abort to 1730c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # minimize the total number of database queries: 1731c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi children = set() 1732c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi new_children = set(host_queue_entries) 1733c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi while new_children: 1734c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi children.update(new_children) 1735c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi new_child_ids = [hqe.job_id for hqe in new_children] 1736c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi new_children = HostQueueEntry.objects.filter( 1737c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi job__parent_job__in=new_child_ids, 1738c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi complete=False, aborted=False).all() 1739c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # To handle circular parental relationships 1740c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi new_children = set(new_children) - children 1741c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1742c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # Associate a user with the host queue entries that we're about 1743c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # to abort so that we can look up who to blame for the aborts. 1744c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi now = datetime.now() 1745c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi user = User.current_user() 1746c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi aborted_hqes = [AbortedHostQueueEntry(queue_entry=hqe, 1747c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi aborted_by=user, aborted_on=now) for hqe in children] 1748c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi AbortedHostQueueEntry.objects.bulk_create(aborted_hqes) 1749c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi # Bulk update all of the HQEs to set the abort bit. 1750c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi child_ids = [hqe.id for hqe in children] 1751c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi HostQueueEntry.objects.filter(id__in=child_ids).update(aborted=True) 1752c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1753c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1754dea67041f8d99fa3a385d7be919e4477cf642a34Alex Miller def abort(self): 1755dea67041f8d99fa3a385d7be919e4477cf642a34Alex Miller """ Aborts this host queue entry. 1756c1b26769845fe59168f2d9b7d6b1d37d93ea4bf1Simran Basi 1757dea67041f8d99fa3a385d7be919e4477cf642a34Alex Miller Abort this host queue entry and all host queue entries of jobs created by 1758dea67041f8d99fa3a385d7be919e4477cf642a34Alex Miller this one. 17597db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 17607db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 1761d3dc199703bfb8784a2f8f072d0514532c86c0a9showard if not self.complete and not self.aborted: 176297582a2097bfb27a41514e9eaa071de50d1f7be6Simran Basi HostQueueEntry.abort_host_queue_entries([self]) 1763e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 1764d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 1765d3dc199703bfb8784a2f8f072d0514532c86c0a9showard @classmethod 1766d3dc199703bfb8784a2f8f072d0514532c86c0a9showard def compute_full_status(cls, status, aborted, complete): 17677db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns a modified status msg if the host queue entry was aborted. 17687db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 17697db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 17707db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param status: The original status message. 17717db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param aborted: Whether the host queue entry was aborted. 17727db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param complete: Whether the host queue entry was completed. 17737db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """ 1774d3dc199703bfb8784a2f8f072d0514532c86c0a9showard if aborted and not complete: 1775d3dc199703bfb8784a2f8f072d0514532c86c0a9showard return 'Aborted (%s)' % status 1776d3dc199703bfb8784a2f8f072d0514532c86c0a9showard return status 1777d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 1778d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 1779d3dc199703bfb8784a2f8f072d0514532c86c0a9showard def full_status(self): 17807db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns the full status of this host queue entry, as a string.""" 1781d3dc199703bfb8784a2f8f072d0514532c86c0a9showard return self.compute_full_status(self.status, self.aborted, 1782d3dc199703bfb8784a2f8f072d0514532c86c0a9showard self.complete) 1783d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 1784d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 1785d3dc199703bfb8784a2f8f072d0514532c86c0a9showard def _postprocess_object_dict(self, object_dict): 1786d3dc199703bfb8784a2f8f072d0514532c86c0a9showard object_dict['full_status'] = self.full_status() 1787d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 1788d3dc199703bfb8784a2f8f072d0514532c86c0a9showard 17890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski class Meta: 17907db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class HostQueueEntry.""" 1791eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_host_queue_entries' 1792e8819cdf80ca0e0602d22551a50f970aa68e108dmbligh 179312f3e3212795a539d95973f893ac570e669e3a22showard 17944c119049441b360505464f31cfd113b5dc90b241showard 1795a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 179612f3e3212795a539d95973f893ac570e669e3a22showard hostname = None 179712f3e3212795a539d95973f893ac570e669e3a22showard if self.host: 179812f3e3212795a539d95973f893ac570e669e3a22showard hostname = self.host.hostname 1799a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward return u"%s/%d (%d)" % (hostname, self.job.id, self.id) 180012f3e3212795a539d95973f893ac570e669e3a22showard 180112f3e3212795a539d95973f893ac570e669e3a22showard 18024c119049441b360505464f31cfd113b5dc90b241showardclass AbortedHostQueueEntry(dbmodels.Model, model_logic.ModelExtensions): 18037db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Represents an aborted host queue entry.""" 18044c119049441b360505464f31cfd113b5dc90b241showard queue_entry = dbmodels.OneToOneField(HostQueueEntry, primary_key=True) 18054c119049441b360505464f31cfd113b5dc90b241showard aborted_by = dbmodels.ForeignKey(User) 180668c7aa02743042196b1f4ce3d41cbe03d1152ec0showard aborted_on = dbmodels.DateTimeField() 18074c119049441b360505464f31cfd113b5dc90b241showard 18084c119049441b360505464f31cfd113b5dc90b241showard objects = model_logic.ExtendedManager() 18094c119049441b360505464f31cfd113b5dc90b241showard 181068c7aa02743042196b1f4ce3d41cbe03d1152ec0showard 1811a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def save(self, *args, **kwargs): 181268c7aa02743042196b1f4ce3d41cbe03d1152ec0showard self.aborted_on = datetime.now() 1813a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward super(AbortedHostQueueEntry, self).save(*args, **kwargs) 181468c7aa02743042196b1f4ce3d41cbe03d1152ec0showard 18154c119049441b360505464f31cfd113b5dc90b241showard class Meta: 18167db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class AbortedHostQueueEntry.""" 1817eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_aborted_host_queue_entries' 181829f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 181929f7cd27e51add2648fb62ab2a0c588f9acb1ec4showard 18206d7b2ff05b2232b1b225a4cb3521d76c0152cad9showardclass SpecialTask(dbmodels.Model, model_logic.ModelExtensions): 18216d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard """\ 18226d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard Tasks to run on hosts at the next time they are in the Ready state. Use this 18236d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard for high-priority tasks, such as forced repair or forced reinstall. 18246d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 18256d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard host: host to run this task on 18262fe3f1df42f5fd1dc6296219df289851dcf77025showard task: special task to run 18276d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard time_requested: date and time the request for this task was made 18286d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard is_active: task is currently running 18296d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard is_complete: task has finished running 18308bb1f7d8766d202d220879642a7d5c40e2775e52beeps is_aborted: task was aborted 18312fe3f1df42f5fd1dc6296219df289851dcf77025showard time_started: date and time the task started 1832d07255493d6301c5fd58762073d70e2b8a520671Dan Shi time_finished: date and time the task finished 18332fe3f1df42f5fd1dc6296219df289851dcf77025showard queue_entry: Host queue entry waiting on this task (or None, if task was not 18342fe3f1df42f5fd1dc6296219df289851dcf77025showard started in preparation of a job) 18356d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard """ 1836dfff2fdc8477be3ba89fd915fde2afe8d3716624Alex Miller Task = enum.Enum('Verify', 'Cleanup', 'Repair', 'Reset', 'Provision', 183707e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi string_values=True) 18386d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 18396d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard host = dbmodels.ForeignKey(Host, blank=False, null=False) 1840a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward task = dbmodels.CharField(max_length=64, choices=Task.choices(), 18416d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard blank=False, null=False) 184276fcf19ec42d5c7580d2e7891e4610e5fe725286jamesren requested_by = dbmodels.ForeignKey(User) 18436d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard time_requested = dbmodels.DateTimeField(auto_now_add=True, blank=False, 18446d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard null=False) 18456d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard is_active = dbmodels.BooleanField(default=False, blank=False, null=False) 18466d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard is_complete = dbmodels.BooleanField(default=False, blank=False, null=False) 18478bb1f7d8766d202d220879642a7d5c40e2775e52beeps is_aborted = dbmodels.BooleanField(default=False, blank=False, null=False) 1848c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard time_started = dbmodels.DateTimeField(null=True, blank=True) 18492fe3f1df42f5fd1dc6296219df289851dcf77025showard queue_entry = dbmodels.ForeignKey(HostQueueEntry, blank=True, null=True) 1850e60e44ece1445d97977a77cb79f0896989b869d7showard success = dbmodels.BooleanField(default=False, blank=False, null=False) 1851d07255493d6301c5fd58762073d70e2b8a520671Dan Shi time_finished = dbmodels.DateTimeField(null=True, blank=True) 18526d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 18536d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard objects = model_logic.ExtendedManager() 18546d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 18556d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 18569bb960b90d5102cce1c8a15314900035c6c4e69ashoward def save(self, **kwargs): 18579bb960b90d5102cce1c8a15314900035c6c4e69ashoward if self.queue_entry: 18589bb960b90d5102cce1c8a15314900035c6c4e69ashoward self.requested_by = User.objects.get( 18599bb960b90d5102cce1c8a15314900035c6c4e69ashoward login=self.queue_entry.job.owner) 18609bb960b90d5102cce1c8a15314900035c6c4e69ashoward super(SpecialTask, self).save(**kwargs) 18619bb960b90d5102cce1c8a15314900035c6c4e69ashoward 18629bb960b90d5102cce1c8a15314900035c6c4e69ashoward 1863ed2afea4ca6e23a82d20d1f2ee1067d0c25a8cc2showard def execution_path(self): 18640c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Returns the execution path for a special task.""" 18650c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return server_utils.get_special_task_exec_path( 18660c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu self.host.hostname, self.id, self.task, self.time_requested) 1867ed2afea4ca6e23a82d20d1f2ee1067d0c25a8cc2showard 1868ed2afea4ca6e23a82d20d1f2ee1067d0c25a8cc2showard 1869c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard # property to emulate HostQueueEntry.status 1870c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard @property 1871c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard def status(self): 18720c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Returns a host queue entry status appropriate for a speical task.""" 18730c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return server_utils.get_special_task_status( 18740c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu self.is_complete, self.success, self.is_active) 1875c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard 1876c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard 1877c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard # property to emulate HostQueueEntry.started_on 1878c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard @property 1879c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard def started_on(self): 18807db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Returns the time at which this special task started.""" 1881c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard return self.time_started 1882c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard 1883c0ac3a79cceec87873257a8b6d41d19c9fb02ec6showard 18846d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard @classmethod 1885c510344e16543a6f810be1608e1a879c3d3bbcdeshoward def schedule_special_task(cls, host, task): 18867db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Schedules a special task on a host if not already scheduled. 18877db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 18887db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param cls: Implicit class object. 18897db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param host: The host to use. 18907db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param task: The task to schedule. 18916d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard """ 1892c510344e16543a6f810be1608e1a879c3d3bbcdeshoward existing_tasks = SpecialTask.objects.filter(host__id=host.id, task=task, 1893c510344e16543a6f810be1608e1a879c3d3bbcdeshoward is_active=False, 1894c510344e16543a6f810be1608e1a879c3d3bbcdeshoward is_complete=False) 1895c510344e16543a6f810be1608e1a879c3d3bbcdeshoward if existing_tasks: 1896c510344e16543a6f810be1608e1a879c3d3bbcdeshoward return existing_tasks[0] 1897c510344e16543a6f810be1608e1a879c3d3bbcdeshoward 1898c510344e16543a6f810be1608e1a879c3d3bbcdeshoward special_task = SpecialTask(host=host, task=task, 1899c510344e16543a6f810be1608e1a879c3d3bbcdeshoward requested_by=User.current_user()) 1900c510344e16543a6f810be1608e1a879c3d3bbcdeshoward special_task.save() 1901c510344e16543a6f810be1608e1a879c3d3bbcdeshoward return special_task 19022fe3f1df42f5fd1dc6296219df289851dcf77025showard 19032fe3f1df42f5fd1dc6296219df289851dcf77025showard 19048bb1f7d8766d202d220879642a7d5c40e2775e52beeps def abort(self): 19058bb1f7d8766d202d220879642a7d5c40e2775e52beeps """ Abort this special task.""" 19068bb1f7d8766d202d220879642a7d5c40e2775e52beeps self.is_aborted = True 19078bb1f7d8766d202d220879642a7d5c40e2775e52beeps self.save() 19088bb1f7d8766d202d220879642a7d5c40e2775e52beeps 19098bb1f7d8766d202d220879642a7d5c40e2775e52beeps 1910ed2afea4ca6e23a82d20d1f2ee1067d0c25a8cc2showard def activate(self): 1911474d136e9a71f288f294167112dda909f9273434showard """ 1912474d136e9a71f288f294167112dda909f9273434showard Sets a task as active and sets the time started to the current time. 19132fe3f1df42f5fd1dc6296219df289851dcf77025showard """ 191497446887819594f1e2a329dcff289ee8e934b626showard logging.info('Starting: %s', self) 19152fe3f1df42f5fd1dc6296219df289851dcf77025showard self.is_active = True 19162fe3f1df42f5fd1dc6296219df289851dcf77025showard self.time_started = datetime.now() 19172fe3f1df42f5fd1dc6296219df289851dcf77025showard self.save() 19182fe3f1df42f5fd1dc6296219df289851dcf77025showard 19192fe3f1df42f5fd1dc6296219df289851dcf77025showard 1920e60e44ece1445d97977a77cb79f0896989b869d7showard def finish(self, success): 19217db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Sets a task as completed. 19227db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey 19237db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey @param success: Whether or not the task was successful. 19242fe3f1df42f5fd1dc6296219df289851dcf77025showard """ 192597446887819594f1e2a329dcff289ee8e934b626showard logging.info('Finished: %s', self) 1926ed2afea4ca6e23a82d20d1f2ee1067d0c25a8cc2showard self.is_active = False 19272fe3f1df42f5fd1dc6296219df289851dcf77025showard self.is_complete = True 1928e60e44ece1445d97977a77cb79f0896989b869d7showard self.success = success 1929d85d611e7560f28baf622c0f76b992e0a7fb2789Dan Shi if self.time_started: 1930d85d611e7560f28baf622c0f76b992e0a7fb2789Dan Shi self.time_finished = datetime.now() 19312fe3f1df42f5fd1dc6296219df289851dcf77025showard self.save() 19326d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 19336d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 19346d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard class Meta: 19357db38bab49bb85027c216bdf89e289d0f24dba16Dennis Jeffrey """Metadata for class SpecialTask.""" 1936eab66ce582bfe05076ff096c3a044d8f0497bbcashoward db_table = 'afe_special_tasks' 19376d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 1938474d136e9a71f288f294167112dda909f9273434showard 1939a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward def __unicode__(self): 1940a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward result = u'Special Task %s (host %s, task %s, time %s)' % ( 1941ed2afea4ca6e23a82d20d1f2ee1067d0c25a8cc2showard self.id, self.host, self.task, self.time_requested) 19426d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard if self.is_complete: 1943a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward result += u' (completed)' 19446d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard elif self.is_active: 1945a5288b4bb2b09aafe914d0b7d5aab79a7e433eafshoward result += u' (active)' 19466d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard 19476d7b2ff05b2232b1b225a4cb3521d76c0152cad9showard return result 19486964fa53798525f39cd490e26481e8d923815406Dan Shi 19496964fa53798525f39cd490e26481e8d923815406Dan Shi 19506964fa53798525f39cd490e26481e8d923815406Dan Shiclass StableVersion(dbmodels.Model, model_logic.ModelExtensions): 19516964fa53798525f39cd490e26481e8d923815406Dan Shi 19526964fa53798525f39cd490e26481e8d923815406Dan Shi board = dbmodels.CharField(max_length=255, unique=True) 19536964fa53798525f39cd490e26481e8d923815406Dan Shi version = dbmodels.CharField(max_length=255) 19546964fa53798525f39cd490e26481e8d923815406Dan Shi 19556964fa53798525f39cd490e26481e8d923815406Dan Shi class Meta: 19566964fa53798525f39cd490e26481e8d923815406Dan Shi """Metadata for class StableVersion.""" 195786248502517d42d6c036a66a57e3a24d4a6dc3b8Fang Deng db_table = 'afe_stable_versions' 1958