models.py revision fb2a7fa621ddc91634dde6c56a47a1c8df2610ef
1from datetime import datetime 2from django.db import models as dbmodels, connection 3from frontend.afe import model_logic 4from frontend import settings, thread_local 5from autotest_lib.client.common_lib import enum, host_protections 6 7 8class AclAccessViolation(Exception): 9 """\ 10 Raised when an operation is attempted with proper permissions as 11 dictated by ACLs. 12 """ 13 14 15class Label(model_logic.ModelWithInvalid, dbmodels.Model): 16 """\ 17 Required: 18 name: label name 19 20 Optional: 21 kernel_config: url/path to kernel config to use for jobs run on this 22 label 23 platform: if True, this is a platform label (defaults to False) 24 """ 25 name = dbmodels.CharField(maxlength=255, unique=True) 26 kernel_config = dbmodels.CharField(maxlength=255, blank=True) 27 platform = dbmodels.BooleanField(default=False) 28 invalid = dbmodels.BooleanField(default=False, 29 editable=settings.FULL_ADMIN) 30 31 name_field = 'name' 32 objects = model_logic.ExtendedManager() 33 valid_objects = model_logic.ValidObjectsManager() 34 35 def clean_object(self): 36 self.host_set.clear() 37 38 39 def enqueue_job(self, job): 40 'Enqueue a job on any host of this label.' 41 queue_entry = HostQueueEntry(meta_host=self, job=job, 42 status=Job.Status.QUEUED, 43 priority=job.priority) 44 queue_entry.save() 45 46 47 class Meta: 48 db_table = 'labels' 49 50 class Admin: 51 list_display = ('name', 'kernel_config') 52 # see Host.Admin 53 manager = model_logic.ValidObjectsManager() 54 55 def __str__(self): 56 return self.name 57 58 59class User(dbmodels.Model, model_logic.ModelExtensions): 60 """\ 61 Required: 62 login :user login name 63 64 Optional: 65 access_level: 0=User (default), 1=Admin, 100=Root 66 """ 67 ACCESS_ROOT = 100 68 ACCESS_ADMIN = 1 69 ACCESS_USER = 0 70 71 login = dbmodels.CharField(maxlength=255, unique=True) 72 access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True) 73 74 name_field = 'login' 75 objects = model_logic.ExtendedManager() 76 77 78 def save(self): 79 # is this a new object being saved for the first time? 80 first_time = (self.id is None) 81 user = thread_local.get_user() 82 if user and not user.is_superuser(): 83 raise AclAccessViolation("You cannot modify users!") 84 super(User, self).save() 85 if first_time: 86 everyone = AclGroup.objects.get(name='Everyone') 87 everyone.users.add(self) 88 89 90 def is_superuser(self): 91 return self.access_level >= self.ACCESS_ROOT 92 93 94 class Meta: 95 db_table = 'users' 96 97 class Admin: 98 list_display = ('login', 'access_level') 99 search_fields = ('login',) 100 101 def __str__(self): 102 return self.login 103 104 105class Host(model_logic.ModelWithInvalid, dbmodels.Model): 106 """\ 107 Required: 108 hostname 109 110 optional: 111 locked: host is locked and will not be queued 112 113 Internal: 114 synch_id: currently unused 115 status: string describing status of host 116 """ 117 Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing', 118 'Repair Failed', 'Dead', 'Rebooting', 119 string_values=True) 120 121 hostname = dbmodels.CharField(maxlength=255, unique=True) 122 labels = dbmodels.ManyToManyField(Label, blank=True, 123 filter_interface=dbmodels.HORIZONTAL) 124 locked = dbmodels.BooleanField(default=False) 125 synch_id = dbmodels.IntegerField(blank=True, null=True, 126 editable=settings.FULL_ADMIN) 127 status = dbmodels.CharField(maxlength=255, default=Status.READY, 128 choices=Status.choices(), 129 editable=settings.FULL_ADMIN) 130 invalid = dbmodels.BooleanField(default=False, 131 editable=settings.FULL_ADMIN) 132 protection = dbmodels.SmallIntegerField(null=False, 133 choices=host_protections.choices, 134 default=host_protections.default) 135 locked_by = dbmodels.ForeignKey(User, null=True, blank=True, editable=False) 136 lock_time = dbmodels.DateTimeField(null=True, blank=True, editable=False) 137 138 name_field = 'hostname' 139 objects = model_logic.ExtendedManager() 140 valid_objects = model_logic.ValidObjectsManager() 141 142 @staticmethod 143 def create_one_time_host(hostname): 144 query = Host.objects.filter(hostname=hostname) 145 if query.count() == 0: 146 host = Host(hostname=hostname, invalid=True) 147 else: 148 host = query[0] 149 if not host.invalid: 150 raise model_logic.ValidationError({ 151 'hostname' : '%s already exists!' % hostname 152 }) 153 host.clean_object() 154 host.status = Host.Status.READY 155 host.save() 156 return host 157 158 def clean_object(self): 159 self.aclgroup_set.clear() 160 self.labels.clear() 161 162 163 def save(self): 164 # extra spaces in the hostname can be a sneaky source of errors 165 self.hostname = self.hostname.strip() 166 # is this a new object being saved for the first time? 167 first_time = (self.id is None) 168 if not first_time: 169 AclGroup.check_for_acl_violation_hosts([self]) 170 if self.locked and not self.locked_by: 171 self.locked_by = thread_local.get_user() 172 self.lock_time = datetime.now() 173 elif not self.locked and self.locked_by: 174 self.locked_by = None 175 self.lock_time = None 176 super(Host, self).save() 177 if first_time: 178 everyone = AclGroup.objects.get(name='Everyone') 179 everyone.hosts.add(self) 180 181 def delete(self): 182 AclGroup.check_for_acl_violation_hosts([self]) 183 for queue_entry in self.hostqueueentry_set.all(): 184 queue_entry.deleted = True 185 queue_entry.abort() 186 super(Host, self).delete() 187 188 189 def enqueue_job(self, job): 190 ' Enqueue a job on this host.' 191 queue_entry = HostQueueEntry(host=self, job=job, 192 status=Job.Status.QUEUED, 193 priority=job.priority) 194 # allow recovery of dead hosts from the frontend 195 if not self.active_queue_entry() and self.is_dead(): 196 self.status = Host.Status.READY 197 self.save() 198 queue_entry.save() 199 200 block = IneligibleHostQueue(job=job, host=self) 201 block.save() 202 203 204 def platform(self): 205 # TODO(showard): slighly hacky? 206 platforms = self.labels.filter(platform=True) 207 if len(platforms) == 0: 208 return None 209 return platforms[0] 210 platform.short_description = 'Platform' 211 212 213 def is_dead(self): 214 return self.status == Host.Status.REPAIR_FAILED 215 216 217 def active_queue_entry(self): 218 active = list(self.hostqueueentry_set.filter(active=True)) 219 if not active: 220 return None 221 assert len(active) == 1, ('More than one active entry for ' 222 'host ' + self.hostname) 223 return active[0] 224 225 226 class Meta: 227 db_table = 'hosts' 228 229 class Admin: 230 # TODO(showard) - showing platform requires a SQL query for 231 # each row (since labels are many-to-many) - should we remove 232 # it? 233 list_display = ('hostname', 'platform', 'locked', 'status') 234 list_filter = ('labels', 'locked', 'protection') 235 search_fields = ('hostname', 'status') 236 # undocumented Django feature - if you set manager here, the 237 # admin code will use it, otherwise it'll use a default Manager 238 manager = model_logic.ValidObjectsManager() 239 240 def __str__(self): 241 return self.hostname 242 243 244class Test(dbmodels.Model, model_logic.ModelExtensions): 245 """\ 246 Required: 247 author: author name 248 description: description of the test 249 name: test name 250 time: short, medium, long 251 test_class: This describes the class for your the test belongs in. 252 test_category: This describes the category for your tests 253 test_type: Client or Server 254 path: path to pass to run_test() 255 synch_type: whether the test should run synchronously or asynchronously 256 sync_count: is a number >=1 (1 being the default). If it's 1, then it's an 257 async job. If it's >1 it's sync job for that number of machines 258 i.e. if sync_count = 2 it is a sync job that requires two 259 machines. 260 Optional: 261 dependencies: What the test requires to run. Comma deliminated list 262 experimental: If this is set to True production servers will ignore the test 263 run_verify: Whether or not the scheduler should run the verify stage 264 """ 265 TestTime = enum.Enum('SHORT', 'MEDIUM', 'LONG', start_value=1) 266 SynchType = enum.Enum('Asynchronous', 'Synchronous', start_value=1) 267 # TODO(showard) - this should be merged with Job.ControlType (but right 268 # now they use opposite values) 269 Types = enum.Enum('Client', 'Server', start_value=1) 270 271 name = dbmodels.CharField(maxlength=255, unique=True) 272 author = dbmodels.CharField(maxlength=255) 273 test_class = dbmodels.CharField(maxlength=255) 274 test_category = dbmodels.CharField(maxlength=255) 275 dependencies = dbmodels.CharField(maxlength=255, blank=True) 276 description = dbmodels.TextField(blank=True) 277 experimental = dbmodels.BooleanField(default=True) 278 run_verify = dbmodels.BooleanField(default=True) 279 test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(), 280 default=TestTime.MEDIUM) 281 test_type = dbmodels.SmallIntegerField(choices=Types.choices()) 282 sync_count = dbmodels.IntegerField(default=1) 283 synch_type = dbmodels.SmallIntegerField(choices=SynchType.choices(), 284 default=SynchType.ASYNCHRONOUS) 285 path = dbmodels.CharField(maxlength=255, unique=True) 286 287 name_field = 'name' 288 objects = model_logic.ExtendedManager() 289 290 291 class Meta: 292 db_table = 'autotests' 293 294 class Admin: 295 fields = ( 296 (None, {'fields' : 297 ('name', 'author', 'test_category', 'test_class', 298 'test_time', 'synch_type', 'test_type', 'sync_count', 299 'path', 'dependencies', 'experimental', 'run_verify', 300 'description')}), 301 ) 302 list_display = ('name', 'test_type', 'description', 'synch_type') 303 search_fields = ('name',) 304 305 def __str__(self): 306 return self.name 307 308 309class Profiler(dbmodels.Model, model_logic.ModelExtensions): 310 """\ 311 Required: 312 name: profiler name 313 test_type: Client or Server 314 315 Optional: 316 description: arbirary text description 317 """ 318 name = dbmodels.CharField(maxlength=255, unique=True) 319 description = dbmodels.TextField(blank=True) 320 321 name_field = 'name' 322 objects = model_logic.ExtendedManager() 323 324 325 class Meta: 326 db_table = 'profilers' 327 328 class Admin: 329 list_display = ('name', 'description') 330 search_fields = ('name',) 331 332 def __str__(self): 333 return self.name 334 335 336class AclGroup(dbmodels.Model, model_logic.ModelExtensions): 337 """\ 338 Required: 339 name: name of ACL group 340 341 Optional: 342 description: arbitrary description of group 343 """ 344 name = dbmodels.CharField(maxlength=255, unique=True) 345 description = dbmodels.CharField(maxlength=255, blank=True) 346 users = dbmodels.ManyToManyField(User, 347 filter_interface=dbmodels.HORIZONTAL) 348 hosts = dbmodels.ManyToManyField(Host, 349 filter_interface=dbmodels.HORIZONTAL) 350 351 name_field = 'name' 352 objects = model_logic.ExtendedManager() 353 354 @staticmethod 355 def check_for_acl_violation_hosts(hosts): 356 user = thread_local.get_user() 357 if user.is_superuser(): 358 return None 359 accessible_host_ids = set( 360 host.id for host in Host.objects.filter(acl_group__users=user)) 361 for host in hosts: 362 # Check if the user has access to this host, 363 # but only if it is not a metahost 364 if (isinstance(host, Host) 365 and int(host.id) not in accessible_host_ids): 366 raise AclAccessViolation("You do not have access to %s" 367 % str(host)) 368 369 def check_for_acl_violation_acl_group(self): 370 user = thread_local.get_user() 371 if user.is_superuser(): 372 return None 373 if not user in self.users.all(): 374 raise AclAccessViolation("You do not have access to %s" 375 % self.name) 376 377 @staticmethod 378 def on_host_membership_change(): 379 everyone = AclGroup.objects.get(name='Everyone') 380 381 # find hosts that aren't in any ACL group and add them to Everyone 382 # TODO(showard): this is a bit of a hack, since the fact that this query 383 # works is kind of a coincidence of Django internals. This trick 384 # doesn't work in general (on all foreign key relationships). I'll 385 # replace it with a better technique when the need arises. 386 orphaned_hosts = Host.valid_objects.filter(acl_group__id__isnull=True) 387 everyone.hosts.add(*orphaned_hosts.distinct()) 388 389 # find hosts in both Everyone and another ACL group, and remove them 390 # from Everyone 391 hosts_in_everyone = Host.valid_objects.filter_custom_join( 392 '_everyone', acl_group__name='Everyone') 393 acled_hosts = hosts_in_everyone.exclude(acl_group__name='Everyone') 394 everyone.hosts.remove(*acled_hosts.distinct()) 395 396 397 def delete(self): 398 if (self.name == 'Everyone'): 399 raise AclAccessViolation("You cannot delete 'Everyone'!") 400 self.check_for_acl_violation_acl_group() 401 super(AclGroup, self).delete() 402 self.on_host_membership_change() 403 404 405 # if you have a model attribute called "Manipulator", Django will 406 # automatically insert it into the beginning of the superclass list 407 # for the model's manipulators 408 class Manipulator(object): 409 """ 410 Custom manipulator to get notification when ACLs are changed through 411 the admin interface. 412 """ 413 def save(self, new_data): 414 user = thread_local.get_user() 415 if (not user.is_superuser() 416 and self.original_object.name == 'Everyone'): 417 raise AclAccessViolation("You cannot modify 'Everyone'!") 418 self.original_object.check_for_acl_violation_acl_group() 419 obj = super(AclGroup.Manipulator, self).save(new_data) 420 obj.on_host_membership_change() 421 return obj 422 423 class Meta: 424 db_table = 'acl_groups' 425 426 class Admin: 427 list_display = ('name', 'description') 428 search_fields = ('name',) 429 430 def __str__(self): 431 return self.name 432 433# hack to make the column name in the many-to-many DB tables match the one 434# generated by ruby 435AclGroup._meta.object_name = 'acl_group' 436 437 438class JobManager(model_logic.ExtendedManager): 439 'Custom manager to provide efficient status counts querying.' 440 def get_status_counts(self, job_ids): 441 """\ 442 Returns a dictionary mapping the given job IDs to their status 443 count dictionaries. 444 """ 445 if not job_ids: 446 return {} 447 id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids) 448 cursor = connection.cursor() 449 cursor.execute(""" 450 SELECT job_id, status, COUNT(*) 451 FROM host_queue_entries 452 WHERE job_id IN %s 453 GROUP BY job_id, status 454 """ % id_list) 455 all_job_counts = {} 456 for job_id in job_ids: 457 all_job_counts[job_id] = {} 458 for job_id, status, count in cursor.fetchall(): 459 all_job_counts[job_id][status] = count 460 return all_job_counts 461 462 463class Job(dbmodels.Model, model_logic.ModelExtensions): 464 """\ 465 owner: username of job owner 466 name: job name (does not have to be unique) 467 priority: Low, Medium, High, Urgent (or 0-3) 468 control_file: contents of control file 469 control_type: Client or Server 470 created_on: date of job creation 471 submitted_on: date of job submission 472 synch_type: Asynchronous or Synchronous (i.e. job must run on all hosts 473 simultaneously; used for server-side control files) 474 synch_count: ??? 475 run_verify: Whether or not to run the verify phase 476 synchronizing: for scheduler use 477 timeout: hours until job times out 478 """ 479 Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent') 480 ControlType = enum.Enum('Server', 'Client', start_value=1) 481 Status = enum.Enum('Created', 'Queued', 'Pending', 'Running', 482 'Completed', 'Abort', 'Aborting', 'Aborted', 483 'Failed', string_values=True) 484 485 owner = dbmodels.CharField(maxlength=255) 486 name = dbmodels.CharField(maxlength=255) 487 priority = dbmodels.SmallIntegerField(choices=Priority.choices(), 488 blank=True, # to allow 0 489 default=Priority.MEDIUM) 490 control_file = dbmodels.TextField() 491 control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(), 492 blank=True) # to allow 0 493 created_on = dbmodels.DateTimeField(auto_now_add=True) 494 synch_type = dbmodels.SmallIntegerField( 495 blank=True, null=True, choices=Test.SynchType.choices()) 496 synch_count = dbmodels.IntegerField(blank=True, null=True) 497 synchronizing = dbmodels.BooleanField(default=False) 498 run_verify = dbmodels.BooleanField(default=True) 499 timeout = dbmodels.IntegerField() 500 501 502 # custom manager 503 objects = JobManager() 504 505 506 def is_server_job(self): 507 return self.control_type == self.ControlType.SERVER 508 509 510 @classmethod 511 def create(cls, owner, name, priority, control_file, control_type, 512 hosts, synch_type, timeout, run_verify): 513 """\ 514 Creates a job by taking some information (the listed args) 515 and filling in the rest of the necessary information. 516 """ 517 AclGroup.check_for_acl_violation_hosts(hosts) 518 job = cls.add_object( 519 owner=owner, name=name, priority=priority, 520 control_file=control_file, control_type=control_type, 521 synch_type=synch_type, timeout=timeout, 522 run_verify=run_verify) 523 524 if job.synch_type == Test.SynchType.SYNCHRONOUS: 525 job.synch_count = len(hosts) 526 else: 527 if len(hosts) == 0: 528 errors = {'hosts': 529 'asynchronous jobs require at least' 530 + ' one host to run on'} 531 raise model_logic.ValidationError(errors) 532 job.save() 533 return job 534 535 536 def queue(self, hosts): 537 'Enqueue a job on the given hosts.' 538 for host in hosts: 539 host.enqueue_job(self) 540 541 542 def requeue(self, new_owner): 543 'Creates a new job identical to this one' 544 hosts = [queue_entry.meta_host or queue_entry.host 545 for queue_entry 546 in self.hostqueueentry_set.filter(deleted=False)] 547 new_job = Job.create( 548 owner=new_owner, name=self.name, priority=self.priority, 549 control_file=self.control_file, 550 control_type=self.control_type, hosts=hosts, 551 synch_type=self.synch_type, timeout=self.timeout, 552 run_verify=self.run_verify) 553 new_job.queue(hosts) 554 return new_job 555 556 557 def abort(self): 558 user = thread_local.get_user() 559 if not user.is_superuser() and user.login != self.owner: 560 raise AclAccessViolation("You cannot abort other people's jobs!") 561 for queue_entry in self.hostqueueentry_set.all(): 562 queue_entry.abort() 563 564 565 def user(self): 566 try: 567 return User.objects.get(login=self.owner) 568 except self.DoesNotExist: 569 return None 570 571 572 class Meta: 573 db_table = 'jobs' 574 575 if settings.FULL_ADMIN: 576 class Admin: 577 list_display = ('id', 'owner', 'name', 'control_type') 578 579 def __str__(self): 580 return '%s (%s-%s)' % (self.name, self.id, self.owner) 581 582 583class IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions): 584 job = dbmodels.ForeignKey(Job) 585 host = dbmodels.ForeignKey(Host) 586 587 objects = model_logic.ExtendedManager() 588 589 class Meta: 590 db_table = 'ineligible_host_queues' 591 592 if settings.FULL_ADMIN: 593 class Admin: 594 list_display = ('id', 'job', 'host') 595 596 597class HostQueueEntry(dbmodels.Model, model_logic.ModelExtensions): 598 job = dbmodels.ForeignKey(Job) 599 host = dbmodels.ForeignKey(Host, blank=True, null=True) 600 priority = dbmodels.SmallIntegerField() 601 status = dbmodels.CharField(maxlength=255) 602 meta_host = dbmodels.ForeignKey(Label, blank=True, null=True, 603 db_column='meta_host') 604 active = dbmodels.BooleanField(default=False) 605 complete = dbmodels.BooleanField(default=False) 606 deleted = dbmodels.BooleanField(default=False) 607 608 objects = model_logic.ExtendedManager() 609 610 611 def is_meta_host_entry(self): 612 'True if this is a entry has a meta_host instead of a host.' 613 return self.host is None and self.meta_host is not None 614 615 def abort(self): 616 if self.active: 617 self.status = Job.Status.ABORT 618 elif not self.complete: 619 self.status = Job.Status.ABORTED 620 self.active = False 621 self.complete = True 622 self.save() 623 624 class Meta: 625 db_table = 'host_queue_entries' 626 627 if settings.FULL_ADMIN: 628 class Admin: 629 list_display = ('id', 'job', 'host', 'status', 630 'meta_host') 631