models.py revision 9976ce9873867a397e448d358543a9dc1d33aa77
1from datetime import datetime 2from django.db import models as dbmodels, connection 3from frontend.afe import model_logic 4from frontend import settings, thread_local 5from autotest_lib.client.common_lib import enum, host_protections, global_config 6 7 8class AclAccessViolation(Exception): 9 """\ 10 Raised when an operation is attempted with proper permissions as 11 dictated by ACLs. 12 """ 13 14 15class Label(model_logic.ModelWithInvalid, dbmodels.Model): 16 """\ 17 Required: 18 name: label name 19 20 Optional: 21 kernel_config: url/path to kernel config to use for jobs run on this 22 label 23 platform: if True, this is a platform label (defaults to False) 24 only_if_needed: if True, a machine with this label can only be used if that 25 that label is requested by the job/test. 26 """ 27 name = dbmodels.CharField(maxlength=255, unique=True) 28 kernel_config = dbmodels.CharField(maxlength=255, blank=True) 29 platform = dbmodels.BooleanField(default=False) 30 invalid = dbmodels.BooleanField(default=False, 31 editable=settings.FULL_ADMIN) 32 only_if_needed = dbmodels.BooleanField(default=False) 33 34 name_field = 'name' 35 objects = model_logic.ExtendedManager() 36 valid_objects = model_logic.ValidObjectsManager() 37 38 def clean_object(self): 39 self.host_set.clear() 40 41 42 def enqueue_job(self, job): 43 'Enqueue a job on any host of this label.' 44 queue_entry = HostQueueEntry(meta_host=self, job=job, 45 status=Job.Status.QUEUED, 46 priority=job.priority) 47 queue_entry.save() 48 49 50 class Meta: 51 db_table = 'labels' 52 53 class Admin: 54 list_display = ('name', 'kernel_config') 55 # see Host.Admin 56 manager = model_logic.ValidObjectsManager() 57 58 def __str__(self): 59 return self.name 60 61 62class User(dbmodels.Model, model_logic.ModelExtensions): 63 """\ 64 Required: 65 login :user login name 66 67 Optional: 68 access_level: 0=User (default), 1=Admin, 100=Root 69 """ 70 ACCESS_ROOT = 100 71 ACCESS_ADMIN = 1 72 ACCESS_USER = 0 73 74 login = dbmodels.CharField(maxlength=255, unique=True) 75 access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True) 76 77 name_field = 'login' 78 objects = model_logic.ExtendedManager() 79 80 81 def save(self): 82 # is this a new object being saved for the first time? 83 first_time = (self.id is None) 84 user = thread_local.get_user() 85 if user and not user.is_superuser(): 86 raise AclAccessViolation("You cannot modify users!") 87 super(User, self).save() 88 if first_time: 89 everyone = AclGroup.objects.get(name='Everyone') 90 everyone.users.add(self) 91 92 93 def is_superuser(self): 94 return self.access_level >= self.ACCESS_ROOT 95 96 97 class Meta: 98 db_table = 'users' 99 100 class Admin: 101 list_display = ('login', 'access_level') 102 search_fields = ('login',) 103 104 def __str__(self): 105 return self.login 106 107 108class Host(model_logic.ModelWithInvalid, dbmodels.Model): 109 """\ 110 Required: 111 hostname 112 113 optional: 114 locked: host is locked and will not be queued 115 116 Internal: 117 synch_id: currently unused 118 status: string describing status of host 119 """ 120 Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing', 121 'Repair Failed', 'Dead', 'Rebooting', 122 string_values=True) 123 124 hostname = dbmodels.CharField(maxlength=255, unique=True) 125 labels = dbmodels.ManyToManyField(Label, blank=True, 126 filter_interface=dbmodels.HORIZONTAL) 127 locked = dbmodels.BooleanField(default=False) 128 synch_id = dbmodels.IntegerField(blank=True, null=True, 129 editable=settings.FULL_ADMIN) 130 status = dbmodels.CharField(maxlength=255, default=Status.READY, 131 choices=Status.choices(), 132 editable=settings.FULL_ADMIN) 133 invalid = dbmodels.BooleanField(default=False, 134 editable=settings.FULL_ADMIN) 135 protection = dbmodels.SmallIntegerField(null=False, blank=True, 136 choices=host_protections.choices, 137 default=host_protections.default) 138 locked_by = dbmodels.ForeignKey(User, null=True, blank=True, editable=False) 139 lock_time = dbmodels.DateTimeField(null=True, blank=True, editable=False) 140 141 name_field = 'hostname' 142 objects = model_logic.ExtendedManager() 143 valid_objects = model_logic.ValidObjectsManager() 144 145 @staticmethod 146 def create_one_time_host(hostname): 147 query = Host.objects.filter(hostname=hostname) 148 if query.count() == 0: 149 host = Host(hostname=hostname, invalid=True) 150 host.do_validate() 151 else: 152 host = query[0] 153 if not host.invalid: 154 raise model_logic.ValidationError({ 155 'hostname' : '%s already exists!' % hostname 156 }) 157 host.clean_object() 158 AclGroup.objects.get(name='Everyone').hosts.add(host) 159 host.status = Host.Status.READY 160 host.protection = host_protections.Protection.DO_NOT_REPAIR 161 host.save() 162 return host 163 164 def clean_object(self): 165 self.aclgroup_set.clear() 166 self.labels.clear() 167 168 169 def save(self): 170 # extra spaces in the hostname can be a sneaky source of errors 171 self.hostname = self.hostname.strip() 172 # is this a new object being saved for the first time? 173 first_time = (self.id is None) 174 if not first_time: 175 AclGroup.check_for_acl_violation_hosts([self]) 176 if self.locked and not self.locked_by: 177 self.locked_by = thread_local.get_user() 178 self.lock_time = datetime.now() 179 elif not self.locked and self.locked_by: 180 self.locked_by = None 181 self.lock_time = None 182 super(Host, self).save() 183 if first_time: 184 everyone = AclGroup.objects.get(name='Everyone') 185 everyone.hosts.add(self) 186 187 def delete(self): 188 AclGroup.check_for_acl_violation_hosts([self]) 189 for queue_entry in self.hostqueueentry_set.all(): 190 queue_entry.deleted = True 191 queue_entry.abort(thread_local.get_user()) 192 super(Host, self).delete() 193 194 195 def enqueue_job(self, job): 196 ' Enqueue a job on this host.' 197 queue_entry = HostQueueEntry(host=self, job=job, 198 status=Job.Status.QUEUED, 199 priority=job.priority) 200 # allow recovery of dead hosts from the frontend 201 if not self.active_queue_entry() and self.is_dead(): 202 self.status = Host.Status.READY 203 self.save() 204 queue_entry.save() 205 206 block = IneligibleHostQueue(job=job, host=self) 207 block.save() 208 209 210 def platform(self): 211 # TODO(showard): slighly hacky? 212 platforms = self.labels.filter(platform=True) 213 if len(platforms) == 0: 214 return None 215 return platforms[0] 216 platform.short_description = 'Platform' 217 218 219 def is_dead(self): 220 return self.status == Host.Status.REPAIR_FAILED 221 222 223 def active_queue_entry(self): 224 active = list(self.hostqueueentry_set.filter(active=True)) 225 if not active: 226 return None 227 assert len(active) == 1, ('More than one active entry for ' 228 'host ' + self.hostname) 229 return active[0] 230 231 232 class Meta: 233 db_table = 'hosts' 234 235 class Admin: 236 # TODO(showard) - showing platform requires a SQL query for 237 # each row (since labels are many-to-many) - should we remove 238 # it? 239 list_display = ('hostname', 'platform', 'locked', 'status') 240 list_filter = ('labels', 'locked', 'protection') 241 search_fields = ('hostname', 'status') 242 # undocumented Django feature - if you set manager here, the 243 # admin code will use it, otherwise it'll use a default Manager 244 manager = model_logic.ValidObjectsManager() 245 246 def __str__(self): 247 return self.hostname 248 249 250class Test(dbmodels.Model, model_logic.ModelExtensions): 251 """\ 252 Required: 253 author: author name 254 description: description of the test 255 name: test name 256 time: short, medium, long 257 test_class: This describes the class for your the test belongs in. 258 test_category: This describes the category for your tests 259 test_type: Client or Server 260 path: path to pass to run_test() 261 synch_type: whether the test should run synchronously or asynchronously 262 sync_count: is a number >=1 (1 being the default). If it's 1, then it's an 263 async job. If it's >1 it's sync job for that number of machines 264 i.e. if sync_count = 2 it is a sync job that requires two 265 machines. 266 Optional: 267 dependencies: What the test requires to run. Comma deliminated list 268 dependency_labels: many-to-many relationship with labels corresponding to 269 test dependencies. 270 experimental: If this is set to True production servers will ignore the test 271 run_verify: Whether or not the scheduler should run the verify stage 272 """ 273 TestTime = enum.Enum('SHORT', 'MEDIUM', 'LONG', start_value=1) 274 SynchType = enum.Enum('Asynchronous', 'Synchronous', start_value=1) 275 # TODO(showard) - this should be merged with Job.ControlType (but right 276 # now they use opposite values) 277 Types = enum.Enum('Client', 'Server', start_value=1) 278 279 name = dbmodels.CharField(maxlength=255, unique=True) 280 author = dbmodels.CharField(maxlength=255) 281 test_class = dbmodels.CharField(maxlength=255) 282 test_category = dbmodels.CharField(maxlength=255) 283 dependencies = dbmodels.CharField(maxlength=255, blank=True) 284 description = dbmodels.TextField(blank=True) 285 experimental = dbmodels.BooleanField(default=True) 286 run_verify = dbmodels.BooleanField(default=True) 287 test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(), 288 default=TestTime.MEDIUM) 289 test_type = dbmodels.SmallIntegerField(choices=Types.choices()) 290 sync_count = dbmodels.IntegerField(default=1) 291 synch_type = dbmodels.SmallIntegerField(choices=SynchType.choices(), 292 default=SynchType.ASYNCHRONOUS) 293 path = dbmodels.CharField(maxlength=255, unique=True) 294 dependency_labels = dbmodels.ManyToManyField( 295 Label, blank=True, filter_interface=dbmodels.HORIZONTAL) 296 297 name_field = 'name' 298 objects = model_logic.ExtendedManager() 299 300 301 class Meta: 302 db_table = 'autotests' 303 304 class Admin: 305 fields = ( 306 (None, {'fields' : 307 ('name', 'author', 'test_category', 'test_class', 308 'test_time', 'synch_type', 'test_type', 'sync_count', 309 'path', 'dependencies', 'experimental', 'run_verify', 310 'description')}), 311 ) 312 list_display = ('name', 'test_type', 'description', 'synch_type') 313 search_fields = ('name',) 314 315 def __str__(self): 316 return self.name 317 318 319class Profiler(dbmodels.Model, model_logic.ModelExtensions): 320 """\ 321 Required: 322 name: profiler name 323 test_type: Client or Server 324 325 Optional: 326 description: arbirary text description 327 """ 328 name = dbmodels.CharField(maxlength=255, unique=True) 329 description = dbmodels.TextField(blank=True) 330 331 name_field = 'name' 332 objects = model_logic.ExtendedManager() 333 334 335 class Meta: 336 db_table = 'profilers' 337 338 class Admin: 339 list_display = ('name', 'description') 340 search_fields = ('name',) 341 342 def __str__(self): 343 return self.name 344 345 346class AclGroup(dbmodels.Model, model_logic.ModelExtensions): 347 """\ 348 Required: 349 name: name of ACL group 350 351 Optional: 352 description: arbitrary description of group 353 """ 354 name = dbmodels.CharField(maxlength=255, unique=True) 355 description = dbmodels.CharField(maxlength=255, blank=True) 356 users = dbmodels.ManyToManyField(User, blank=True, 357 filter_interface=dbmodels.HORIZONTAL) 358 hosts = dbmodels.ManyToManyField(Host, 359 filter_interface=dbmodels.HORIZONTAL) 360 361 name_field = 'name' 362 objects = model_logic.ExtendedManager() 363 364 @staticmethod 365 def check_for_acl_violation_hosts(hosts): 366 user = thread_local.get_user() 367 if user.is_superuser(): 368 return 369 accessible_host_ids = set( 370 host.id for host in Host.objects.filter(acl_group__users=user)) 371 for host in hosts: 372 # Check if the user has access to this host, 373 # but only if it is not a metahost 374 if (isinstance(host, Host) 375 and int(host.id) not in accessible_host_ids): 376 raise AclAccessViolation("You do not have access to %s" 377 % str(host)) 378 379 380 @staticmethod 381 def check_for_acl_violation_queue_entries(queue_entries): 382 user = thread_local.get_user() 383 if user.is_superuser(): 384 return 385 for queue_entry in queue_entries: 386 job = queue_entry.job 387 if user.login != job.owner: 388 raise AclAccessViolation('You cannot abort job %d owned by %s' % 389 (job.id, job.owner)) 390 391 392 def check_for_acl_violation_acl_group(self): 393 user = thread_local.get_user() 394 if user.is_superuser(): 395 return None 396 if not user in self.users.all(): 397 raise AclAccessViolation("You do not have access to %s" 398 % self.name) 399 400 @staticmethod 401 def on_host_membership_change(): 402 everyone = AclGroup.objects.get(name='Everyone') 403 404 # find hosts that aren't in any ACL group and add them to Everyone 405 # TODO(showard): this is a bit of a hack, since the fact that this query 406 # works is kind of a coincidence of Django internals. This trick 407 # doesn't work in general (on all foreign key relationships). I'll 408 # replace it with a better technique when the need arises. 409 orphaned_hosts = Host.valid_objects.filter(acl_group__id__isnull=True) 410 everyone.hosts.add(*orphaned_hosts.distinct()) 411 412 # find hosts in both Everyone and another ACL group, and remove them 413 # from Everyone 414 hosts_in_everyone = Host.valid_objects.filter_custom_join( 415 '_everyone', acl_group__name='Everyone') 416 acled_hosts = hosts_in_everyone.exclude(acl_group__name='Everyone') 417 everyone.hosts.remove(*acled_hosts.distinct()) 418 419 420 def delete(self): 421 if (self.name == 'Everyone'): 422 raise AclAccessViolation("You cannot delete 'Everyone'!") 423 self.check_for_acl_violation_acl_group() 424 super(AclGroup, self).delete() 425 self.on_host_membership_change() 426 427 428 def add_current_user_if_empty(self): 429 if not self.users.count(): 430 self.users.add(thread_local.get_user()) 431 432 433 # if you have a model attribute called "Manipulator", Django will 434 # automatically insert it into the beginning of the superclass list 435 # for the model's manipulators 436 class Manipulator(object): 437 """ 438 Custom manipulator to get notification when ACLs are changed through 439 the admin interface. 440 """ 441 def save(self, new_data): 442 user = thread_local.get_user() 443 if hasattr(self, 'original_object'): 444 if (not user.is_superuser() 445 and self.original_object.name == 'Everyone'): 446 raise AclAccessViolation("You cannot modify 'Everyone'!") 447 self.original_object.check_for_acl_violation_acl_group() 448 obj = super(AclGroup.Manipulator, self).save(new_data) 449 if not hasattr(self, 'original_object'): 450 obj.users.add(thread_local.get_user()) 451 obj.add_current_user_if_empty() 452 obj.on_host_membership_change() 453 return obj 454 455 class Meta: 456 db_table = 'acl_groups' 457 458 class Admin: 459 list_display = ('name', 'description') 460 search_fields = ('name',) 461 462 def __str__(self): 463 return self.name 464 465# hack to make the column name in the many-to-many DB tables match the one 466# generated by ruby 467AclGroup._meta.object_name = 'acl_group' 468 469 470class JobManager(model_logic.ExtendedManager): 471 'Custom manager to provide efficient status counts querying.' 472 def get_status_counts(self, job_ids): 473 """\ 474 Returns a dictionary mapping the given job IDs to their status 475 count dictionaries. 476 """ 477 if not job_ids: 478 return {} 479 id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids) 480 cursor = connection.cursor() 481 cursor.execute(""" 482 SELECT job_id, status, COUNT(*) 483 FROM host_queue_entries 484 WHERE job_id IN %s 485 GROUP BY job_id, status 486 """ % id_list) 487 all_job_counts = {} 488 for job_id in job_ids: 489 all_job_counts[job_id] = {} 490 for job_id, status, count in cursor.fetchall(): 491 all_job_counts[job_id][status] = count 492 return all_job_counts 493 494 495 def populate_dependencies(self, jobs): 496 if not jobs: 497 return 498 job_ids = ','.join(str(job['id']) for job in jobs) 499 cursor = connection.cursor() 500 cursor.execute(""" 501 SELECT jobs.id, labels.name 502 FROM jobs 503 INNER JOIN jobs_dependency_labels 504 ON jobs.id = jobs_dependency_labels.job_id 505 INNER JOIN labels ON jobs_dependency_labels.label_id = labels.id 506 WHERE jobs.id IN (%s) 507 """ % job_ids) 508 job_dependencies = {} 509 for job_id, dependency in cursor.fetchall(): 510 job_dependencies.setdefault(job_id, []).append(dependency) 511 for job in jobs: 512 dependencies = ','.join(job_dependencies.get(job['id'], [])) 513 job['dependencies'] = dependencies 514 515 516class Job(dbmodels.Model, model_logic.ModelExtensions): 517 """\ 518 owner: username of job owner 519 name: job name (does not have to be unique) 520 priority: Low, Medium, High, Urgent (or 0-3) 521 control_file: contents of control file 522 control_type: Client or Server 523 created_on: date of job creation 524 submitted_on: date of job submission 525 synch_type: Asynchronous or Synchronous (i.e. job must run on all hosts 526 simultaneously; used for server-side control files) 527 synch_count: ??? 528 run_verify: Whether or not to run the verify phase 529 synchronizing: for scheduler use 530 timeout: hours until job times out 531 email_list: list of people to email on completion delimited by any of: 532 white space, ',', ':', ';' 533 dependency_labels: many-to-many relationship with labels corresponding to 534 job dependencies 535 """ 536 DEFAULT_TIMEOUT = global_config.global_config.get_config_value( 537 'AUTOTEST_WEB', 'job_timeout_default', default=240) 538 539 Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent') 540 ControlType = enum.Enum('Server', 'Client', start_value=1) 541 Status = enum.Enum('Created', 'Queued', 'Pending', 'Running', 542 'Completed', 'Abort', 'Aborting', 'Aborted', 543 'Failed', 'Starting', string_values=True) 544 545 owner = dbmodels.CharField(maxlength=255) 546 name = dbmodels.CharField(maxlength=255) 547 priority = dbmodels.SmallIntegerField(choices=Priority.choices(), 548 blank=True, # to allow 0 549 default=Priority.MEDIUM) 550 control_file = dbmodels.TextField() 551 control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(), 552 blank=True, # to allow 0 553 default=ControlType.CLIENT) 554 created_on = dbmodels.DateTimeField() 555 synch_type = dbmodels.SmallIntegerField( 556 blank=True, null=True, choices=Test.SynchType.choices()) 557 synch_count = dbmodels.IntegerField(blank=True, null=True) 558 synchronizing = dbmodels.BooleanField(default=False) 559 timeout = dbmodels.IntegerField(default=DEFAULT_TIMEOUT) 560 run_verify = dbmodels.BooleanField(default=True) 561 email_list = dbmodels.CharField(maxlength=250, blank=True) 562 dependency_labels = dbmodels.ManyToManyField( 563 Label, blank=True, filter_interface=dbmodels.HORIZONTAL) 564 565 566 # custom manager 567 objects = JobManager() 568 569 570 def is_server_job(self): 571 return self.control_type == self.ControlType.SERVER 572 573 574 @classmethod 575 def create(cls, owner, name, priority, control_file, control_type, 576 hosts, synch_type, timeout, run_verify, email_list, 577 dependencies): 578 """\ 579 Creates a job by taking some information (the listed args) 580 and filling in the rest of the necessary information. 581 """ 582 AclGroup.check_for_acl_violation_hosts(hosts) 583 job = cls.add_object( 584 owner=owner, name=name, priority=priority, 585 control_file=control_file, control_type=control_type, 586 synch_type=synch_type, timeout=timeout, 587 run_verify=run_verify, email_list=email_list, 588 created_on=datetime.now()) 589 590 if job.synch_type == Test.SynchType.SYNCHRONOUS: 591 job.synch_count = len(hosts) 592 else: 593 if len(hosts) == 0: 594 errors = {'hosts': 595 'asynchronous jobs require at least' 596 + ' one host to run on'} 597 raise model_logic.ValidationError(errors) 598 job.save() 599 job.dependency_labels = dependencies 600 return job 601 602 603 def queue(self, hosts): 604 'Enqueue a job on the given hosts.' 605 for host in hosts: 606 host.enqueue_job(self) 607 608 609 def user(self): 610 try: 611 return User.objects.get(login=self.owner) 612 except self.DoesNotExist: 613 return None 614 615 616 class Meta: 617 db_table = 'jobs' 618 619 if settings.FULL_ADMIN: 620 class Admin: 621 list_display = ('id', 'owner', 'name', 'control_type') 622 623 def __str__(self): 624 return '%s (%s-%s)' % (self.name, self.id, self.owner) 625 626 627class IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions): 628 job = dbmodels.ForeignKey(Job) 629 host = dbmodels.ForeignKey(Host) 630 631 objects = model_logic.ExtendedManager() 632 633 class Meta: 634 db_table = 'ineligible_host_queues' 635 636 if settings.FULL_ADMIN: 637 class Admin: 638 list_display = ('id', 'job', 'host') 639 640 641class HostQueueEntry(dbmodels.Model, model_logic.ModelExtensions): 642 job = dbmodels.ForeignKey(Job) 643 host = dbmodels.ForeignKey(Host, blank=True, null=True) 644 priority = dbmodels.SmallIntegerField() 645 status = dbmodels.CharField(maxlength=255) 646 meta_host = dbmodels.ForeignKey(Label, blank=True, null=True, 647 db_column='meta_host') 648 active = dbmodels.BooleanField(default=False) 649 complete = dbmodels.BooleanField(default=False) 650 deleted = dbmodels.BooleanField(default=False) 651 652 objects = model_logic.ExtendedManager() 653 654 655 def is_meta_host_entry(self): 656 'True if this is a entry has a meta_host instead of a host.' 657 return self.host is None and self.meta_host is not None 658 659 def log_abort(self, user): 660 abort_log = AbortedHostQueueEntry(queue_entry=self, aborted_by=user) 661 abort_log.save() 662 663 def abort(self, user): 664 if self.active: 665 self.status = Job.Status.ABORT 666 self.log_abort(user) 667 elif not self.complete: 668 self.status = Job.Status.ABORTED 669 self.active = False 670 self.complete = True 671 self.log_abort(user) 672 self.save() 673 674 class Meta: 675 db_table = 'host_queue_entries' 676 677 if settings.FULL_ADMIN: 678 class Admin: 679 list_display = ('id', 'job', 'host', 'status', 680 'meta_host') 681 682 683class AbortedHostQueueEntry(dbmodels.Model, model_logic.ModelExtensions): 684 queue_entry = dbmodels.OneToOneField(HostQueueEntry, primary_key=True) 685 aborted_by = dbmodels.ForeignKey(User) 686 aborted_on = dbmodels.DateTimeField() 687 688 objects = model_logic.ExtendedManager() 689 690 691 def save(self): 692 self.aborted_on = datetime.now() 693 super(AbortedHostQueueEntry, self).save() 694 695 class Meta: 696 db_table = 'aborted_host_queue_entries' 697