models.py revision 1385b161fa9391242a2121027878e4943ce3c81f
1import datetime 2from django.db import models as dbmodels 3from frontend.afe import enum 4from frontend import settings 5 6 7class ValidationError(Exception): 8 """\ 9 Data validation error in adding or updating an object. The associated 10 value is a dictionary mapping field names to error strings. 11 """ 12 13 14class AclAccessViolation(Exception): 15 """\ 16 Raised when an operation is attempted with proper permissions as 17 dictated by ACLs. 18 """ 19 20 21class ModelExtensions(object): 22 """\ 23 Mixin with convenience functions for models, built on top of the 24 default Django model functions. 25 """ 26 # TODO: at least some of these functions really belong in a custom 27 # Manager class 28 29 field_dict = None 30 # subclasses should override if they want to support smart_get() by name 31 name_field = None 32 33 34 @classmethod 35 def get_field_dict(cls): 36 if cls.field_dict is None: 37 cls.field_dict = {} 38 for field in cls._meta.fields: 39 cls.field_dict[field.name] = field 40 return cls.field_dict 41 42 43 @classmethod 44 def clean_foreign_keys(cls, data): 45 """\ 46 Convert foreign key fields in data from <field>_id to just 47 <field> 48 """ 49 for field in cls._meta.fields: 50 if field.rel: 51 data[field.name] = data[field.column] 52 del data[field.column] 53 54 55 # TODO(showard) - is there a way to not have to do this? 56 @classmethod 57 def provide_default_values(cls, data): 58 """\ 59 Provide default values for fields with default values which have 60 nothing passed in. 61 62 For CharField and TextField fields with "blank=True", if nothing 63 is passed, we fill in an empty string value, even if there's no 64 default set. 65 """ 66 new_data = dict(data) 67 field_dict = cls.get_field_dict() 68 for name, obj in field_dict.iteritems(): 69 if data.get(name) is not None: 70 continue 71 if obj.default is not dbmodels.fields.NOT_PROVIDED: 72 new_data[name] = obj.default 73 elif (isinstance(obj, dbmodels.CharField) or 74 isinstance(obj, dbmodels.TextField)): 75 new_data[name] = '' 76 return new_data 77 78 79 @classmethod 80 def convert_human_readable_values(cls, data, to_human_readable=False): 81 """\ 82 Performs conversions on user-supplied field data, to make it 83 easier for users to pass human-readable data. 84 85 For all fields that have choice sets, convert their values 86 from human-readable strings to enum values, if necessary. This 87 allows users to pass strings instead of the corresponding 88 integer values. 89 90 For all foreign key fields, call smart_get with the supplied 91 data. This allows the user to pass either an ID value or 92 the name of the object as a string. 93 94 If to_human_readable=True, perform the inverse - i.e. convert 95 numeric values to human readable values. 96 """ 97 new_data = dict(data) 98 field_dict = cls.get_field_dict() 99 for field_name in data: 100 field_obj = field_dict[field_name] 101 # convert enum values 102 if field_obj.choices: 103 for choice_data in field_obj.choices: 104 # choice_data is (value, name) 105 if to_human_readable: 106 from_val, to_val = choice_data 107 else: 108 to_val, from_val = choice_data 109 if from_val == data[field_name]: 110 new_data[field_name] = to_val 111 break 112 # convert foreign key values 113 elif field_obj.rel: 114 dest_obj = field_obj.rel.to.smart_get( 115 data[field_name]) 116 if to_human_readable: 117 new_data[field_name] = ( 118 getattr(dest_obj, 119 dest_obj.name_field)) 120 else: 121 new_data[field_name] = dest_obj.id 122 return new_data 123 124 125 @classmethod 126 def validate_field_names(cls, data): 127 'Checks for extraneous fields in data.' 128 errors = {} 129 field_dict = cls.get_field_dict() 130 for field_name in data: 131 if field_name not in field_dict: 132 errors[field_name] = 'No field of this name' 133 return errors 134 135 136 @classmethod 137 def prepare_data_args(cls, data, kwargs): 138 'Common preparation for add_object and update_object' 139 data = dict(data) # don't modify the default keyword arg 140 data.update(kwargs) 141 # must check for extraneous field names here, while we have the 142 # data in a dict 143 errors = cls.validate_field_names(data) 144 if errors: 145 raise ValidationError(errors) 146 data = cls.convert_human_readable_values(data) 147 return data 148 149 150 def validate_unique(self): 151 """\ 152 Validate that unique fields are unique. Django manipulators do 153 this too, but they're a huge pain to use manually. Trust me. 154 """ 155 errors = {} 156 cls = type(self) 157 field_dict = self.get_field_dict() 158 for field_name, field_obj in field_dict.iteritems(): 159 if not field_obj.unique: 160 continue 161 162 value = getattr(self, field_name) 163 existing_objs = cls.objects.filter( 164 **{field_name : value}) 165 num_existing = existing_objs.count() 166 167 if num_existing == 0: 168 continue 169 if num_existing == 1 and existing_objs[0].id == self.id: 170 continue 171 errors[field_name] = ( 172 'This value must be unique (%s)' % (value)) 173 return errors 174 175 176 def do_validate(self): 177 errors = self.validate() 178 unique_errors = self.validate_unique() 179 for field_name, error in unique_errors.iteritems(): 180 errors.setdefault(field_name, error) 181 if errors: 182 raise ValidationError(errors) 183 184 185 # actually (externally) useful methods follow 186 187 @classmethod 188 def add_object(cls, data={}, **kwargs): 189 """\ 190 Returns a new object created with the given data (a dictionary 191 mapping field names to values). Merges any extra keyword args 192 into data. 193 """ 194 data = cls.prepare_data_args(data, kwargs) 195 data = cls.provide_default_values(data) 196 obj = cls(**data) 197 obj.do_validate() 198 obj.save() 199 return obj 200 201 202 def update_object(self, data={}, **kwargs): 203 """\ 204 Updates the object with the given data (a dictionary mapping 205 field names to values). Merges any extra keyword args into 206 data. 207 """ 208 data = self.prepare_data_args(data, kwargs) 209 for field_name, value in data.iteritems(): 210 if value is not None: 211 setattr(self, field_name, value) 212 self.do_validate() 213 self.save() 214 215 216 @classmethod 217 def query_objects(cls, filter_data): 218 """\ 219 Returns a QuerySet object for querying the given model_class 220 with the given filter_data. Optional special arguments in 221 filter_data include: 222 -query_start: index of first return to return 223 -query_limit: maximum number of results to return 224 -sort_by: name of field to sort on. prefixing a '-' onto this 225 argument changes the sort to descending order. 226 -extra_args: keyword args to pass to query.extra() (see Django 227 DB layer documentation) 228 """ 229 query_start = filter_data.pop('query_start', None) 230 query_limit = filter_data.pop('query_limit', None) 231 if query_start and not query_limit: 232 raise ValueError('Cannot pass query_start without ' 233 'query_limit') 234 sort_by = filter_data.pop('sort_by', None) 235 extra_args = filter_data.pop('extra_args', None) 236 query_dict = {} 237 for field, value in filter_data.iteritems(): 238 query_dict[field] = value 239 query = cls.objects.filter(**query_dict) 240 if extra_args: 241 query = query.extra(**extra_args) 242 if sort_by: 243 query = query.order_by(sort_by) 244 if query_start is not None and query_limit is not None: 245 query_limit += query_start 246 return query[query_start:query_limit] 247 248 249 @classmethod 250 def clean_object_dicts(cls, field_dicts): 251 """\ 252 Take a list of dicts corresponding to object (as returned by 253 query.values()) and clean the data to be more suitable for 254 returning to the user. 255 """ 256 for i in range(len(field_dicts)): 257 cls.clean_foreign_keys(field_dicts[i]) 258 field_dicts[i] = cls.convert_human_readable_values( 259 field_dicts[i], to_human_readable=True) 260 261 @classmethod 262 def list_objects(cls, filter_data): 263 """\ 264 Like query_objects, but return a list of dictionaries. 265 """ 266 query = cls.query_objects(filter_data) 267 field_dicts = list(query.values()) 268 cls.clean_object_dicts(field_dicts) 269 return field_dicts 270 271 272 @classmethod 273 def smart_get(cls, *args, **kwargs): 274 """\ 275 smart_get(integer) -> get object by ID 276 smart_get(string) -> get object by name_field 277 smart_get(keyword args) -> normal ModelClass.objects.get() 278 """ 279 assert bool(args) ^ bool(kwargs) 280 if args: 281 assert len(args) == 1 282 arg = args[0] 283 if isinstance(arg, int) or isinstance(arg, long): 284 return cls.objects.get(id=arg) 285 if isinstance(arg, str) or isinstance(arg, unicode): 286 return cls.objects.get( 287 **{cls.name_field : arg}) 288 raise ValueError( 289 'Invalid positional argument: %s (%s)' % ( 290 str(arg), type(arg))) 291 return cls.objects.get(**kwargs) 292 293 294 def get_object_dict(self): 295 """\ 296 Return a dictionary mapping fields to this object's values. 297 """ 298 return dict((field_name, getattr(self, field_name)) 299 for field_name in self.get_field_dict().iterkeys()) 300 301 302class Label(dbmodels.Model, ModelExtensions): 303 """\ 304 Required: 305 name: label name 306 307 Optional: 308 kernel_config: url/path to kernel config to use for jobs run on this 309 label 310 platform: if True, this is a platform label (defaults to False) 311 """ 312 name = dbmodels.CharField(maxlength=255, unique=True) 313 kernel_config = dbmodels.CharField(maxlength=255, blank=True) 314 platform = dbmodels.BooleanField(default=False) 315 316 name_field = 'name' 317 318 319 def enqueue_job(self, job): 320 'Enqueue a job on any host of this label.' 321 queue_entry = HostQueueEntry(meta_host=self, job=job, 322 status=Job.Status.QUEUED, 323 priority=job.priority) 324 queue_entry.save() 325 326 327 def block_auto_assign(self, job): 328 """\ 329 Placeholder to allow labels to be used in place of hosts 330 (as meta-hosts). 331 """ 332 pass 333 334 335 class Meta: 336 db_table = 'labels' 337 338 class Admin: 339 list_display = ('name', 'kernel_config') 340 341 def __str__(self): 342 return self.name 343 344 345class Host(dbmodels.Model, ModelExtensions): 346 """\ 347 Required: 348 hostname 349 350 optional: 351 locked: host is locked and will not be queued 352 353 Internal: 354 synch_id: currently unused 355 status: string describing status of host 356 """ 357 Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing', 358 'Repair Failed', 'Dead', string_values=True) 359 360 hostname = dbmodels.CharField(maxlength=255, unique=True) 361 labels = dbmodels.ManyToManyField(Label, blank=True, 362 filter_interface=dbmodels.HORIZONTAL) 363 locked = dbmodels.BooleanField(default=False) 364 synch_id = dbmodels.IntegerField(blank=True, null=True) 365 status = dbmodels.CharField(maxlength=255, default=Status.READY, 366 choices=Status.choices()) 367 368 name_field = 'hostname' 369 370 371 def save(self): 372 # is this a new object being saved for the first time? 373 first_time = (self.id is None) 374 super(Host, self).save() 375 if first_time: 376 everyone = AclGroup.objects.get(name='Everyone') 377 everyone.hosts.add(self) 378 379 380 def enqueue_job(self, job): 381 ' Enqueue a job on this host.' 382 queue_entry = HostQueueEntry(host=self, job=job, 383 status=Job.Status.QUEUED, 384 priority=job.priority) 385 # allow recovery of dead hosts from the frontend 386 if not self.active_queue_entry() and self.is_dead(): 387 self.status = Host.Status.READY 388 self.save() 389 queue_entry.save() 390 391 392 def block_auto_assign(self, job): 393 'Block this host from being assigned to a job.' 394 block = IneligibleHostQueue(job=job, host=self) 395 block.save() 396 397 398 def platform(self): 399 # TODO(showard): slighly hacky? 400 platforms = self.labels.filter(platform=True) 401 if len(platforms) == 0: 402 return None 403 return platforms[0] 404 platform.short_description = 'Platform' 405 406 407 def is_dead(self): 408 return self.status == Host.Status.REPAIR_FAILED 409 410 411 def active_queue_entry(self): 412 active = list(self.hostqueueentry_set.filter(active=True)) 413 if not active: 414 return None 415 assert len(active) == 1, ('More than one active entry for ' 416 'host ' + self.hostname) 417 return active[0] 418 419 420 class Meta: 421 db_table = 'hosts' 422 423 class Admin: 424 # TODO(showard) - showing platform requires a SQL query for 425 # each row (since labels are many-to-many) - should we remove 426 # it? 427 if not settings.FULL_ADMIN: 428 fields = ( 429 (None, {'fields' : 430 ('hostname', 'status', 'locked', 431 'labels')}), 432 ) 433 list_display = ('hostname', 'platform', 'locked', 'status') 434 list_filter = ('labels', 'locked') 435 search_fields = ('hostname', 'status') 436 437 def __str__(self): 438 return self.hostname 439 440 441class Test(dbmodels.Model, ModelExtensions): 442 """\ 443 Required: 444 name: test name 445 test_type: Client or Server 446 path: path to pass to run_test() 447 synch_type: whether the test should run synchronously or asynchronously 448 449 Optional: 450 test_class: used for categorization of tests 451 description: arbirary text description 452 """ 453 Classes = enum.Enum('Kernel', 'Hardware', 'Canned Test Sets', 454 string_values=True) 455 SynchType = enum.Enum('Asynchronous', 'Synchronous', start_value=1) 456 # TODO(showard) - this should be merged with Job.ControlType (but right 457 # now they use opposite values) 458 Types = enum.Enum('Client', 'Server', start_value=1) 459 460 name = dbmodels.CharField(maxlength=255, unique=True) 461 test_class = dbmodels.CharField(maxlength=255, 462 choices=Classes.choices()) 463 description = dbmodels.TextField(blank=True) 464 test_type = dbmodels.SmallIntegerField(choices=Types.choices()) 465 synch_type = dbmodels.SmallIntegerField(choices=SynchType.choices(), 466 default=SynchType.ASYNCHRONOUS) 467 path = dbmodels.CharField(maxlength=255) 468 469 name_field = 'name' 470 471 472 class Meta: 473 db_table = 'autotests' 474 475 class Admin: 476 fields = ( 477 (None, {'fields' : 478 ('name', 'test_class', 'test_type', 'synch_type', 479 'path', 'description')}), 480 ) 481 list_display = ('name', 'test_type', 'synch_type', 482 'description') 483 search_fields = ('name',) 484 485 def __str__(self): 486 return self.name 487 488 489class User(dbmodels.Model, ModelExtensions): 490 """\ 491 Required: 492 login :user login name 493 494 Optional: 495 access_level: 0=User (default), 1=Admin, 100=Root 496 """ 497 ACCESS_ROOT = 100 498 ACCESS_ADMIN = 1 499 ACCESS_USER = 0 500 501 login = dbmodels.CharField(maxlength=255, unique=True) 502 access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True) 503 504 name_field = 'login' 505 506 507 def save(self): 508 # is this a new object being saved for the first time? 509 first_time = (self.id is None) 510 super(User, self).save() 511 if first_time: 512 everyone = AclGroup.objects.get(name='Everyone') 513 everyone.users.add(self) 514 515 516 def has_access(self, target): 517 if self.access_level >= self.ACCESS_ROOT: 518 return True 519 520 if isinstance(target, int): 521 return self.access_level >= target 522 if isinstance(target, Job): 523 return (target.owner == self.login or 524 self.access_level >= self.ACCESS_ADMIN) 525 if isinstance(target, Host): 526 acl_intersect = [group 527 for group in self.aclgroup_set.all() 528 if group in target.aclgroup_set.all()] 529 return bool(acl_intersect) 530 if isinstance(target, User): 531 return self.access_level >= target.access_level 532 raise ValueError('Invalid target type') 533 534 535 class Meta: 536 db_table = 'users' 537 538 class Admin: 539 list_display = ('login', 'access_level') 540 search_fields = ('login',) 541 542 def __str__(self): 543 return self.login 544 545 546class AclGroup(dbmodels.Model, ModelExtensions): 547 """\ 548 Required: 549 name: name of ACL group 550 551 Optional: 552 description: arbitrary description of group 553 """ 554 name = dbmodels.CharField(maxlength=255, unique=True) 555 description = dbmodels.CharField(maxlength=255, blank=True) 556 users = dbmodels.ManyToManyField(User, 557 filter_interface=dbmodels.HORIZONTAL) 558 hosts = dbmodels.ManyToManyField(Host, 559 filter_interface=dbmodels.HORIZONTAL) 560 561 name_field = 'name' 562 563 564 class Meta: 565 db_table = 'acl_groups' 566 567 class Admin: 568 list_display = ('name', 'description') 569 search_fields = ('name',) 570 571 def __str__(self): 572 return self.name 573 574# hack to make the column name in the many-to-many DB tables match the one 575# generated by ruby 576AclGroup._meta.object_name = 'acl_group' 577 578 579class JobManager(dbmodels.Manager): 580 'Custom manager to provide efficient status counts querying.' 581 def get_status_counts(self, job_ids): 582 """\ 583 Returns a dictionary mapping the given job IDs to their status 584 count dictionaries. 585 """ 586 if not job_ids: 587 return {} 588 id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids) 589 from django.db import connection 590 cursor = connection.cursor() 591 cursor.execute(""" 592 SELECT job_id, status, COUNT(*) 593 FROM host_queue_entries 594 WHERE job_id IN %s 595 GROUP BY job_id, status 596 """ % id_list) 597 all_job_counts = {} 598 for job_id in job_ids: 599 all_job_counts[job_id] = {} 600 for job_id, status, count in cursor.fetchall(): 601 all_job_counts[job_id][status] = count 602 return all_job_counts 603 604 605class Job(dbmodels.Model, ModelExtensions): 606 """\ 607 owner: username of job owner 608 name: job name (does not have to be unique) 609 priority: Low, Medium, High, Urgent (or 0-3) 610 control_file: contents of control file 611 control_type: Client or Server 612 created_on: date of job creation 613 submitted_on: date of job submission 614 synch_type: Asynchronous or Synchronous (i.e. job must run on all hosts 615 simultaneously; used for server-side control files) 616 synch_count: ??? 617 synchronizing: for scheduler use 618 """ 619 Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent') 620 ControlType = enum.Enum('Server', 'Client', start_value=1) 621 Status = enum.Enum('Created', 'Queued', 'Pending', 'Running', 622 'Completed', 'Abort', 'Aborting', 'Aborted', 623 'Failed', string_values=True) 624 625 owner = dbmodels.CharField(maxlength=255) 626 name = dbmodels.CharField(maxlength=255) 627 priority = dbmodels.SmallIntegerField(choices=Priority.choices(), 628 blank=True, # to allow 0 629 default=Priority.MEDIUM) 630 control_file = dbmodels.TextField() 631 control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(), 632 blank=True) # to allow 0 633 created_on = dbmodels.DateTimeField(auto_now_add=True) 634 synch_type = dbmodels.SmallIntegerField( 635 blank=True, null=True, choices=Test.SynchType.choices()) 636 synch_count = dbmodels.IntegerField(blank=True, null=True) 637 synchronizing = dbmodels.BooleanField(default=False) 638 639 640 # custom manager 641 objects = JobManager() 642 643 644 def is_server_job(self): 645 return self.control_type == self.ControlType.SERVER 646 647 648 @classmethod 649 def create(cls, owner, name, priority, control_file, control_type, 650 hosts, synch_type): 651 """\ 652 Creates a job by taking some information (the listed args) 653 and filling in the rest of the necessary information. 654 """ 655 job = cls.add_object( 656 owner=owner, name=name, priority=priority, 657 control_file=control_file, control_type=control_type, 658 synch_type=synch_type) 659 660 if job.synch_type == Test.SynchType.SYNCHRONOUS: 661 job.synch_count = len(hosts) 662 else: 663 if len(hosts) == 0: 664 errors = {'hosts': 665 'asynchronous jobs require at least' 666 + ' one host to run on'} 667 raise ValidationError(errors) 668 job.save() 669 return job 670 671 672 def queue(self, hosts): 673 'Enqueue a job on the given hosts.' 674 for host in hosts: 675 host.enqueue_job(self) 676 host.block_auto_assign(self) 677 678 679 def requeue(self, new_owner): 680 'Creates a new job identical to this one' 681 hosts = [queue_entry.meta_host or queue_entry.host 682 for queue_entry in self.hostqueueentry_set.all()] 683 new_job = Job.create( 684 owner=new_owner, name=self.name, priority=self.priority, 685 control_file=self.control_file, 686 control_type=self.control_type, hosts=hosts, 687 synch_type=self.synch_type) 688 new_job.queue(hosts) 689 return new_job 690 691 692 def abort(self): 693 for queue_entry in self.hostqueueentry_set.all(): 694 if queue_entry.active: 695 queue_entry.status = Job.Status.ABORT 696 elif not queue_entry.complete: 697 queue_entry.status = Job.Status.ABORTED 698 queue_entry.active = False 699 queue_entry.complete = True 700 queue_entry.save() 701 702 703 def user(self): 704 try: 705 return User.objects.get(login=self.owner) 706 except self.DoesNotExist: 707 return None 708 709 710 class Meta: 711 db_table = 'jobs' 712 713 if settings.FULL_ADMIN: 714 class Admin: 715 list_display = ('id', 'owner', 'name', 'control_type', 716 'status') 717 718 def __str__(self): 719 return '%s (%s-%s)' % (self.name, self.id, self.owner) 720 721 722class IneligibleHostQueue(dbmodels.Model): 723 job = dbmodels.ForeignKey(Job) 724 host = dbmodels.ForeignKey(Host) 725 726 class Meta: 727 db_table = 'ineligible_host_queues' 728 729 if settings.FULL_ADMIN: 730 class Admin: 731 list_display = ('id', 'job', 'host') 732 733 734class HostQueueEntry(dbmodels.Model): 735 job = dbmodels.ForeignKey(Job) 736 host = dbmodels.ForeignKey(Host, blank=True, null=True) 737 priority = dbmodels.SmallIntegerField() 738 status = dbmodels.CharField(maxlength=255) 739 meta_host = dbmodels.ForeignKey(Label, blank=True, null=True, 740 db_column='meta_host') 741 active = dbmodels.BooleanField(default=False) 742 complete = dbmodels.BooleanField(default=False) 743 744 745 def is_meta_host_entry(self): 746 'True if this is a entry has a meta_host instead of a host.' 747 return self.host is None and self.meta_host is not None 748 749 750 def save(self): 751 if self.host: 752 user = self.job.user() 753 if user is None or not user.has_access(self.host): 754 raise AclAccessViolation( 755 'User %s does not have permission to ' 756 'access host %s' % (self.job.owner, 757 self.host.hostname)) 758 super(HostQueueEntry, self).save() 759 760 761 class Meta: 762 db_table = 'host_queue_entries' 763 764 if settings.FULL_ADMIN: 765 class Admin: 766 list_display = ('id', 'job', 'host', 'status', 767 'meta_host') 768