models.py revision 97446887819594f1e2a329dcff289ee8e934b626
1import logging
2from datetime import datetime
3from django.db import models as dbmodels, connection
4import common
5from autotest_lib.frontend.afe import model_logic
6from autotest_lib.frontend import settings, thread_local
7from autotest_lib.client.common_lib import enum, host_protections, global_config
8
9# job options and user preferences
10RebootBefore = enum.Enum('Never', 'If dirty', 'Always')
11DEFAULT_REBOOT_BEFORE = RebootBefore.IF_DIRTY
12RebootAfter = enum.Enum('Never', 'If all tests passed', 'Always')
13DEFAULT_REBOOT_AFTER = RebootBefore.ALWAYS
14
15
16class AclAccessViolation(Exception):
17    """\
18    Raised when an operation is attempted with proper permissions as
19    dictated by ACLs.
20    """
21
22
23class AtomicGroup(model_logic.ModelWithInvalid, dbmodels.Model):
24    """\
25    An atomic group defines a collection of hosts which must only be scheduled
26    all at once.  Any host with a label having an atomic group will only be
27    scheduled for a job at the same time as other hosts sharing that label.
28
29    Required:
30      name: A name for this atomic group.  ex: 'rack23' or 'funky_net'
31      max_number_of_machines: The maximum number of machines that will be
32              scheduled at once when scheduling jobs to this atomic group.
33              The job.synch_count is considered the minimum.
34
35    Optional:
36      description: Arbitrary text description of this group's purpose.
37    """
38    name = dbmodels.CharField(maxlength=255, unique=True)
39    description = dbmodels.TextField(blank=True)
40    # This magic value is the default to simplify the scheduler logic.
41    # It must be "large".  The common use of atomic groups is to want all
42    # machines in the group to be used, limits on which subset used are
43    # often chosen via dependency labels.
44    INFINITE_MACHINES = 333333333
45    max_number_of_machines = dbmodels.IntegerField(default=INFINITE_MACHINES)
46    invalid = dbmodels.BooleanField(default=False,
47                                    editable=settings.FULL_ADMIN)
48
49    name_field = 'name'
50    objects = model_logic.ExtendedManager()
51    valid_objects = model_logic.ValidObjectsManager()
52
53
54    def enqueue_job(self, job, is_template=False):
55        """Enqueue a job on an associated atomic group of hosts."""
56        queue_entry = HostQueueEntry.create(atomic_group=self, job=job,
57                                            is_template=is_template)
58        queue_entry.save()
59
60
61    def clean_object(self):
62        self.label_set.clear()
63
64
65    class Meta:
66        db_table = 'atomic_groups'
67
68    class Admin:
69        list_display = ('name', 'description', 'max_number_of_machines')
70        # see Host.Admin
71        manager = model_logic.ValidObjectsManager()
72
73    def __str__(self):
74        return self.name
75
76
77class Label(model_logic.ModelWithInvalid, dbmodels.Model):
78    """\
79    Required:
80      name: label name
81
82    Optional:
83      kernel_config: URL/path to kernel config for jobs run on this label.
84      platform: If True, this is a platform label (defaults to False).
85      only_if_needed: If True, a Host with this label can only be used if that
86              label is requested by the job/test (either as the meta_host or
87              in the job_dependencies).
88      atomic_group: The atomic group associated with this label.
89    """
90    name = dbmodels.CharField(maxlength=255, unique=True)
91    kernel_config = dbmodels.CharField(maxlength=255, blank=True)
92    platform = dbmodels.BooleanField(default=False)
93    invalid = dbmodels.BooleanField(default=False,
94                                    editable=settings.FULL_ADMIN)
95    only_if_needed = dbmodels.BooleanField(default=False)
96
97    name_field = 'name'
98    objects = model_logic.ExtendedManager()
99    valid_objects = model_logic.ValidObjectsManager()
100    atomic_group = dbmodels.ForeignKey(AtomicGroup, null=True, blank=True)
101
102
103    def clean_object(self):
104        self.host_set.clear()
105        self.test_set.clear()
106
107
108    def enqueue_job(self, job, atomic_group=None, is_template=False):
109        """Enqueue a job on any host of this label."""
110        queue_entry = HostQueueEntry.create(meta_host=self, job=job,
111                                            is_template=is_template,
112                                            atomic_group=atomic_group)
113        queue_entry.save()
114
115
116    class Meta:
117        db_table = 'labels'
118
119    class Admin:
120        list_display = ('name', 'kernel_config')
121        # see Host.Admin
122        manager = model_logic.ValidObjectsManager()
123
124    def __str__(self):
125        return self.name
126
127
128class User(dbmodels.Model, model_logic.ModelExtensions):
129    """\
130    Required:
131    login :user login name
132
133    Optional:
134    access_level: 0=User (default), 1=Admin, 100=Root
135    """
136    ACCESS_ROOT = 100
137    ACCESS_ADMIN = 1
138    ACCESS_USER = 0
139
140    login = dbmodels.CharField(maxlength=255, unique=True)
141    access_level = dbmodels.IntegerField(default=ACCESS_USER, blank=True)
142
143    # user preferences
144    reboot_before = dbmodels.SmallIntegerField(choices=RebootBefore.choices(),
145                                               blank=True,
146                                               default=DEFAULT_REBOOT_BEFORE)
147    reboot_after = dbmodels.SmallIntegerField(choices=RebootAfter.choices(),
148                                              blank=True,
149                                              default=DEFAULT_REBOOT_AFTER)
150    show_experimental = dbmodels.BooleanField(default=False)
151
152    name_field = 'login'
153    objects = model_logic.ExtendedManager()
154
155
156    def save(self):
157        # is this a new object being saved for the first time?
158        first_time = (self.id is None)
159        user = thread_local.get_user()
160        if user and not user.is_superuser() and user.login != self.login:
161            raise AclAccessViolation("You cannot modify user " + self.login)
162        super(User, self).save()
163        if first_time:
164            everyone = AclGroup.objects.get(name='Everyone')
165            everyone.users.add(self)
166
167
168    def is_superuser(self):
169        return self.access_level >= self.ACCESS_ROOT
170
171
172    class Meta:
173        db_table = 'users'
174
175    class Admin:
176        list_display = ('login', 'access_level')
177        search_fields = ('login',)
178
179    def __str__(self):
180        return self.login
181
182
183class Host(model_logic.ModelWithInvalid, dbmodels.Model,
184           model_logic.ModelWithAttributes):
185    """\
186    Required:
187    hostname
188
189    optional:
190    locked: if true, host is locked and will not be queued
191
192    Internal:
193    synch_id: currently unused
194    status: string describing status of host
195    invalid: true if the host has been deleted
196    protection: indicates what can be done to this host during repair
197    locked_by: user that locked the host, or null if the host is unlocked
198    lock_time: DateTime at which the host was locked
199    dirty: true if the host has been used without being rebooted
200    """
201    Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing',
202                       'Repair Failed', 'Dead', 'Cleaning', 'Pending',
203                       string_values=True)
204
205    hostname = dbmodels.CharField(maxlength=255, unique=True)
206    labels = dbmodels.ManyToManyField(Label, blank=True,
207                                      filter_interface=dbmodels.HORIZONTAL)
208    locked = dbmodels.BooleanField(default=False)
209    synch_id = dbmodels.IntegerField(blank=True, null=True,
210                                     editable=settings.FULL_ADMIN)
211    status = dbmodels.CharField(maxlength=255, default=Status.READY,
212                                choices=Status.choices(),
213                                editable=settings.FULL_ADMIN)
214    invalid = dbmodels.BooleanField(default=False,
215                                    editable=settings.FULL_ADMIN)
216    protection = dbmodels.SmallIntegerField(null=False, blank=True,
217                                            choices=host_protections.choices,
218                                            default=host_protections.default)
219    locked_by = dbmodels.ForeignKey(User, null=True, blank=True, editable=False)
220    lock_time = dbmodels.DateTimeField(null=True, blank=True, editable=False)
221    dirty = dbmodels.BooleanField(default=True, editable=settings.FULL_ADMIN)
222
223    name_field = 'hostname'
224    objects = model_logic.ExtendedManager()
225    valid_objects = model_logic.ValidObjectsManager()
226
227
228    def __init__(self, *args, **kwargs):
229        super(Host, self).__init__(*args, **kwargs)
230        self._record_attributes(['status'])
231
232
233    @staticmethod
234    def create_one_time_host(hostname):
235        query = Host.objects.filter(hostname=hostname)
236        if query.count() == 0:
237            host = Host(hostname=hostname, invalid=True)
238            host.do_validate()
239        else:
240            host = query[0]
241            if not host.invalid:
242                raise model_logic.ValidationError({
243                    'hostname' : '%s already exists in the autotest DB.  '
244                        'Select it rather than entering it as a one time '
245                        'host.' % hostname
246                    })
247            host.status = Host.Status.READY
248        host.protection = host_protections.Protection.DO_NOT_REPAIR
249        host.locked = False
250        host.save()
251        host.clean_object()
252        return host
253
254
255    def clean_object(self):
256        self.aclgroup_set.clear()
257        self.labels.clear()
258
259
260    def save(self):
261        # extra spaces in the hostname can be a sneaky source of errors
262        self.hostname = self.hostname.strip()
263        # is this a new object being saved for the first time?
264        first_time = (self.id is None)
265        if not first_time:
266            AclGroup.check_for_acl_violation_hosts([self])
267        if self.locked and not self.locked_by:
268            self.locked_by = thread_local.get_user()
269            self.lock_time = datetime.now()
270            self.dirty = True
271        elif not self.locked and self.locked_by:
272            self.locked_by = None
273            self.lock_time = None
274        super(Host, self).save()
275        if first_time:
276            everyone = AclGroup.objects.get(name='Everyone')
277            everyone.hosts.add(self)
278        self._check_for_updated_attributes()
279
280
281    def delete(self):
282        AclGroup.check_for_acl_violation_hosts([self])
283        for queue_entry in self.hostqueueentry_set.all():
284            queue_entry.deleted = True
285            queue_entry.abort(thread_local.get_user())
286        super(Host, self).delete()
287
288
289    def on_attribute_changed(self, attribute, old_value):
290        assert attribute == 'status'
291        logging.info(self.hostname + ' -> ' + self.status)
292
293
294    def enqueue_job(self, job, atomic_group=None, is_template=False):
295        """Enqueue a job on this host."""
296        queue_entry = HostQueueEntry.create(host=self, job=job,
297                                            is_template=is_template,
298                                            atomic_group=atomic_group)
299        # allow recovery of dead hosts from the frontend
300        if not self.active_queue_entry() and self.is_dead():
301            self.status = Host.Status.READY
302            self.save()
303        queue_entry.save()
304
305        block = IneligibleHostQueue(job=job, host=self)
306        block.save()
307
308
309    def platform(self):
310        # TODO(showard): slighly hacky?
311        platforms = self.labels.filter(platform=True)
312        if len(platforms) == 0:
313            return None
314        return platforms[0]
315    platform.short_description = 'Platform'
316
317
318    @classmethod
319    def check_no_platform(cls, hosts):
320        Host.objects.populate_relationships(hosts, Label, 'label_list')
321        errors = []
322        for host in hosts:
323            platforms = [label.name for label in host.label_list
324                         if label.platform]
325            if platforms:
326                # do a join, just in case this host has multiple platforms,
327                # we'll be able to see it
328                errors.append('Host %s already has a platform: %s' % (
329                              host.hostname, ', '.join(platforms)))
330        if errors:
331            raise model_logic.ValidationError({'labels': '; '.join(errors)})
332
333
334    def is_dead(self):
335        return self.status == Host.Status.REPAIR_FAILED
336
337
338    def active_queue_entry(self):
339        active = list(self.hostqueueentry_set.filter(active=True))
340        if not active:
341            return None
342        assert len(active) == 1, ('More than one active entry for '
343                                  'host ' + self.hostname)
344        return active[0]
345
346
347    def _get_attribute_model_and_args(self, attribute):
348        return HostAttribute, dict(host=self, attribute=attribute)
349
350
351    class Meta:
352        db_table = 'hosts'
353
354    class Admin:
355        # TODO(showard) - showing platform requires a SQL query for
356        # each row (since labels are many-to-many) - should we remove
357        # it?
358        list_display = ('hostname', 'platform', 'locked', 'status')
359        list_filter = ('labels', 'locked', 'protection')
360        search_fields = ('hostname', 'status')
361        # undocumented Django feature - if you set manager here, the
362        # admin code will use it, otherwise it'll use a default Manager
363        manager = model_logic.ValidObjectsManager()
364
365    def __str__(self):
366        return self.hostname
367
368
369class HostAttribute(dbmodels.Model):
370    """Arbitrary keyvals associated with hosts."""
371    host = dbmodels.ForeignKey(Host)
372    attribute = dbmodels.CharField(maxlength=90)
373    value = dbmodels.CharField(maxlength=300)
374
375    objects = model_logic.ExtendedManager()
376
377    class Meta:
378        db_table = 'host_attributes'
379
380
381class Test(dbmodels.Model, model_logic.ModelExtensions):
382    """\
383    Required:
384    author: author name
385    description: description of the test
386    name: test name
387    time: short, medium, long
388    test_class: This describes the class for your the test belongs in.
389    test_category: This describes the category for your tests
390    test_type: Client or Server
391    path: path to pass to run_test()
392    sync_count:  is a number >=1 (1 being the default). If it's 1, then it's an
393                 async job. If it's >1 it's sync job for that number of machines
394                 i.e. if sync_count = 2 it is a sync job that requires two
395                 machines.
396    Optional:
397    dependencies: What the test requires to run. Comma deliminated list
398    dependency_labels: many-to-many relationship with labels corresponding to
399                       test dependencies.
400    experimental: If this is set to True production servers will ignore the test
401    run_verify: Whether or not the scheduler should run the verify stage
402    """
403    TestTime = enum.Enum('SHORT', 'MEDIUM', 'LONG', start_value=1)
404    # TODO(showard) - this should be merged with Job.ControlType (but right
405    # now they use opposite values)
406    Types = enum.Enum('Client', 'Server', start_value=1)
407
408    name = dbmodels.CharField(maxlength=255, unique=True)
409    author = dbmodels.CharField(maxlength=255)
410    test_class = dbmodels.CharField(maxlength=255)
411    test_category = dbmodels.CharField(maxlength=255)
412    dependencies = dbmodels.CharField(maxlength=255, blank=True)
413    description = dbmodels.TextField(blank=True)
414    experimental = dbmodels.BooleanField(default=True)
415    run_verify = dbmodels.BooleanField(default=True)
416    test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(),
417                                           default=TestTime.MEDIUM)
418    test_type = dbmodels.SmallIntegerField(choices=Types.choices())
419    sync_count = dbmodels.IntegerField(default=1)
420    path = dbmodels.CharField(maxlength=255, unique=True)
421    dependency_labels = dbmodels.ManyToManyField(
422        Label, blank=True, filter_interface=dbmodels.HORIZONTAL)
423
424    name_field = 'name'
425    objects = model_logic.ExtendedManager()
426
427
428    class Meta:
429        db_table = 'autotests'
430
431    class Admin:
432        fields = (
433            (None, {'fields' :
434                    ('name', 'author', 'test_category', 'test_class',
435                     'test_time', 'sync_count', 'test_type', 'path',
436                     'dependencies', 'experimental', 'run_verify',
437                     'description')}),
438            )
439        list_display = ('name', 'test_type', 'description', 'sync_count')
440        search_fields = ('name',)
441
442    def __str__(self):
443        return self.name
444
445
446class Profiler(dbmodels.Model, model_logic.ModelExtensions):
447    """\
448    Required:
449    name: profiler name
450    test_type: Client or Server
451
452    Optional:
453    description: arbirary text description
454    """
455    name = dbmodels.CharField(maxlength=255, unique=True)
456    description = dbmodels.TextField(blank=True)
457
458    name_field = 'name'
459    objects = model_logic.ExtendedManager()
460
461
462    class Meta:
463        db_table = 'profilers'
464
465    class Admin:
466        list_display = ('name', 'description')
467        search_fields = ('name',)
468
469    def __str__(self):
470        return self.name
471
472
473class AclGroup(dbmodels.Model, model_logic.ModelExtensions):
474    """\
475    Required:
476    name: name of ACL group
477
478    Optional:
479    description: arbitrary description of group
480    """
481    name = dbmodels.CharField(maxlength=255, unique=True)
482    description = dbmodels.CharField(maxlength=255, blank=True)
483    users = dbmodels.ManyToManyField(User, blank=True,
484                                     filter_interface=dbmodels.HORIZONTAL)
485    hosts = dbmodels.ManyToManyField(Host,
486                                     filter_interface=dbmodels.HORIZONTAL)
487
488    name_field = 'name'
489    objects = model_logic.ExtendedManager()
490
491    @staticmethod
492    def check_for_acl_violation_hosts(hosts):
493        user = thread_local.get_user()
494        if user.is_superuser():
495            return
496        accessible_host_ids = set(
497            host.id for host in Host.objects.filter(aclgroup__users=user))
498        for host in hosts:
499            # Check if the user has access to this host,
500            # but only if it is not a metahost or a one-time-host
501            no_access = (isinstance(host, Host)
502                         and not host.invalid
503                         and int(host.id) not in accessible_host_ids)
504            if no_access:
505                raise AclAccessViolation("You do not have access to %s"
506                                         % str(host))
507
508
509    @staticmethod
510    def check_abort_permissions(queue_entries):
511        """
512        look for queue entries that aren't abortable, meaning
513         * the job isn't owned by this user, and
514           * the machine isn't ACL-accessible, or
515           * the machine is in the "Everyone" ACL
516        """
517        user = thread_local.get_user()
518        if user.is_superuser():
519            return
520        not_owned = queue_entries.exclude(job__owner=user.login)
521        # I do this using ID sets instead of just Django filters because
522        # filtering on M2M fields is broken in Django 0.96.  It's better in 1.0.
523        accessible_ids = set(
524            entry.id for entry
525            in not_owned.filter(host__aclgroup__users__login=user.login))
526        public_ids = set(entry.id for entry
527                         in not_owned.filter(host__aclgroup__name='Everyone'))
528        cannot_abort = [entry for entry in not_owned.select_related()
529                        if entry.id not in accessible_ids
530                        or entry.id in public_ids]
531        if len(cannot_abort) == 0:
532            return
533        entry_names = ', '.join('%s-%s/%s' % (entry.job.id, entry.job.owner,
534                                              entry.host_or_metahost_name())
535                                for entry in cannot_abort)
536        raise AclAccessViolation('You cannot abort the following job entries: '
537                                 + entry_names)
538
539
540    def check_for_acl_violation_acl_group(self):
541        user = thread_local.get_user()
542        if user.is_superuser():
543            return None
544        if not user in self.users.all():
545            raise AclAccessViolation("You do not have access to %s"
546                                     % self.name)
547
548    @staticmethod
549    def on_host_membership_change():
550        everyone = AclGroup.objects.get(name='Everyone')
551
552        # find hosts that aren't in any ACL group and add them to Everyone
553        # TODO(showard): this is a bit of a hack, since the fact that this query
554        # works is kind of a coincidence of Django internals.  This trick
555        # doesn't work in general (on all foreign key relationships).  I'll
556        # replace it with a better technique when the need arises.
557        orphaned_hosts = Host.valid_objects.filter(aclgroup__id__isnull=True)
558        everyone.hosts.add(*orphaned_hosts.distinct())
559
560        # find hosts in both Everyone and another ACL group, and remove them
561        # from Everyone
562        hosts_in_everyone = Host.valid_objects.filter_custom_join(
563            '_everyone', aclgroup__name='Everyone')
564        acled_hosts = hosts_in_everyone.exclude(aclgroup__name='Everyone')
565        everyone.hosts.remove(*acled_hosts.distinct())
566
567
568    def delete(self):
569        if (self.name == 'Everyone'):
570            raise AclAccessViolation("You cannot delete 'Everyone'!")
571        self.check_for_acl_violation_acl_group()
572        super(AclGroup, self).delete()
573        self.on_host_membership_change()
574
575
576    def add_current_user_if_empty(self):
577        if not self.users.count():
578            self.users.add(thread_local.get_user())
579
580
581    # if you have a model attribute called "Manipulator", Django will
582    # automatically insert it into the beginning of the superclass list
583    # for the model's manipulators
584    class Manipulator(object):
585        """
586        Custom manipulator to get notification when ACLs are changed through
587        the admin interface.
588        """
589        def save(self, new_data):
590            user = thread_local.get_user()
591            if hasattr(self, 'original_object'):
592                if (not user.is_superuser()
593                    and self.original_object.name == 'Everyone'):
594                    raise AclAccessViolation("You cannot modify 'Everyone'!")
595                self.original_object.check_for_acl_violation_acl_group()
596            obj = super(AclGroup.Manipulator, self).save(new_data)
597            if not hasattr(self, 'original_object'):
598                obj.users.add(thread_local.get_user())
599            obj.add_current_user_if_empty()
600            obj.on_host_membership_change()
601            return obj
602
603    class Meta:
604        db_table = 'acl_groups'
605
606    class Admin:
607        list_display = ('name', 'description')
608        search_fields = ('name',)
609
610    def __str__(self):
611        return self.name
612
613
614class JobManager(model_logic.ExtendedManager):
615    'Custom manager to provide efficient status counts querying.'
616    def get_status_counts(self, job_ids):
617        """\
618        Returns a dictionary mapping the given job IDs to their status
619        count dictionaries.
620        """
621        if not job_ids:
622            return {}
623        id_list = '(%s)' % ','.join(str(job_id) for job_id in job_ids)
624        cursor = connection.cursor()
625        cursor.execute("""
626            SELECT job_id, status, aborted, complete, COUNT(*)
627            FROM host_queue_entries
628            WHERE job_id IN %s
629            GROUP BY job_id, status, aborted, complete
630            """ % id_list)
631        all_job_counts = dict((job_id, {}) for job_id in job_ids)
632        for job_id, status, aborted, complete, count in cursor.fetchall():
633            job_dict = all_job_counts[job_id]
634            full_status = HostQueueEntry.compute_full_status(status, aborted,
635                                                             complete)
636            job_dict.setdefault(full_status, 0)
637            job_dict[full_status] += count
638        return all_job_counts
639
640
641class Job(dbmodels.Model, model_logic.ModelExtensions):
642    """\
643    owner: username of job owner
644    name: job name (does not have to be unique)
645    priority: Low, Medium, High, Urgent (or 0-3)
646    control_file: contents of control file
647    control_type: Client or Server
648    created_on: date of job creation
649    submitted_on: date of job submission
650    synch_count: how many hosts should be used per autoserv execution
651    run_verify: Whether or not to run the verify phase
652    timeout: hours from queuing time until job times out
653    max_runtime_hrs: hours from job starting time until job times out
654    email_list: list of people to email on completion delimited by any of:
655                white space, ',', ':', ';'
656    dependency_labels: many-to-many relationship with labels corresponding to
657                       job dependencies
658    reboot_before: Never, If dirty, or Always
659    reboot_after: Never, If all tests passed, or Always
660    parse_failed_repair: if True, a failed repair launched by this job will have
661    its results parsed as part of the job.
662    """
663    DEFAULT_TIMEOUT = global_config.global_config.get_config_value(
664        'AUTOTEST_WEB', 'job_timeout_default', default=240)
665    DEFAULT_MAX_RUNTIME_HRS = global_config.global_config.get_config_value(
666        'AUTOTEST_WEB', 'job_max_runtime_hrs_default', default=72)
667    DEFAULT_PARSE_FAILED_REPAIR = global_config.global_config.get_config_value(
668        'AUTOTEST_WEB', 'parse_failed_repair_default', type=bool,
669        default=False)
670
671    Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent')
672    ControlType = enum.Enum('Server', 'Client', start_value=1)
673
674    owner = dbmodels.CharField(maxlength=255)
675    name = dbmodels.CharField(maxlength=255)
676    priority = dbmodels.SmallIntegerField(choices=Priority.choices(),
677                                          blank=True, # to allow 0
678                                          default=Priority.MEDIUM)
679    control_file = dbmodels.TextField()
680    control_type = dbmodels.SmallIntegerField(choices=ControlType.choices(),
681                                              blank=True, # to allow 0
682                                              default=ControlType.CLIENT)
683    created_on = dbmodels.DateTimeField()
684    synch_count = dbmodels.IntegerField(null=True, default=1)
685    timeout = dbmodels.IntegerField(default=DEFAULT_TIMEOUT)
686    run_verify = dbmodels.BooleanField(default=True)
687    email_list = dbmodels.CharField(maxlength=250, blank=True)
688    dependency_labels = dbmodels.ManyToManyField(
689        Label, blank=True, filter_interface=dbmodels.HORIZONTAL)
690    reboot_before = dbmodels.SmallIntegerField(choices=RebootBefore.choices(),
691                                               blank=True,
692                                               default=DEFAULT_REBOOT_BEFORE)
693    reboot_after = dbmodels.SmallIntegerField(choices=RebootAfter.choices(),
694                                              blank=True,
695                                              default=DEFAULT_REBOOT_AFTER)
696    parse_failed_repair = dbmodels.BooleanField(
697        default=DEFAULT_PARSE_FAILED_REPAIR)
698    max_runtime_hrs = dbmodels.IntegerField(default=DEFAULT_MAX_RUNTIME_HRS)
699
700
701    # custom manager
702    objects = JobManager()
703
704
705    def is_server_job(self):
706        return self.control_type == self.ControlType.SERVER
707
708
709    @classmethod
710    def create(cls, owner, options, hosts):
711        """\
712        Creates a job by taking some information (the listed args)
713        and filling in the rest of the necessary information.
714        """
715        AclGroup.check_for_acl_violation_hosts(hosts)
716        job = cls.add_object(
717            owner=owner,
718            name=options['name'],
719            priority=options['priority'],
720            control_file=options['control_file'],
721            control_type=options['control_type'],
722            synch_count=options.get('synch_count'),
723            timeout=options.get('timeout'),
724            max_runtime_hrs=options.get('max_runtime_hrs'),
725            run_verify=options.get('run_verify'),
726            email_list=options.get('email_list'),
727            reboot_before=options.get('reboot_before'),
728            reboot_after=options.get('reboot_after'),
729            parse_failed_repair=options.get('parse_failed_repair'),
730            created_on=datetime.now())
731
732        job.dependency_labels = options['dependencies']
733        return job
734
735
736    def queue(self, hosts, atomic_group=None, is_template=False):
737        """Enqueue a job on the given hosts."""
738        if atomic_group and not hosts:
739            # No hosts or labels are required to queue an atomic group
740            # Job.  However, if they are given, we respect them below.
741            atomic_group.enqueue_job(self, is_template=is_template)
742        for host in hosts:
743            host.enqueue_job(self, atomic_group=atomic_group,
744                             is_template=is_template)
745
746
747    def create_recurring_job(self, start_date, loop_period, loop_count, owner):
748        rec = RecurringRun(job=self, start_date=start_date,
749                           loop_period=loop_period,
750                           loop_count=loop_count,
751                           owner=User.objects.get(login=owner))
752        rec.save()
753        return rec.id
754
755
756    def user(self):
757        try:
758            return User.objects.get(login=self.owner)
759        except self.DoesNotExist:
760            return None
761
762
763    def abort(self, aborted_by):
764        for queue_entry in self.hostqueueentry_set.all():
765            queue_entry.abort(aborted_by)
766
767
768    class Meta:
769        db_table = 'jobs'
770
771    if settings.FULL_ADMIN:
772        class Admin:
773            list_display = ('id', 'owner', 'name', 'control_type')
774
775    def __str__(self):
776        return '%s (%s-%s)' % (self.name, self.id, self.owner)
777
778
779class IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions):
780    job = dbmodels.ForeignKey(Job)
781    host = dbmodels.ForeignKey(Host)
782
783    objects = model_logic.ExtendedManager()
784
785    class Meta:
786        db_table = 'ineligible_host_queues'
787
788    if settings.FULL_ADMIN:
789        class Admin:
790            list_display = ('id', 'job', 'host')
791
792
793class HostQueueEntry(dbmodels.Model, model_logic.ModelExtensions):
794    Status = enum.Enum('Queued', 'Starting', 'Verifying', 'Pending', 'Running',
795                       'Gathering', 'Parsing', 'Aborted', 'Completed',
796                       'Failed', 'Stopped', 'Template', string_values=True)
797    ACTIVE_STATUSES = (Status.STARTING, Status.VERIFYING, Status.PENDING,
798                       Status.RUNNING, Status.GATHERING)
799    COMPLETE_STATUSES = (Status.ABORTED, Status.COMPLETED, Status.FAILED,
800                         Status.STOPPED, Status.TEMPLATE)
801
802    job = dbmodels.ForeignKey(Job)
803    host = dbmodels.ForeignKey(Host, blank=True, null=True)
804    status = dbmodels.CharField(maxlength=255)
805    meta_host = dbmodels.ForeignKey(Label, blank=True, null=True,
806                                    db_column='meta_host')
807    active = dbmodels.BooleanField(default=False)
808    complete = dbmodels.BooleanField(default=False)
809    deleted = dbmodels.BooleanField(default=False)
810    execution_subdir = dbmodels.CharField(maxlength=255, blank=True, default='')
811    # If atomic_group is set, this is a virtual HostQueueEntry that will
812    # be expanded into many actual hosts within the group at schedule time.
813    atomic_group = dbmodels.ForeignKey(AtomicGroup, blank=True, null=True)
814    aborted = dbmodels.BooleanField(default=False)
815    started_on = dbmodels.DateTimeField(null=True)
816
817    objects = model_logic.ExtendedManager()
818
819
820    def __init__(self, *args, **kwargs):
821        super(HostQueueEntry, self).__init__(*args, **kwargs)
822        self._record_attributes(['status'])
823
824
825    @classmethod
826    def create(cls, job, host=None, meta_host=None, atomic_group=None,
827                 is_template=False):
828        if is_template:
829            status = cls.Status.TEMPLATE
830        else:
831            status = cls.Status.QUEUED
832
833        return cls(job=job, host=host, meta_host=meta_host,
834                   atomic_group=atomic_group, status=status)
835
836
837    def save(self):
838        self._set_active_and_complete()
839        super(HostQueueEntry, self).save()
840        self._check_for_updated_attributes()
841
842
843    def execution_path(self):
844        """
845        Path to this entry's results (relative to the base results directory).
846        """
847        return self.execution_subdir
848
849
850    def host_or_metahost_name(self):
851        if self.host:
852            return self.host.hostname
853        else:
854            assert self.meta_host
855            return self.meta_host.name
856
857
858    def _set_active_and_complete(self):
859        if self.status in self.ACTIVE_STATUSES:
860            self.active, self.complete = True, False
861        elif self.status in self.COMPLETE_STATUSES:
862            self.active, self.complete = False, True
863        else:
864            self.active, self.complete = False, False
865
866
867    def on_attribute_changed(self, attribute, old_value):
868        assert attribute == 'status'
869        logging.info('%s/%d (%d) -> %s' % (self.host, self.job.id, self.id,
870                                           self.status))
871
872
873    def is_meta_host_entry(self):
874        'True if this is a entry has a meta_host instead of a host.'
875        return self.host is None and self.meta_host is not None
876
877
878    def log_abort(self, user):
879        if user is None:
880            # automatic system abort (i.e. job timeout)
881            return
882        abort_log = AbortedHostQueueEntry(queue_entry=self, aborted_by=user)
883        abort_log.save()
884
885
886    def abort(self, user):
887        # this isn't completely immune to race conditions since it's not atomic,
888        # but it should be safe given the scheduler's behavior.
889        if not self.complete and not self.aborted:
890            self.log_abort(user)
891            self.aborted = True
892            self.save()
893
894
895    @classmethod
896    def compute_full_status(cls, status, aborted, complete):
897        if aborted and not complete:
898            return 'Aborted (%s)' % status
899        return status
900
901
902    def full_status(self):
903        return self.compute_full_status(self.status, self.aborted,
904                                        self.complete)
905
906
907    def _postprocess_object_dict(self, object_dict):
908        object_dict['full_status'] = self.full_status()
909
910
911    class Meta:
912        db_table = 'host_queue_entries'
913
914
915    if settings.FULL_ADMIN:
916        class Admin:
917            list_display = ('id', 'job', 'host', 'status',
918                            'meta_host')
919
920
921    def __str__(self):
922        hostname = None
923        if self.host:
924            hostname = self.host.hostname
925        return "%s/%d (%d)" % (hostname, self.job.id, self.id)
926
927
928class AbortedHostQueueEntry(dbmodels.Model, model_logic.ModelExtensions):
929    queue_entry = dbmodels.OneToOneField(HostQueueEntry, primary_key=True)
930    aborted_by = dbmodels.ForeignKey(User)
931    aborted_on = dbmodels.DateTimeField()
932
933    objects = model_logic.ExtendedManager()
934
935
936    def save(self):
937        self.aborted_on = datetime.now()
938        super(AbortedHostQueueEntry, self).save()
939
940    class Meta:
941        db_table = 'aborted_host_queue_entries'
942
943
944class RecurringRun(dbmodels.Model, model_logic.ModelExtensions):
945    """\
946    job: job to use as a template
947    owner: owner of the instantiated template
948    start_date: Run the job at scheduled date
949    loop_period: Re-run (loop) the job periodically
950                 (in every loop_period seconds)
951    loop_count: Re-run (loop) count
952    """
953
954    job = dbmodels.ForeignKey(Job)
955    owner = dbmodels.ForeignKey(User)
956    start_date = dbmodels.DateTimeField()
957    loop_period = dbmodels.IntegerField(blank=True)
958    loop_count = dbmodels.IntegerField(blank=True)
959
960    objects = model_logic.ExtendedManager()
961
962    class Meta:
963        db_table = 'recurring_run'
964
965    def __str__(self):
966        return 'RecurringRun(job %s, start %s, period %s, count %s)' % (
967            self.job.id, self.start_date, self.loop_period, self.loop_count)
968
969
970class SpecialTask(dbmodels.Model, model_logic.ModelExtensions):
971    """\
972    Tasks to run on hosts at the next time they are in the Ready state. Use this
973    for high-priority tasks, such as forced repair or forced reinstall.
974
975    host: host to run this task on
976    task: special task to run
977    time_requested: date and time the request for this task was made
978    is_active: task is currently running
979    is_complete: task has finished running
980    time_started: date and time the task started
981    queue_entry: Host queue entry waiting on this task (or None, if task was not
982                 started in preparation of a job)
983    """
984    Task = enum.Enum('Verify', 'Cleanup', 'Repair', string_values=True)
985
986    host = dbmodels.ForeignKey(Host, blank=False, null=False)
987    task = dbmodels.CharField(maxlength=64, choices=Task.choices(),
988                              blank=False, null=False)
989    time_requested = dbmodels.DateTimeField(auto_now_add=True, blank=False,
990                                            null=False)
991    is_active = dbmodels.BooleanField(default=False, blank=False, null=False)
992    is_complete = dbmodels.BooleanField(default=False, blank=False, null=False)
993    time_started = dbmodels.DateTimeField(null=True, blank=True)
994    queue_entry = dbmodels.ForeignKey(HostQueueEntry, blank=True, null=True)
995
996    objects = model_logic.ExtendedManager()
997
998
999    def execution_path(self):
1000        """@see HostQueueEntry.execution_path()"""
1001        return 'hosts/%s/%s-%s' % (self.host.hostname, self.id,
1002                                   self.task.lower())
1003
1004
1005    # property to emulate HostQueueEntry.status
1006    @property
1007    def status(self):
1008        """
1009        Return a host queue entry status appropriate for this task.  Although
1010        SpecialTasks are not HostQueueEntries, it is helpful to the user to
1011        present similar statuses.
1012        """
1013        if self.is_complete:
1014            return HostQueueEntry.Status.COMPLETED
1015        if self.is_active:
1016            return HostQueueEntry.Status.RUNNING
1017        return HostQueueEntry.Status.QUEUED
1018
1019
1020    # property to emulate HostQueueEntry.started_on
1021    @property
1022    def started_on(self):
1023        return self.time_started
1024
1025
1026    @classmethod
1027    def schedule_special_task(cls, hosts, task):
1028        """\
1029        Schedules hosts for a special task, if the task is not already scheduled
1030        """
1031        for host in hosts:
1032            if not SpecialTask.objects.filter(host__id=host.id, task=task,
1033                                              is_active=False,
1034                                              is_complete=False):
1035                special_task = SpecialTask(host=host, task=task)
1036                special_task.save()
1037
1038
1039    @classmethod
1040    def prepare(cls, agent, task):
1041        """\
1042        Creates a new special task if necessary, and prepares it to be run. Sets
1043        as active, and sets the time started to the current time.
1044
1045        agent: scheduler agent that will be taking this task
1046        task: task to prepare, or None if a new task should be created
1047        """
1048        if not task:
1049            if not hasattr(agent, 'TASK_TYPE'):
1050                raise ValueError("Can only prepare special tasks for "
1051                                 "verify, cleanup, or repair")
1052            task = cls.objects.create(host=agent.host, task=agent.TASK_TYPE,
1053                                      queue_entry=agent.queue_entry)
1054
1055        return task
1056
1057
1058    def activate(self):
1059        """\
1060        Sets a task as active.
1061        """
1062        logging.info('Starting: %s', self)
1063        self.is_active = True
1064        self.time_started = datetime.now()
1065        self.save()
1066
1067
1068    def finish(self):
1069        """\
1070        Sets a task as completed
1071        """
1072        logging.info('Finished: %s', self)
1073        self.is_active = False
1074        self.is_complete = True
1075        self.save()
1076
1077
1078    class Meta:
1079        db_table = 'special_tasks'
1080
1081    def __str__(self):
1082        result = 'Special Task %s (host %s, task %s, time %s)' % (
1083            self.id, self.host, self.task, self.time_requested)
1084        if self.is_complete:
1085            result += ' (completed)'
1086        elif self.is_active:
1087            result += ' (active)'
1088
1089        return result
1090