site_monitor_db.py revision 3d89973417ed013f328b57a7f750dcc2a788fd21
152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis# Use of this source code is governed by a BSD-style license that can be 3e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis# found in the LICENSE file. 4e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis 5e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtisimport os 6a858a233889949263ded6d0d6578495aba54a9ebSimran Basiimport logging 7e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis 852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hongfrom autotest_lib.client.common_lib import global_config 952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hongfrom autotest_lib.frontend.afe import models 10a858a233889949263ded6d0d6578495aba54a9ebSimran Basifrom autotest_lib.scheduler import scheduler_config, scheduler_models 11e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis 12e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis# Override default parser with our site parser. 13e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtisdef parser_path(install_dir): 14e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis return os.path.join(install_dir, 'tko', 'site_parse') 1552ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 1652ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 1752ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hongclass SiteAgentTask(object): 1852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 1952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong SiteAgentTask subclasses BaseAgentTask in monitor_db. 2052ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 2152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 2252ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 2352ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong def _archive_results(self, queue_entries): 2452ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 2552ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong Set the status of queue_entries to ARCHIVING. 2652ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 2752ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong This method sets the status of the queue_entries to ARCHIVING 2852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong if the enable_archiving flag is true in global_config.ini. 2952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong Otherwise, it bypasses the archiving step and sets the queue entries 3052ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong to the final status of current step. 3152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 3252ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong enable_archiving = global_config.global_config.get_config_value( 3352ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong scheduler_config.CONFIG_SECTION, 'enable_archiving', type=bool) 3452ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong # Set the status of the queue entries to archiving or self final status 3552ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong if enable_archiving: 3652ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong status = models.HostQueueEntry.Status.ARCHIVING 3752ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong else: 3852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong status = self._final_status() 3952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 4052ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong for queue_entry in self.queue_entries: 4152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong queue_entry.set_status(status) 42a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 43a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 44a858a233889949263ded6d0d6578495aba54a9ebSimran Basiclass SiteDispatcher(object): 45a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 46a858a233889949263ded6d0d6578495aba54a9ebSimran Basi SiteDispatcher subclasses BaseDispatcher in monitor_db. 47a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 48a858a233889949263ded6d0d6578495aba54a9ebSimran Basi DEFAULT_REQUESTED_BY_USER_ID = 1 49a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 50a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 51a858a233889949263ded6d0d6578495aba54a9ebSimran Basi def _reverify_hosts_where(self, where, 52a858a233889949263ded6d0d6578495aba54a9ebSimran Basi print_message='Reverifying host %s'): 53a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 54a858a233889949263ded6d0d6578495aba54a9ebSimran Basi This is an altered version of _reverify_hosts_where the class to 55a858a233889949263ded6d0d6578495aba54a9ebSimran Basi models.SpecialTask.objects.create passes in an argument for 56a858a233889949263ded6d0d6578495aba54a9ebSimran Basi requested_by, in order to allow the Cleanup task to be created 57a858a233889949263ded6d0d6578495aba54a9ebSimran Basi properly. 58a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 59a858a233889949263ded6d0d6578495aba54a9ebSimran Basi full_where='locked = 0 AND invalid = 0 AND ' + where 60a858a233889949263ded6d0d6578495aba54a9ebSimran Basi for host in scheduler_models.Host.fetch(where=full_where): 61a858a233889949263ded6d0d6578495aba54a9ebSimran Basi if self.host_has_agent(host): 62a858a233889949263ded6d0d6578495aba54a9ebSimran Basi # host has already been recovered in some way 63a858a233889949263ded6d0d6578495aba54a9ebSimran Basi continue 64a858a233889949263ded6d0d6578495aba54a9ebSimran Basi if self._host_has_scheduled_special_task(host): 65a858a233889949263ded6d0d6578495aba54a9ebSimran Basi # host will have a special task scheduled on the next cycle 66a858a233889949263ded6d0d6578495aba54a9ebSimran Basi continue 67a858a233889949263ded6d0d6578495aba54a9ebSimran Basi if print_message: 68a858a233889949263ded6d0d6578495aba54a9ebSimran Basi logging.error(print_message, host.hostname) 69a858a233889949263ded6d0d6578495aba54a9ebSimran Basi try: 70a858a233889949263ded6d0d6578495aba54a9ebSimran Basi user = models.User.objects.get(login='autotest_system') 71a858a233889949263ded6d0d6578495aba54a9ebSimran Basi except models.User.DoesNotExist: 72a858a233889949263ded6d0d6578495aba54a9ebSimran Basi user = models.User.objects.get( 73a858a233889949263ded6d0d6578495aba54a9ebSimran Basi id=SiteDispatcher.DEFAULT_REQUESTED_BY_USER_ID) 74a858a233889949263ded6d0d6578495aba54a9ebSimran Basi models.SpecialTask.objects.create( 75a858a233889949263ded6d0d6578495aba54a9ebSimran Basi task=models.SpecialTask.Task.CLEANUP, 76a858a233889949263ded6d0d6578495aba54a9ebSimran Basi host=models.Host.objects.get(id=host.id), 773d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi requested_by=user) 783d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi 793d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi 803d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi def _check_for_unrecovered_verifying_entries(self): 813d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi queue_entries = scheduler_models.HostQueueEntry.fetch( 823d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi where='status = "%s"' % models.HostQueueEntry.Status.VERIFYING) 833d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi for queue_entry in queue_entries: 843d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi special_tasks = models.SpecialTask.objects.filter( 853d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi task__in=(models.SpecialTask.Task.CLEANUP, 863d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi models.SpecialTask.Task.VERIFY), 873d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi queue_entry__id=queue_entry.id, 883d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi is_complete=False) 893d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi if special_tasks.count() == 0: 903d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi logging.error('Unrecovered Verifying host queue entry: %s. ' 913d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi 'Setting status to Queued.', str(queue_entry)) 923d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # Essentially this host queue entry was set to be Verifying 933d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # however no special task exists for entry. This occurs if the 943d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # scheduler dies between changing the status and creating the 953d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # special task. By setting it to queued, the job can restart 963d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # from the beginning and proceed correctly. This is much more 973d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # preferable than having monitor_db not launching. 983d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi queue_entry.set_status('Queued')