152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis# Use of this source code is governed by a BSD-style license that can be 3e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis# found in the LICENSE file. 4e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis 507e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi#pylint: disable-msg=C0111 607e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi 7e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtisimport os 8a858a233889949263ded6d0d6578495aba54a9ebSimran Basiimport logging 9baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Dengimport time 10e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis 1152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hongfrom autotest_lib.client.common_lib import global_config 121e1c41b1b4a1b97c0b7086b8430856ed45e064d3Gabe Blackfrom autotest_lib.client.common_lib.cros.graphite import autotest_stats 1352ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hongfrom autotest_lib.frontend.afe import models 14bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basifrom autotest_lib.scheduler import email_manager 15a858a233889949263ded6d0d6578495aba54a9ebSimran Basifrom autotest_lib.scheduler import scheduler_config, scheduler_models 16da8c60af1e1e3ee97170c700d0b72991687e35a2Michael Liang 17e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis 18e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis# Override default parser with our site parser. 19e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtisdef parser_path(install_dir): 2007e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi """Return site implementation of parser. 2107e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi 2207e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi @param install_dir: installation directory. 2307e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi """ 24e5436f3561713c9f68eebc4c9347ffb59e36bd8fDale Curtis return os.path.join(install_dir, 'tko', 'site_parse') 2552ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 2652ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 2752ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hongclass SiteAgentTask(object): 2852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 2952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong SiteAgentTask subclasses BaseAgentTask in monitor_db. 3052ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 3152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 3252ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 3352ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong def _archive_results(self, queue_entries): 3452ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 3552ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong Set the status of queue_entries to ARCHIVING. 3652ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 3752ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong This method sets the status of the queue_entries to ARCHIVING 3852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong if the enable_archiving flag is true in global_config.ini. 3952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong Otherwise, it bypasses the archiving step and sets the queue entries 4052ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong to the final status of current step. 4152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong """ 4252ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong enable_archiving = global_config.global_config.get_config_value( 4352ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong scheduler_config.CONFIG_SECTION, 'enable_archiving', type=bool) 4452ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong # Set the status of the queue entries to archiving or self final status 4552ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong if enable_archiving: 4652ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong status = models.HostQueueEntry.Status.ARCHIVING 4752ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong else: 4852ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong status = self._final_status() 4952ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong 5052ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong for queue_entry in self.queue_entries: 5152ce11d6291bbbd1bde435a62afcaf364db1b502Yu-Ju Hong queue_entry.set_status(status) 52a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 53a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 54bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi def _check_queue_entry_statuses(self, queue_entries, allowed_hqe_statuses, 55bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi allowed_host_statuses=None): 56bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi """ 57bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi Forked from monitor_db.py 58bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi """ 59bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi class_name = self.__class__.__name__ 60bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi for entry in queue_entries: 61bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi if entry.status not in allowed_hqe_statuses: 62bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi # In the orignal code, here we raise an exception. In an 63bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi # effort to prevent downtime we will instead abort the job and 64bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi # send out an email notifying us this has occured. 65bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi error_message = ('%s attempting to start entry with invalid ' 66bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi 'status %s: %s. Aborting Job: %s.' 67bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi % (class_name, entry.status, entry, 68bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi entry.job)) 69bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi logging.error(error_message) 70bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi email_manager.manager.enqueue_notify_email( 71bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi 'Job Aborted - Invalid Host Queue Entry Status', 72bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi error_message) 73bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi entry.job.request_abort() 74bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi invalid_host_status = ( 75bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi allowed_host_statuses is not None 76bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi and entry.host.status not in allowed_host_statuses) 77bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi if invalid_host_status: 78bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi # In the orignal code, here we raise an exception. In an 79bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi # effort to prevent downtime we will instead abort the job and 80bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi # send out an email notifying us this has occured. 81bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi error_message = ('%s attempting to start on queue entry with ' 82bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi 'invalid host status %s: %s. Aborting Job: %s' 83bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi % (class_name, entry.host.status, entry, 84bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi entry.job)) 85bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi logging.error(error_message) 86bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi email_manager.manager.enqueue_notify_email( 87bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi 'Job Aborted - Invalid Host Status', error_message) 88bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi entry.job.request_abort() 89bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi 90bf2e21f65ba1e665554f7cb917bb06fdef238ad2Simran Basi 91a858a233889949263ded6d0d6578495aba54a9ebSimran Basiclass SiteDispatcher(object): 92a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 93a858a233889949263ded6d0d6578495aba54a9ebSimran Basi SiteDispatcher subclasses BaseDispatcher in monitor_db. 94a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 95a858a233889949263ded6d0d6578495aba54a9ebSimran Basi DEFAULT_REQUESTED_BY_USER_ID = 1 96a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 97a858a233889949263ded6d0d6578495aba54a9ebSimran Basi 981e1c41b1b4a1b97c0b7086b8430856ed45e064d3Gabe Black _timer = autotest_stats.Timer('scheduler') 99baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng _gauge = autotest_stats.Gauge('scheduler_rel') 100baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng _tick_start = None 10105d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller 10205d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller 10305d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller @_timer.decorate 10405d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller def tick(self): 105baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._tick_start = time.time() 10605d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller super(SiteDispatcher, self).tick() 107baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('tick', time.time() - self._tick_start) 10805d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller 1091d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1101d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _garbage_collection(self): 1111d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._garbage_collection() 112baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 113baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_garbage_collection', 114baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng time.time() - self._tick_start) 1151d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1161d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1171d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _run_cleanup(self): 1181d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._run_cleanup() 119baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 120baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_run_cleanup', time.time() - self._tick_start) 1211d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1221d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1231d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _find_aborting(self): 1241d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._find_aborting() 125baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 126baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_find_aborting', time.time() - self._tick_start) 1271d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1281d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1291d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _process_recurring_runs(self): 1301d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._process_recurring_runs() 131baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 132baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_process_recurring_runs', 133baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng time.time() - self._tick_start) 1341d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1351d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1361d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _schedule_delay_tasks(self): 1371d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._schedule_delay_tasks() 138baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 139baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_schedule_delay_tasks', 140baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng time.time() - self._tick_start) 1411d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1421d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1431d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _schedule_running_host_queue_entries(self): 1441d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._schedule_running_host_queue_entries() 145baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 146baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_schedule_running_host_queue_entries', 147baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng time.time() - self._tick_start) 1481d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1491d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1501d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _schedule_special_tasks(self): 1511d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._schedule_special_tasks() 152baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 153baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_schedule_special_tasks', 154baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng time.time() - self._tick_start) 1551d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1561d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1571d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _schedule_new_jobs(self): 1581d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._schedule_new_jobs() 159baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 160baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_schedule_new_jobs', 161baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng time.time() - self._tick_start) 162baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng 1631d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng 1641d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng @_timer.decorate 1651d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng def _handle_agents(self): 1661d6c2a0c4cc8d4a7e5e455e5bcddf36446868e49Fang Deng super(SiteDispatcher, self)._handle_agents() 167baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng if self._tick_start: 168baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng self._gauge.send('_handle_agents', time.time() - self._tick_start) 169baf4d37b1c12b1e23a0dcd9f6fad0449a0d11af6Fang Deng 17005d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller 171a858a233889949263ded6d0d6578495aba54a9ebSimran Basi def _reverify_hosts_where(self, where, 172a858a233889949263ded6d0d6578495aba54a9ebSimran Basi print_message='Reverifying host %s'): 173a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 174a858a233889949263ded6d0d6578495aba54a9ebSimran Basi This is an altered version of _reverify_hosts_where the class to 175a858a233889949263ded6d0d6578495aba54a9ebSimran Basi models.SpecialTask.objects.create passes in an argument for 17607e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi requested_by, in order to allow the Reset task to be created 177a858a233889949263ded6d0d6578495aba54a9ebSimran Basi properly. 178a858a233889949263ded6d0d6578495aba54a9ebSimran Basi """ 179a858a233889949263ded6d0d6578495aba54a9ebSimran Basi full_where='locked = 0 AND invalid = 0 AND ' + where 180a858a233889949263ded6d0d6578495aba54a9ebSimran Basi for host in scheduler_models.Host.fetch(where=full_where): 181a858a233889949263ded6d0d6578495aba54a9ebSimran Basi if self.host_has_agent(host): 182a858a233889949263ded6d0d6578495aba54a9ebSimran Basi # host has already been recovered in some way 183a858a233889949263ded6d0d6578495aba54a9ebSimran Basi continue 184a858a233889949263ded6d0d6578495aba54a9ebSimran Basi if self._host_has_scheduled_special_task(host): 185a858a233889949263ded6d0d6578495aba54a9ebSimran Basi # host will have a special task scheduled on the next cycle 186a858a233889949263ded6d0d6578495aba54a9ebSimran Basi continue 187a858a233889949263ded6d0d6578495aba54a9ebSimran Basi if print_message: 188a858a233889949263ded6d0d6578495aba54a9ebSimran Basi logging.error(print_message, host.hostname) 189a858a233889949263ded6d0d6578495aba54a9ebSimran Basi try: 190a858a233889949263ded6d0d6578495aba54a9ebSimran Basi user = models.User.objects.get(login='autotest_system') 191a858a233889949263ded6d0d6578495aba54a9ebSimran Basi except models.User.DoesNotExist: 192a858a233889949263ded6d0d6578495aba54a9ebSimran Basi user = models.User.objects.get( 193a858a233889949263ded6d0d6578495aba54a9ebSimran Basi id=SiteDispatcher.DEFAULT_REQUESTED_BY_USER_ID) 194a858a233889949263ded6d0d6578495aba54a9ebSimran Basi models.SpecialTask.objects.create( 19507e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi task=models.SpecialTask.Task.RESET, 196a858a233889949263ded6d0d6578495aba54a9ebSimran Basi host=models.Host.objects.get(id=host.id), 1973d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi requested_by=user) 1983d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi 1993d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi 2003d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi def _check_for_unrecovered_verifying_entries(self): 20107e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi # Verify is replaced by Reset. 2023d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi queue_entries = scheduler_models.HostQueueEntry.fetch( 20307e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi where='status = "%s"' % models.HostQueueEntry.Status.RESETTING) 2043d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi for queue_entry in queue_entries: 2053d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi special_tasks = models.SpecialTask.objects.filter( 2063d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi task__in=(models.SpecialTask.Task.CLEANUP, 20707e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi models.SpecialTask.Task.VERIFY, 20807e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi models.SpecialTask.Task.RESET), 2093d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi queue_entry__id=queue_entry.id, 2103d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi is_complete=False) 2113d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi if special_tasks.count() == 0: 21207e09aff0baf871b33e5479e337e5e3e0523b729Dan Shi logging.error('Unrecovered Resetting host queue entry: %s. ' 2133d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi 'Setting status to Queued.', str(queue_entry)) 2143d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # Essentially this host queue entry was set to be Verifying 2153d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # however no special task exists for entry. This occurs if the 2163d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # scheduler dies between changing the status and creating the 2173d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # special task. By setting it to queued, the job can restart 2183d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # from the beginning and proceed correctly. This is much more 2193d89973417ed013f328b57a7f750dcc2a788fd21Simran Basi # preferable than having monitor_db not launching. 22005d7b4cd023d4dcaee3c0744dc960f3e01ec6fbeAlex Miller queue_entry.set_status('Queued') 221