1#!/usr/bin/python 2# 3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""CrOS suite scheduler. Will schedule suites based on configured triggers. 8 9The Scheduler understands two main primitives: Events and Tasks. Each stanza 10in the config file specifies a Task that triggers on a given Event. 11 12Events: 13 The scheduler supports two kinds of Events: timed events, and 14 build system events -- like a particular build artifact becoming available. 15 Every Event has a set of Tasks that get run whenever the event happens. 16 17Tasks: 18 Basically, event handlers. A Task is specified in the config file like so: 19 [NightlyPower] 20 suite: power 21 run_on: nightly 22 pool: remote_power 23 branch_specs: >=R20,factory 24 25 This specifies a Task that gets run whenever the 'nightly' event occurs. 26 The Task schedules a suite of tests called 'power' on the pool of machines 27 called 'remote_power', for both the factory branch and all active release 28 branches from R20 on. 29 30 31On startup, the scheduler reads in a config file that provides a few 32parameters for certain supported Events (the time/day of the 'weekly' 33and 'nightly' triggers, for example), and configures all the Tasks 34that will be in play. 35""" 36 37import getpass, logging, logging.handlers, optparse, os, re, signal, sys 38import traceback 39import common 40import board_enumerator, deduping_scheduler, driver, forgiving_config_parser 41import manifest_versions, sanity, task 42from autotest_lib.client.common_lib import global_config 43from autotest_lib.client.common_lib import utils 44from autotest_lib.client.common_lib import logging_config, logging_manager 45from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 46try: 47 from autotest_lib.frontend import setup_django_environment 48 # server_manager_utils depend on django which 49 # may not be available when people run checks with --sanity 50 from autotest_lib.site_utils import server_manager_utils 51except ImportError: 52 server_manager_utils = None 53 logging.debug('Could not load server_manager_utils module, expected ' 54 'if you are running sanity check or pre-submit hook') 55 56try: 57 from chromite.lib import ts_mon_config 58except ImportError: 59 ts_mon_config = utils.metrics_mock 60 61 62CONFIG_SECTION = 'SCHEDULER' 63 64CONFIG_SECTION_SERVER = 'SERVER' 65 66 67def signal_handler(signal, frame): 68 """Singnal hanlder to exit gracefully. 69 70 @param signal: signum 71 @param frame: stack frame object 72 """ 73 logging.info('Signal %d received. Exiting gracefully...', signal) 74 sys.exit(0) 75 76 77class SeverityFilter(logging.Filter): 78 """Filters out messages of anything other than self._level""" 79 def __init__(self, level): 80 self._level = level 81 82 83 def filter(self, record): 84 """Causes only messages of |self._level| severity to be logged.""" 85 return record.levelno == self._level 86 87 88class SchedulerLoggingConfig(logging_config.LoggingConfig): 89 """Configure loggings for scheduler, e.g., email setup.""" 90 def __init__(self): 91 super(SchedulerLoggingConfig, self).__init__() 92 self._from_address = global_config.global_config.get_config_value( 93 CONFIG_SECTION, "notify_email_from", default=getpass.getuser()) 94 95 self._notify_address = global_config.global_config.get_config_value( 96 CONFIG_SECTION, "notify_email", 97 default='chromeos-lab-admins@google.com') 98 99 self._smtp_server = global_config.global_config.get_config_value( 100 CONFIG_SECTION_SERVER, "smtp_server", default='localhost') 101 102 self._smtp_port = global_config.global_config.get_config_value( 103 CONFIG_SECTION_SERVER, "smtp_port", default=None) 104 105 self._smtp_user = global_config.global_config.get_config_value( 106 CONFIG_SECTION_SERVER, "smtp_user", default='') 107 108 self._smtp_password = global_config.global_config.get_config_value( 109 CONFIG_SECTION_SERVER, "smtp_password", default='') 110 111 112 @classmethod 113 def get_log_name(cls): 114 """Get timestamped log name of suite_scheduler, e.g., 115 suite_scheduler.log.2013-2-1-02-05-06. 116 117 @param cls: class 118 """ 119 return cls.get_timestamped_log_name('suite_scheduler') 120 121 122 def add_smtp_handler(self, subject, level=logging.ERROR): 123 """Add smtp handler to logging handler to trigger email when logging 124 occurs. 125 126 @param subject: email subject. 127 @param level: level of logging to trigger smtp handler. 128 """ 129 if not self._smtp_user or not self._smtp_password: 130 creds = None 131 else: 132 creds = (self._smtp_user, self._smtp_password) 133 server = self._smtp_server 134 if self._smtp_port: 135 server = (server, self._smtp_port) 136 137 handler = logging.handlers.SMTPHandler(server, 138 self._from_address, 139 [self._notify_address], 140 subject, 141 creds) 142 handler.setLevel(level) 143 # We want to send mail for the given level, and only the given level. 144 # One can add more handlers to send messages for other levels. 145 handler.addFilter(SeverityFilter(level)) 146 handler.setFormatter( 147 logging.Formatter('%(asctime)s %(levelname)-5s %(message)s')) 148 self.logger.addHandler(handler) 149 return handler 150 151 152 def configure_logging(self, log_dir=None): 153 super(SchedulerLoggingConfig, self).configure_logging(use_console=True) 154 155 if not log_dir: 156 return 157 base = self.get_log_name() 158 159 self.add_file_handler(base + '.DEBUG', logging.DEBUG, log_dir=log_dir) 160 self.add_file_handler(base + '.INFO', logging.INFO, log_dir=log_dir) 161 self.add_smtp_handler('Suite scheduler ERROR', logging.ERROR) 162 self.add_smtp_handler('Suite scheduler WARNING', logging.WARN) 163 164 165def parse_options(): 166 """Parse commandline options.""" 167 usage = "usage: %prog [options]" 168 parser = optparse.OptionParser(usage=usage) 169 parser.add_option('-f', '--config_file', dest='config_file', 170 metavar='/path/to/config', default='suite_scheduler.ini', 171 help='Scheduler config. Defaults to suite_scheduler.ini') 172 parser.add_option('-e', '--events', dest='events', 173 metavar='list,of,events', 174 help='Handle listed events once each, then exit. '\ 175 'Must also specify a build to test.') 176 parser.add_option('-i', '--build', dest='build', 177 help='If handling a list of events, the build to test.'\ 178 ' Ignored otherwise.') 179 parser.add_option('-o', '--os_type', dest='os_type', 180 default=task.OS_TYPE_CROS, 181 help='If handling a list of events, the OS type to test.'\ 182 ' Ignored otherwise. This argument allows the test to ' 183 'know if it\'s testing ChromeOS or Launch Control ' 184 'builds. suite scheduler that runs without a build ' 185 'specified(using -i), does not need this argument.') 186 parser.add_option('-d', '--log_dir', dest='log_dir', 187 help='Log to a file in the specified directory.') 188 parser.add_option('-l', '--list_events', dest='list', 189 action='store_true', default=False, 190 help='List supported events and exit.') 191 parser.add_option('-r', '--repo_dir', dest='tmp_repo_dir', default=None, 192 help=('Path to a tmpdir containing manifest versions. ' 193 'This option is only used for testing.')) 194 parser.add_option('-t', '--sanity', dest='sanity', action='store_true', 195 default=False, 196 help='Check the config file for any issues.') 197 parser.add_option('-b', '--file_bug', dest='file_bug', action='store_true', 198 default=False, 199 help='File bugs for known suite scheduling exceptions.') 200 201 202 options, args = parser.parse_args() 203 return parser, options, args 204 205 206def main(): 207 """Entry point for suite_scheduler.py""" 208 signal.signal(signal.SIGINT, signal_handler) 209 signal.signal(signal.SIGHUP, signal_handler) 210 signal.signal(signal.SIGTERM, signal_handler) 211 212 parser, options, args = parse_options() 213 if args or options.events and not options.build: 214 parser.print_help() 215 return 1 216 217 if options.config_file and not os.path.exists(options.config_file): 218 logging.error('Specified config file %s does not exist.', 219 options.config_file) 220 return 1 221 222 config = forgiving_config_parser.ForgivingConfigParser() 223 config.read(options.config_file) 224 225 if options.list: 226 print 'Supported events:' 227 for event_class in driver.Driver.EVENT_CLASSES: 228 print ' ', event_class.KEYWORD 229 return 0 230 231 # If we're just sanity checking, we can stop after we've parsed the 232 # config file. 233 if options.sanity: 234 # config_file_getter generates a high amount of noise at DEBUG level 235 logging.getLogger().setLevel(logging.WARNING) 236 d = driver.Driver(None, None, True) 237 d.SetUpEventsAndTasks(config, None) 238 tasks_per_event = d.TasksFromConfig(config) 239 # flatten [[a]] -> [a] 240 tasks = [x for y in tasks_per_event.values() for x in y] 241 control_files_exist = sanity.CheckControlFileExistence(tasks) 242 return control_files_exist 243 244 logging_manager.configure_logging(SchedulerLoggingConfig(), 245 log_dir=options.log_dir) 246 if not options.log_dir: 247 logging.info('Not logging to a file, as --log_dir was not passed.') 248 249 # If server database is enabled, check if the server has role 250 # `suite_scheduler`. If the server does not have suite_scheduler role, 251 # exception will be raised and suite scheduler will not continue to run. 252 if not server_manager_utils: 253 raise ImportError( 254 'Could not import autotest_lib.site_utils.server_manager_utils') 255 if server_manager_utils.use_server_db(): 256 server_manager_utils.confirm_server_has_role(hostname='localhost', 257 role='suite_scheduler') 258 259 afe_server = global_config.global_config.get_config_value( 260 CONFIG_SECTION_SERVER, "suite_scheduler_afe", default=None) 261 262 afe = frontend_wrappers.RetryingAFE( 263 server=afe_server, timeout_min=10, delay_sec=5, debug=False) 264 logging.info('Connecting to: %s' , afe.server) 265 enumerator = board_enumerator.BoardEnumerator(afe) 266 scheduler = deduping_scheduler.DedupingScheduler(afe, options.file_bug) 267 mv = manifest_versions.ManifestVersions(options.tmp_repo_dir) 268 d = driver.Driver(scheduler, enumerator) 269 d.SetUpEventsAndTasks(config, mv) 270 271 # Set up metrics upload for Monarch. 272 ts_mon_config.SetupTsMonGlobalState('autotest_suite_scheduler') 273 274 try: 275 if options.events: 276 # Act as though listed events have just happened. 277 keywords = re.split('\s*,\s*', options.events) 278 if not options.tmp_repo_dir: 279 logging.warn('To run a list of events, you may need to use ' 280 '--repo_dir to specify a folder that already has ' 281 'manifest repo set up. This is needed for suites ' 282 'requiring firmware update.') 283 logging.info('Forcing events: %r', keywords) 284 d.ForceEventsOnceForBuild(keywords, options.build, options.os_type) 285 else: 286 if not options.tmp_repo_dir: 287 mv.Initialize() 288 d.RunForever(config, mv) 289 except Exception as e: 290 logging.error('Fatal exception in suite_scheduler: %r\n%s', e, 291 traceback.format_exc()) 292 return 1 293 294if __name__ == "__main__": 295 sys.exit(main()) 296