1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6import logging 7import time 8from multiprocessing import pool 9 10import base_event, board_enumerator, build_event, deduping_scheduler 11import task, timed_event 12 13import common 14from autotest_lib.client.common_lib import utils 15from autotest_lib.server import utils 16 17try: 18 from chromite.lib import metrics 19except ImportError: 20 metrics = utils.metrics_mock 21 22 23POOL_SIZE = 32 24 25class Driver(object): 26 """Implements the main loop of the suite_scheduler. 27 28 @var EVENT_CLASSES: list of the event classes Driver supports. 29 @var _LOOP_INTERVAL_SECONDS: seconds to wait between loop iterations. 30 31 @var _scheduler: a DedupingScheduler, used to schedule jobs with the AFE. 32 @var _enumerator: a BoardEnumerator, used to list plaforms known to 33 the AFE 34 @var _events: dict of BaseEvents to be handled each time through main loop. 35 """ 36 37 EVENT_CLASSES = [timed_event.Nightly, timed_event.Weekly, 38 build_event.NewBuild] 39 _LOOP_INTERVAL_SECONDS = 5 * 60 40 41 # Cache for known ChromeOS boards. The cache helps to avoid unnecessary 42 # repeated calls to Launch Control API. 43 _cros_boards = set() 44 45 def __init__(self, scheduler, enumerator, is_sanity=False): 46 """Constructor 47 48 @param scheduler: an instance of deduping_scheduler.DedupingScheduler. 49 @param enumerator: an instance of board_enumerator.BoardEnumerator. 50 @param is_sanity: Set to True if the driver is created for sanity check. 51 Default is set to False. 52 """ 53 self._scheduler = scheduler 54 self._enumerator = enumerator 55 task.TotMilestoneManager.is_sanity = is_sanity 56 57 58 def RereadAndReprocessConfig(self, config, mv): 59 """Re-read config, re-populate self._events and recreate task lists. 60 61 @param config: an instance of ForgivingConfigParser. 62 @param mv: an instance of ManifestVersions. 63 """ 64 config.reread() 65 new_events = self._CreateEventsWithTasks(config, mv) 66 for keyword, event in self._events.iteritems(): 67 event.Merge(new_events[keyword]) 68 69 70 def SetUpEventsAndTasks(self, config, mv): 71 """Populate self._events and create task lists from config. 72 73 @param config: an instance of ForgivingConfigParser. 74 @param mv: an instance of ManifestVersions. 75 """ 76 self._events = self._CreateEventsWithTasks(config, mv) 77 78 79 def _CreateEventsWithTasks(self, config, mv): 80 """Create task lists from config, and assign to newly-minted events. 81 82 Calling multiple times should start afresh each time. 83 84 @param config: an instance of ForgivingConfigParser. 85 @param mv: an instance of ManifestVersions. 86 """ 87 events = {} 88 for klass in self.EVENT_CLASSES: 89 events[klass.KEYWORD] = klass.CreateFromConfig(config, mv) 90 91 tasks = self.TasksFromConfig(config) 92 for keyword, task_list in tasks.iteritems(): 93 if keyword in events: 94 events[keyword].tasks = task_list 95 else: 96 logging.warning('%s, is an unknown keyword.', keyword) 97 return events 98 99 100 def TasksFromConfig(self, config): 101 """Generate a dict of {event_keyword: [tasks]} mappings from |config|. 102 103 For each section in |config| that encodes a Task, instantiate a Task 104 object. Determine the event that Task is supposed to run_on and 105 append the object to a list associated with the appropriate event 106 keyword. Return a dictionary of these keyword: list of task mappings. 107 108 @param config: a ForgivingConfigParser containing tasks to be parsed. 109 @return dict of {event_keyword: [tasks]} mappings. 110 @raise MalformedConfigEntry on a task parsing error. 111 """ 112 tasks = {} 113 for section in config.sections(): 114 if not base_event.HonoredSection(section): 115 try: 116 keyword, new_task = task.Task.CreateFromConfigSection( 117 config, section) 118 except task.MalformedConfigEntry as e: 119 logging.warning('%s is malformed: %s', section, str(e)) 120 continue 121 tasks.setdefault(keyword, []).append(new_task) 122 return tasks 123 124 125 def RunForever(self, config, mv): 126 """Main loop of the scheduler. Runs til the process is killed. 127 128 @param config: an instance of ForgivingConfigParser. 129 @param mv: an instance of manifest_versions.ManifestVersions. 130 """ 131 for event in self._events.itervalues(): 132 event.Prepare() 133 while True: 134 try: 135 self.HandleEventsOnce(mv) 136 except board_enumerator.EnumeratorException as e: 137 logging.warning('Failed to enumerate boards: %r', e) 138 mv.Update() 139 task.TotMilestoneManager().refresh() 140 time.sleep(self._LOOP_INTERVAL_SECONDS) 141 self.RereadAndReprocessConfig(config, mv) 142 143 144 @staticmethod 145 def HandleBoard(inputs): 146 """Handle event based on given inputs. 147 148 @param inputs: A dictionary of the arguments needed to handle an event. 149 Keys include: 150 scheduler: a DedupingScheduler, used to schedule jobs with the AFE. 151 event: An event object to be handled. 152 board: Name of the board. 153 """ 154 scheduler = inputs['scheduler'] 155 event = inputs['event'] 156 board = inputs['board'] 157 158 # Try to get builds from LaunchControl first. If failed, the board could 159 # be ChromeOS. Use the cache Driver._cros_boards to avoid unnecessary 160 # repeated call to LaunchControl API. 161 launch_control_builds = None 162 if board not in Driver._cros_boards: 163 launch_control_builds = event.GetLaunchControlBuildsForBoard(board) 164 if launch_control_builds: 165 event.Handle(scheduler, branch_builds=None, board=board, 166 launch_control_builds=launch_control_builds) 167 else: 168 branch_builds = event.GetBranchBuildsForBoard(board) 169 if branch_builds: 170 Driver._cros_boards.add(board) 171 logging.info('Found ChromeOS build for board %s. This should ' 172 'be a ChromeOS board.', board) 173 event.Handle(scheduler, branch_builds, board) 174 logging.info('Finished handling %s event for board %s', event.keyword, 175 board) 176 177 @metrics.SecondsTimerDecorator('chromeos/autotest/suite_scheduler/' 178 'handle_events_once_duration') 179 def HandleEventsOnce(self, mv): 180 """One turn through the loop. Separated out for unit testing. 181 182 @param mv: an instance of manifest_versions.ManifestVersions. 183 @raise EnumeratorException if we can't enumerate any supported boards. 184 """ 185 boards = self._enumerator.Enumerate() 186 logging.info('%d boards currently in the lab: %r', len(boards), boards) 187 thread_pool = pool.ThreadPool(POOL_SIZE) 188 with contextlib.closing(thread_pool): 189 for e in self._events.itervalues(): 190 if not e.ShouldHandle(): 191 continue 192 # Reset the value of delay_minutes, as this is the beginning of 193 # handling an event for all boards. 194 self._scheduler.delay_minutes = 0 195 self._scheduler.delay_minutes_interval = ( 196 deduping_scheduler.DELAY_MINUTES_INTERVAL) 197 logging.info('Handling %s event for %d boards', e.keyword, 198 len(boards)) 199 args = [] 200 for board in boards: 201 args.append({'scheduler': self._scheduler, 202 'event': e, 203 'board': board}) 204 thread_pool.map(self.HandleBoard, args) 205 logging.info('Finished handling %s event for %d boards', 206 e.keyword, len(boards)) 207 e.UpdateCriteria() 208 209 210 def ForceEventsOnceForBuild(self, keywords, build_name, 211 os_type=task.OS_TYPE_CROS): 212 """Force events with provided keywords to happen, with given build. 213 214 @param keywords: iterable of event keywords to force 215 @param build_name: instead of looking up builds to test, test this one. 216 @param os_type: Type of the OS to test, default to cros. 217 """ 218 branch_builds = None 219 launch_control_builds = None 220 if os_type == task.OS_TYPE_CROS: 221 board, type, milestone, manifest = utils.ParseBuildName(build_name) 222 branch_builds = {task.PickBranchName(type, milestone): [build_name]} 223 logging.info('Testing build R%s-%s on %s', milestone, manifest, 224 board) 225 else: 226 logging.info('Build is not a ChromeOS build, try to parse as a ' 227 'Launch Control build.') 228 _,target,_ = utils.parse_launch_control_build(build_name) 229 board = utils.parse_launch_control_target(target)[0] 230 # Translate board name in build target to the actual board name. 231 board = utils.ANDROID_TARGET_TO_BOARD_MAP.get(board, board) 232 launch_control_builds = [build_name] 233 logging.info('Testing Launch Control build %s on %s', build_name, 234 board) 235 236 for e in self._events.itervalues(): 237 if e.keyword in keywords: 238 e.Handle(self._scheduler, branch_builds, board, force=True, 239 launch_control_builds=launch_control_builds) 240