1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import contextlib
6import logging
7import time
8from multiprocessing import pool
9
10import base_event, board_enumerator, build_event, deduping_scheduler
11import task, timed_event
12
13import common
14from autotest_lib.client.common_lib import utils
15from autotest_lib.server import utils
16
17try:
18    from chromite.lib import metrics
19except ImportError:
20    metrics = utils.metrics_mock
21
22
23POOL_SIZE = 32
24
25class Driver(object):
26    """Implements the main loop of the suite_scheduler.
27
28    @var EVENT_CLASSES: list of the event classes Driver supports.
29    @var _LOOP_INTERVAL_SECONDS: seconds to wait between loop iterations.
30
31    @var _scheduler: a DedupingScheduler, used to schedule jobs with the AFE.
32    @var _enumerator: a BoardEnumerator, used to list plaforms known to
33                      the AFE
34    @var _events: dict of BaseEvents to be handled each time through main loop.
35    """
36
37    EVENT_CLASSES = [timed_event.Nightly, timed_event.Weekly,
38                     build_event.NewBuild]
39    _LOOP_INTERVAL_SECONDS = 5 * 60
40
41    # Cache for known ChromeOS boards. The cache helps to avoid unnecessary
42    # repeated calls to Launch Control API.
43    _cros_boards = set()
44
45    def __init__(self, scheduler, enumerator, is_sanity=False):
46        """Constructor
47
48        @param scheduler: an instance of deduping_scheduler.DedupingScheduler.
49        @param enumerator: an instance of board_enumerator.BoardEnumerator.
50        @param is_sanity: Set to True if the driver is created for sanity check.
51                          Default is set to False.
52        """
53        self._scheduler = scheduler
54        self._enumerator = enumerator
55        task.TotMilestoneManager.is_sanity = is_sanity
56
57
58    def RereadAndReprocessConfig(self, config, mv):
59        """Re-read config, re-populate self._events and recreate task lists.
60
61        @param config: an instance of ForgivingConfigParser.
62        @param mv: an instance of ManifestVersions.
63        """
64        config.reread()
65        new_events = self._CreateEventsWithTasks(config, mv)
66        for keyword, event in self._events.iteritems():
67            event.Merge(new_events[keyword])
68
69
70    def SetUpEventsAndTasks(self, config, mv):
71        """Populate self._events and create task lists from config.
72
73        @param config: an instance of ForgivingConfigParser.
74        @param mv: an instance of ManifestVersions.
75        """
76        self._events = self._CreateEventsWithTasks(config, mv)
77
78
79    def _CreateEventsWithTasks(self, config, mv):
80        """Create task lists from config, and assign to newly-minted events.
81
82        Calling multiple times should start afresh each time.
83
84        @param config: an instance of ForgivingConfigParser.
85        @param mv: an instance of ManifestVersions.
86        """
87        events = {}
88        for klass in self.EVENT_CLASSES:
89            events[klass.KEYWORD] = klass.CreateFromConfig(config, mv)
90
91        tasks = self.TasksFromConfig(config)
92        for keyword, task_list in tasks.iteritems():
93            if keyword in events:
94                events[keyword].tasks = task_list
95            else:
96                logging.warning('%s, is an unknown keyword.', keyword)
97        return events
98
99
100    def TasksFromConfig(self, config):
101        """Generate a dict of {event_keyword: [tasks]} mappings from |config|.
102
103        For each section in |config| that encodes a Task, instantiate a Task
104        object.  Determine the event that Task is supposed to run_on and
105        append the object to a list associated with the appropriate event
106        keyword.  Return a dictionary of these keyword: list of task mappings.
107
108        @param config: a ForgivingConfigParser containing tasks to be parsed.
109        @return dict of {event_keyword: [tasks]} mappings.
110        @raise MalformedConfigEntry on a task parsing error.
111        """
112        tasks = {}
113        for section in config.sections():
114            if not base_event.HonoredSection(section):
115                try:
116                    keyword, new_task = task.Task.CreateFromConfigSection(
117                            config, section)
118                except task.MalformedConfigEntry as e:
119                    logging.warning('%s is malformed: %s', section, str(e))
120                    continue
121                tasks.setdefault(keyword, []).append(new_task)
122        return tasks
123
124
125    def RunForever(self, config, mv):
126        """Main loop of the scheduler.  Runs til the process is killed.
127
128        @param config: an instance of ForgivingConfigParser.
129        @param mv: an instance of manifest_versions.ManifestVersions.
130        """
131        for event in self._events.itervalues():
132            event.Prepare()
133        while True:
134            try:
135                self.HandleEventsOnce(mv)
136            except board_enumerator.EnumeratorException as e:
137                logging.warning('Failed to enumerate boards: %r', e)
138            mv.Update()
139            task.TotMilestoneManager().refresh()
140            time.sleep(self._LOOP_INTERVAL_SECONDS)
141            self.RereadAndReprocessConfig(config, mv)
142
143
144    @staticmethod
145    def HandleBoard(inputs):
146        """Handle event based on given inputs.
147
148        @param inputs: A dictionary of the arguments needed to handle an event.
149            Keys include:
150            scheduler: a DedupingScheduler, used to schedule jobs with the AFE.
151            event: An event object to be handled.
152            board: Name of the board.
153        """
154        scheduler = inputs['scheduler']
155        event = inputs['event']
156        board = inputs['board']
157
158        # Try to get builds from LaunchControl first. If failed, the board could
159        # be ChromeOS. Use the cache Driver._cros_boards to avoid unnecessary
160        # repeated call to LaunchControl API.
161        launch_control_builds = None
162        if board not in Driver._cros_boards:
163            launch_control_builds = event.GetLaunchControlBuildsForBoard(board)
164        if launch_control_builds:
165            event.Handle(scheduler, branch_builds=None, board=board,
166                         launch_control_builds=launch_control_builds)
167        else:
168            branch_builds = event.GetBranchBuildsForBoard(board)
169            if branch_builds:
170                Driver._cros_boards.add(board)
171                logging.info('Found ChromeOS build for board %s. This should '
172                             'be a ChromeOS board.', board)
173            event.Handle(scheduler, branch_builds, board)
174        logging.info('Finished handling %s event for board %s', event.keyword,
175                     board)
176
177    @metrics.SecondsTimerDecorator('chromeos/autotest/suite_scheduler/'
178                                   'handle_events_once_duration')
179    def HandleEventsOnce(self, mv):
180        """One turn through the loop.  Separated out for unit testing.
181
182        @param mv: an instance of manifest_versions.ManifestVersions.
183        @raise EnumeratorException if we can't enumerate any supported boards.
184        """
185        boards = self._enumerator.Enumerate()
186        logging.info('%d boards currently in the lab: %r', len(boards), boards)
187        thread_pool = pool.ThreadPool(POOL_SIZE)
188        with contextlib.closing(thread_pool):
189            for e in self._events.itervalues():
190                if not e.ShouldHandle():
191                    continue
192                # Reset the value of delay_minutes, as this is the beginning of
193                # handling an event for all boards.
194                self._scheduler.delay_minutes = 0
195                self._scheduler.delay_minutes_interval = (
196                        deduping_scheduler.DELAY_MINUTES_INTERVAL)
197                logging.info('Handling %s event for %d boards', e.keyword,
198                             len(boards))
199                args = []
200                for board in boards:
201                    args.append({'scheduler': self._scheduler,
202                                 'event': e,
203                                 'board': board})
204                thread_pool.map(self.HandleBoard, args)
205                logging.info('Finished handling %s event for %d boards',
206                             e.keyword, len(boards))
207                e.UpdateCriteria()
208
209
210    def ForceEventsOnceForBuild(self, keywords, build_name,
211                                os_type=task.OS_TYPE_CROS):
212        """Force events with provided keywords to happen, with given build.
213
214        @param keywords: iterable of event keywords to force
215        @param build_name: instead of looking up builds to test, test this one.
216        @param os_type: Type of the OS to test, default to cros.
217        """
218        branch_builds = None
219        launch_control_builds = None
220        if os_type == task.OS_TYPE_CROS:
221            board, type, milestone, manifest = utils.ParseBuildName(build_name)
222            branch_builds = {task.PickBranchName(type, milestone): [build_name]}
223            logging.info('Testing build R%s-%s on %s', milestone, manifest,
224                         board)
225        else:
226            logging.info('Build is not a ChromeOS build, try to parse as a '
227                         'Launch Control build.')
228            _,target,_ = utils.parse_launch_control_build(build_name)
229            board = utils.parse_launch_control_target(target)[0]
230            # Translate board name in build target to the actual board name.
231            board = utils.ANDROID_TARGET_TO_BOARD_MAP.get(board, board)
232            launch_control_builds = [build_name]
233            logging.info('Testing Launch Control build %s on %s', build_name,
234                         board)
235
236        for e in self._events.itervalues():
237            if e.keyword in keywords:
238                e.Handle(self._scheduler, branch_builds, board, force=True,
239                         launch_control_builds=launch_control_builds)
240