1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6import logging
7import re
8import subprocess
9
10import base_event
11import deduping_scheduler
12import driver
13import error
14import manifest_versions
15from distutils import version
16from constants import Labels
17from constants import Builds
18
19import common
20from autotest_lib.client.common_lib import global_config
21from autotest_lib.client.common_lib import priorities
22from autotest_lib.server import utils as server_utils
23from autotest_lib.server.cros.dynamic_suite import constants
24
25
26CONFIG = global_config.global_config
27
28OS_TYPE_CROS = 'cros'
29OS_TYPE_BRILLO = 'brillo'
30OS_TYPE_ANDROID = 'android'
31OS_TYPES = {OS_TYPE_CROS, OS_TYPE_BRILLO, OS_TYPE_ANDROID}
32OS_TYPES_LAUNCH_CONTROL = {OS_TYPE_BRILLO, OS_TYPE_ANDROID}
33
34_WEEKDAYS = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
35             'Sunday']
36
37# regex to parse the dut count from board label. Note that the regex makes sure
38# there is only one board specified in `boards`
39TESTBED_DUT_COUNT_REGEX = '[^,]*-(\d+)'
40
41BARE_BRANCHES = ['factory', 'firmware']
42
43
44def PickBranchName(type, milestone):
45    """Pick branch name. If type is among BARE_BRANCHES, return type,
46    otherwise, return milestone.
47
48    @param type: type of the branch, e.g., 'release', 'factory', or 'firmware'
49    @param milestone: CrOS milestone number
50    """
51    if type in BARE_BRANCHES:
52        return type
53    return milestone
54
55
56class TotMilestoneManager(object):
57    """A class capable of converting tot string to milestone numbers.
58
59    This class is used as a cache for the tot milestone, so we don't
60    repeatedly hit google storage for all O(100) tasks in suite
61    scheduler's ini file.
62    """
63
64    __metaclass__ = server_utils.Singleton
65
66    # True if suite_scheduler is running for sanity check. When it's set to
67    # True, the code won't make gsutil call to get the actual tot milestone to
68    # avoid dependency on the installation of gsutil to run sanity check.
69    is_sanity = False
70
71
72    @staticmethod
73    def _tot_milestone():
74        """Get the tot milestone, eg: R40
75
76        @returns: A string representing the Tot milestone as declared by
77            the LATEST_BUILD_URL, or an empty string if LATEST_BUILD_URL
78            doesn't exist.
79        """
80        if TotMilestoneManager.is_sanity:
81            logging.info('suite_scheduler is running for sanity purpose, no '
82                         'need to get the actual tot milestone string.')
83            return 'R40'
84
85        cmd = ['gsutil', 'cat', constants.LATEST_BUILD_URL]
86        proc = subprocess.Popen(
87                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
88        stdout, stderr = proc.communicate()
89        if proc.poll():
90            logging.warning('Failed to get latest build: %s', stderr)
91            return ''
92        return stdout.split('-')[0]
93
94
95    def refresh(self):
96        """Refresh the tot milestone string managed by this class."""
97        self.tot = self._tot_milestone()
98
99
100    def __init__(self):
101        """Initialize a TotMilestoneManager."""
102        self.refresh()
103
104
105    def ConvertTotSpec(self, tot_spec):
106        """Converts a tot spec to the appropriate milestone.
107
108        Assume tot is R40:
109        tot   -> R40
110        tot-1 -> R39
111        tot-2 -> R38
112        tot-(any other numbers) -> R40
113
114        With the last option one assumes that a malformed configuration that has
115        'tot' in it, wants at least tot.
116
117        @param tot_spec: A string representing the tot spec.
118        @raises MalformedConfigEntry: If the tot_spec doesn't match the
119            expected format.
120        """
121        tot_spec = tot_spec.lower()
122        match = re.match('(tot)[-]?(1$|2$)?', tot_spec)
123        if not match:
124            raise error.MalformedConfigEntry(
125                    "%s isn't a valid branch spec." % tot_spec)
126        tot_mstone = self.tot
127        num_back = match.groups()[1]
128        if num_back:
129            tot_mstone_num = tot_mstone.lstrip('R')
130            tot_mstone = tot_mstone.replace(
131                    tot_mstone_num, str(int(tot_mstone_num)-int(num_back)))
132        return tot_mstone
133
134
135class Task(object):
136    """Represents an entry from the scheduler config.  Can schedule itself.
137
138    Each entry from the scheduler config file maps one-to-one to a
139    Task.  Each instance has enough info to schedule itself
140    on-demand with the AFE.
141
142    This class also overrides __hash__() and all comparator methods to enable
143    correct use in dicts, sets, etc.
144    """
145
146
147    @staticmethod
148    def CreateFromConfigSection(config, section, board_lists={}):
149        """Create a Task from a section of a config file.
150
151        The section to parse should look like this:
152        [TaskName]
153        suite: suite_to_run  # Required
154        run_on: event_on which to run  # Required
155        hour: integer of the hour to run, only applies to nightly. # Optional
156        branch_specs: factory,firmware,>=R12 or ==R12 # Optional
157        pool: pool_of_devices  # Optional
158        num: sharding_factor  # int, Optional
159        boards: board1, board2  # comma seperated string, Optional
160        # Settings for Launch Control builds only:
161        os_type: brillo # Type of OS, e.g., cros, brillo, android. Default is
162                 cros. Required for android/brillo builds.
163        branches: git_mnc_release # comma separated string of Launch Control
164                  branches. Required and only applicable for android/brillo
165                  builds.
166        targets: dragonboard-eng # comma separated string of build targets.
167                 Required and only applicable for android/brillo builds.
168        testbed_dut_count: Number of duts to test when using a testbed.
169
170        By default, Tasks run on all release branches, not factory or firmware.
171
172        @param config: a ForgivingConfigParser.
173        @param section: the section to parse into a Task.
174        @param board_lists: a dict including all board whitelist for tasks.
175        @return keyword, Task object pair.  One or both will be None on error.
176        @raise MalformedConfigEntry if there's a problem parsing |section|.
177        """
178        if not config.has_section(section):
179            raise error.MalformedConfigEntry('unknown section %s' % section)
180
181        allowed = set(['suite', 'run_on', 'branch_specs', 'pool', 'num',
182                       'boards', 'file_bugs', 'cros_build_spec',
183                       'firmware_rw_build_spec', 'firmware_ro_build_spec',
184                       'test_source', 'job_retry', 'hour', 'day', 'branches',
185                       'targets', 'os_type', 'no_delay', 'owner', 'priority',
186                       'timeout'])
187        # The parameter of union() is the keys under the section in the config
188        # The union merges this with the allowed set, so if any optional keys
189        # are omitted, then they're filled in. If any extra keys are present,
190        # then they will expand unioned set, causing it to fail the following
191        # comparison against the allowed set.
192        section_headers = allowed.union(dict(config.items(section)).keys())
193        if allowed != section_headers:
194            raise error.MalformedConfigEntry('unknown entries: %s' %
195                      ", ".join(map(str, section_headers.difference(allowed))))
196
197        keyword = config.getstring(section, 'run_on')
198        hour = config.getstring(section, 'hour')
199        suite = config.getstring(section, 'suite')
200        branch_specs = config.getstring(section, 'branch_specs')
201        pool = config.getstring(section, 'pool')
202        boards = config.getstring(section, 'boards')
203        file_bugs = config.getboolean(section, 'file_bugs')
204        cros_build_spec = config.getstring(section, 'cros_build_spec')
205        firmware_rw_build_spec = config.getstring(
206                section, 'firmware_rw_build_spec')
207        firmware_ro_build_spec = config.getstring(
208                section, 'firmware_ro_build_spec')
209        test_source = config.getstring(section, 'test_source')
210        job_retry = config.getboolean(section, 'job_retry')
211        no_delay = config.getboolean(section, 'no_delay')
212        # In case strings empty use sane low priority defaults.
213        priority = 0
214        timeout = 24
215        # Set priority/timeout based on the event type.
216        for klass in driver.Driver.EVENT_CLASSES:
217            if klass.KEYWORD == keyword:
218                priority = klass.PRIORITY
219                timeout = klass.TIMEOUT
220                break
221        # Set priority/timeout from config file explicitly if set.
222        priority_string = config.getstring(section, 'priority')
223        if priority_string:
224            # Try to parse priority as int first. If failed, then use the
225            # global string->priority mapping to lookup its value.
226            try:
227                try:
228                    priority = int(priority_string)
229                except ValueError:
230                    priority = priorities.Priority.get_value(priority_string)
231            except ValueError:
232                raise error.MalformedConfigEntry("Priority string not "
233                                                 "recognized as value (%s).",
234                                                 priority_string)
235        timeout_value = config.getint(section, 'timeout')
236        if timeout_value:
237            timeout = timeout_value
238
239        # Sanity Check for priority and timeout.
240        if priority < 0 or priority > 100:
241            raise error.MalformedConfigEntry('Priority(%d) should be inside '
242                                             'the range 0-100.' % priority)
243        if timeout <= 0:
244            raise error.MalformedConfigEntry('Timeout(%d) needs to be positive '
245                                             'integer (hours).' % timeout)
246
247        try:
248            num = config.getint(section, 'num')
249        except ValueError as e:
250            raise error.MalformedConfigEntry("Ill-specified 'num': %r" % e)
251        if not keyword:
252            raise error.MalformedConfigEntry('No event to |run_on|.')
253        if not suite:
254            raise error.MalformedConfigEntry('No |suite|')
255        try:
256            hour = config.getint(section, 'hour')
257        except ValueError as e:
258            raise error.MalformedConfigEntry("Ill-specified 'hour': %r" % e)
259        if hour is not None and (hour < 0 or hour > 23):
260            raise error.MalformedConfigEntry(
261                    '`hour` must be an integer between 0 and 23.')
262        if hour is not None and keyword != 'nightly':
263            raise error.MalformedConfigEntry(
264                    '`hour` is the trigger time that can only apply to nightly '
265                    'event.')
266
267        testbed_dut_count = None
268        if boards:
269            match = re.match(TESTBED_DUT_COUNT_REGEX, boards)
270            if match:
271                testbed_dut_count = int(match.group(1))
272
273        try:
274            day = config.getint(section, 'day')
275        except ValueError as e:
276            raise error.MalformedConfigEntry("Ill-specified 'day': %r" % e)
277        if day is not None and (day < 0 or day > 6):
278            raise error.MalformedConfigEntry(
279                    '`day` must be an integer between 0 and 6, where 0 is for '
280                    'Monday and 6 is for Sunday.')
281        if day is not None and keyword != 'weekly':
282            raise error.MalformedConfigEntry(
283                    '`day` is the trigger of the day of a week, that can only '
284                    'apply to weekly events.')
285
286        specs = []
287        if branch_specs:
288            specs = re.split('\s*,\s*', branch_specs)
289            Task.CheckBranchSpecs(specs)
290
291        os_type = config.getstring(section, 'os_type') or OS_TYPE_CROS
292        if os_type not in OS_TYPES:
293            raise error.MalformedConfigEntry(
294                    '`os_type` must be one of %s' % OS_TYPES)
295
296        lc_branches = config.getstring(section, 'branches')
297        lc_targets = config.getstring(section, 'targets')
298        if os_type == OS_TYPE_CROS and (lc_branches or lc_targets):
299            raise error.MalformedConfigEntry(
300                    '`branches` and `targets` are only supported for Launch '
301                    'Control builds, not ChromeOS builds.')
302        if (os_type in OS_TYPES_LAUNCH_CONTROL and
303            (not lc_branches or not lc_targets)):
304            raise error.MalformedConfigEntry(
305                    '`branches` and `targets` must be specified for Launch '
306                    'Control builds.')
307        if (os_type in OS_TYPES_LAUNCH_CONTROL and boards and
308            not testbed_dut_count):
309            raise error.MalformedConfigEntry(
310                    '`boards` for Launch Control builds are retrieved from '
311                    '`targets` setting, it should not be set for Launch '
312                    'Control builds.')
313        if os_type == OS_TYPE_CROS and testbed_dut_count:
314            raise error.MalformedConfigEntry(
315                    'testbed_dut_count is only supported for Launch Control '
316                    'builds testing with testbed.')
317
318        # Extract boards from targets list.
319        if os_type in OS_TYPES_LAUNCH_CONTROL:
320            boards = ''
321            for target in lc_targets.split(','):
322                board_name, _ = server_utils.parse_launch_control_target(
323                        target.strip())
324                # Translate board name in build target to the actual board name.
325                board_name = server_utils.ANDROID_TARGET_TO_BOARD_MAP.get(
326                        board_name, board_name)
327                boards += '%s,' % board_name
328            boards = boards.strip(',')
329        elif os_type == OS_TYPE_CROS:
330            if board_lists:
331                if boards not in board_lists:
332                    logging.debug(
333                            'The board_list name %s does not exist in '
334                            'section board_lists in config.', boards)
335                    # TODO(xixuan): Raise MalformedConfigEntry when a CrOS task
336                    # specify a 'boards' which is not defined in board_lists.
337                    # Currently exception won't be raised to make sure suite
338                    # scheduler keeps running when developers are in the middle
339                    # of migrating boards.
340                else:
341                    boards = board_lists[boards]
342
343        return keyword, Task(section, suite, specs, pool, num, boards,
344                             priority, timeout,
345                             file_bugs=file_bugs if file_bugs else False,
346                             cros_build_spec=cros_build_spec,
347                             firmware_rw_build_spec=firmware_rw_build_spec,
348                             firmware_ro_build_spec=firmware_ro_build_spec,
349                             test_source=test_source, job_retry=job_retry,
350                             hour=hour, day=day, os_type=os_type,
351                             launch_control_branches=lc_branches,
352                             launch_control_targets=lc_targets,
353                             testbed_dut_count=testbed_dut_count,
354                             no_delay=no_delay)
355
356
357    @staticmethod
358    def CheckBranchSpecs(branch_specs):
359        """Make sure entries in the list branch_specs are correctly formed.
360
361        We accept any of BARE_BRANCHES in |branch_specs|, as
362        well as _one_ string of the form '>=RXX' or '==RXX', where 'RXX' is a
363        CrOS milestone number.
364
365        @param branch_specs: an iterable of branch specifiers.
366        @raise MalformedConfigEntry if there's a problem parsing |branch_specs|.
367        """
368        have_seen_numeric_constraint = False
369        for branch in branch_specs:
370            if branch in BARE_BRANCHES:
371                continue
372            if not have_seen_numeric_constraint:
373                #TODO(beeps): Why was <= dropped on the floor?
374                if branch.startswith('>=R') or branch.startswith('==R'):
375                    have_seen_numeric_constraint = True
376                elif 'tot' in branch:
377                    TotMilestoneManager().ConvertTotSpec(
378                            branch[branch.index('tot'):])
379                    have_seen_numeric_constraint = True
380                continue
381            raise error.MalformedConfigEntry(
382                    "%s isn't a valid branch spec.'" % branch)
383
384
385    def __init__(self, name, suite, branch_specs, pool=None, num=None,
386                 boards=None, priority=None, timeout=None, file_bugs=False,
387                 cros_build_spec=None, firmware_rw_build_spec=None,
388                 firmware_ro_build_spec=None, test_source=None, job_retry=False,
389                 hour=None, day=None, os_type=OS_TYPE_CROS,
390                 launch_control_branches=None, launch_control_targets=None,
391                 testbed_dut_count=None, no_delay=False):
392        """Constructor
393
394        Given an iterable in |branch_specs|, pre-vetted using CheckBranchSpecs,
395        we'll store them such that _FitsSpec() can be used to check whether a
396        given branch 'fits' with the specifications passed in here.
397        For example, given branch_specs = ['factory', '>=R18'], we'd set things
398        up so that _FitsSpec() would return True for 'factory', or 'RXX'
399        where XX is a number >= 18. Same check is done for branch_specs = [
400        'factory', '==R18'], which limit the test to only one specific branch.
401
402        Given branch_specs = ['factory', 'firmware'], _FitsSpec()
403        would pass only those two specific strings.
404
405        Example usage:
406          t = Task('Name', 'suite', ['factory', '>=R18'])
407          t._FitsSpec('factory')  # True
408          t._FitsSpec('R19')  # True
409          t._FitsSpec('R17')  # False
410          t._FitsSpec('firmware')  # False
411          t._FitsSpec('goober')  # False
412
413          t = Task('Name', 'suite', ['factory', '==R18'])
414          t._FitsSpec('R19')  # False, branch does not equal to 18
415          t._FitsSpec('R18')  # True
416          t._FitsSpec('R17')  # False
417
418        cros_build_spec and firmware_rw_build_spec are set for tests require
419        firmware update on the dut. Only one of them can be set.
420        For example:
421        branch_specs: ==tot
422        firmware_rw_build_spec: firmware
423        test_source: cros
424        This will run test using latest build on firmware branch, and the latest
425        ChromeOS build on ToT. The test source build is ChromeOS build.
426
427        branch_specs: firmware
428        cros_build_spec: ==tot-1
429        test_source: firmware_rw
430        This will run test using latest build on firmware branch, and the latest
431        ChromeOS build on dev channel (ToT-1). The test source build is the
432        firmware RW build.
433
434        branch_specs: ==tot
435        firmware_rw_build_spec: cros
436        test_source: cros
437        This will run test using latest ChromeOS and firmware RW build on ToT.
438        ChromeOS build on ToT. The test source build is ChromeOS build.
439
440        @param name: name of this task, e.g. 'NightlyPower'
441        @param suite: the name of the suite to run, e.g. 'bvt'
442        @param branch_specs: a pre-vetted iterable of branch specifiers,
443                             e.g. ['>=R18', 'factory']
444        @param pool: the pool of machines to use for scheduling purposes.
445                     Default: None
446        @param num: the number of devices across which to shard the test suite.
447                    Type: integer or None
448                    Default: None
449        @param boards: A comma separated list of boards to run this task on.
450                       Default: Run on all boards.
451        @param priority: The string name of a priority from
452                         client.common_lib.priorities.Priority.
453        @param timeout: The max lifetime of the suite in hours.
454        @param file_bugs: True if bug filing is desired for the suite created
455                          for this task.
456        @param cros_build_spec: Spec used to determine the ChromeOS build to
457                                test with a firmware build, e.g., tot, R41 etc.
458        @param firmware_rw_build_spec: Spec used to determine the firmware RW
459                                       build test with a ChromeOS build.
460        @param firmware_ro_build_spec: Spec used to determine the firmware RO
461                                       build test with a ChromeOS build.
462        @param test_source: The source of test code when firmware will be
463                            updated in the test. The value can be `firmware_rw`,
464                            `firmware_ro` or `cros`.
465        @param job_retry: Set to True to enable job-level retry. Default is
466                          False.
467        @param hour: An integer specifying the hour that a nightly run should
468                     be triggered, default is set to 21.
469        @param day: An integer specifying the day of a week that a weekly run
470                should be triggered, default is set to 5, which is Saturday.
471        @param os_type: Type of OS, e.g., cros, brillo, android. Default is
472                cros. The argument is required for android/brillo builds.
473        @param launch_control_branches: Comma separated string of Launch Control
474                branches. The argument is required and only applicable for
475                android/brillo builds.
476        @param launch_control_targets: Comma separated string of build targets
477                for Launch Control builds. The argument is required and only
478                applicable for android/brillo builds.
479        @param testbed_dut_count: Number of duts to test when using a testbed.
480        @param no_delay: Set to True to allow suite to be created without
481                configuring delay_minutes. Default is False.
482        """
483        self._name = name
484        self._suite = suite
485        self._branch_specs = branch_specs
486        self._pool = pool
487        self._num = num
488        self._priority = priority
489        self._timeout = timeout
490        self._file_bugs = file_bugs
491        self._cros_build_spec = cros_build_spec
492        self._firmware_rw_build_spec = firmware_rw_build_spec
493        self._firmware_ro_build_spec = firmware_ro_build_spec
494        self._test_source = test_source
495        self._job_retry = job_retry
496        self._hour = hour
497        self._day = day
498        self._os_type = os_type
499        self._launch_control_branches = (
500                [b.strip() for b in launch_control_branches.split(',')]
501                if launch_control_branches else [])
502        self._launch_control_targets = (
503                [t.strip() for t in launch_control_targets.split(',')]
504                if launch_control_targets else [])
505        self._testbed_dut_count = testbed_dut_count
506        self._no_delay = no_delay
507
508        if ((self._firmware_rw_build_spec or self._firmware_ro_build_spec or
509             cros_build_spec) and
510            not self.test_source in [Builds.FIRMWARE_RW, Builds.FIRMWARE_RO,
511                                     Builds.CROS]):
512            raise error.MalformedConfigEntry(
513                    'You must specify the build for test source. It can only '
514                    'be `firmware_rw`, `firmware_ro` or `cros`.')
515        if self._firmware_rw_build_spec and cros_build_spec:
516            raise error.MalformedConfigEntry(
517                    'You cannot specify both firmware_rw_build_spec and '
518                    'cros_build_spec. firmware_rw_build_spec is used to specify'
519                    ' a firmware build when the suite requires firmware to be '
520                    'updated in the dut, its value can only be `firmware` or '
521                    '`cros`. cros_build_spec is used to specify a ChromeOS '
522                    'build when build_specs is set to firmware.')
523        if (self._firmware_rw_build_spec and
524            self._firmware_rw_build_spec not in ['firmware', 'cros']):
525            raise error.MalformedConfigEntry(
526                    'firmware_rw_build_spec can only be empty, firmware or '
527                    'cros. It does not support other build type yet.')
528
529        if os_type not in OS_TYPES_LAUNCH_CONTROL and self._testbed_dut_count:
530            raise error.MalformedConfigEntry(
531                    'testbed_dut_count is only applicable to testbed to run '
532                    'test with builds from Launch Control.')
533
534        self._bare_branches = []
535        self._version_equal_constraint = False
536        self._version_gte_constraint = False
537        self._version_lte_constraint = False
538        if not branch_specs:
539            # Any milestone is OK.
540            self._numeric_constraint = version.LooseVersion('0')
541        else:
542            self._numeric_constraint = None
543            for spec in branch_specs:
544                if 'tot' in spec.lower():
545                    tot_str = spec[spec.index('tot'):]
546                    spec = spec.replace(
547                            tot_str, TotMilestoneManager().ConvertTotSpec(
548                                    tot_str))
549                if spec.startswith('>='):
550                    self._numeric_constraint = version.LooseVersion(
551                            spec.lstrip('>=R'))
552                    self._version_gte_constraint = True
553                elif spec.startswith('<='):
554                    self._numeric_constraint = version.LooseVersion(
555                            spec.lstrip('<=R'))
556                    self._version_lte_constraint = True
557                elif spec.startswith('=='):
558                    self._version_equal_constraint = True
559                    self._numeric_constraint = version.LooseVersion(
560                            spec.lstrip('==R'))
561                else:
562                    self._bare_branches.append(spec)
563
564        # Since we expect __hash__() and other comparator methods to be used
565        # frequently by set operations, and they use str() a lot, pre-compute
566        # the string representation of this object.
567        if num is None:
568            numStr = '[Default num]'
569        else:
570            numStr = '%d' % num
571
572        if boards is None:
573            self._boards = set()
574            boardsStr = '[All boards]'
575        else:
576            self._boards = set([x.strip() for x in boards.split(',')])
577            boardsStr = boards
578
579        time_str = ''
580        if self._hour:
581            time_str = ' Run at %d:00.' % self._hour
582        elif self._day:
583            time_str = ' Run on %s.' % _WEEKDAYS[self._day]
584        if os_type == OS_TYPE_CROS:
585            self._str = ('%s: %s on %s with pool %s, boards [%s], file_bugs = '
586                         '%s across %s machines.%s' %
587                         (self.__class__.__name__, suite, branch_specs, pool,
588                          boardsStr, self._file_bugs, numStr, time_str))
589        else:
590            testbed_dut_count_str = '.'
591            if self._testbed_dut_count:
592                testbed_dut_count_str = (', each with %d duts.' %
593                                         self._testbed_dut_count)
594            self._str = ('%s: %s on branches %s and targets %s with pool %s, '
595                         'boards [%s], file_bugs = %s across %s machines%s%s' %
596                         (self.__class__.__name__, suite,
597                          launch_control_branches, launch_control_targets,
598                          pool, boardsStr, self._file_bugs, numStr,
599                          testbed_dut_count_str, time_str))
600
601
602    def _FitsSpec(self, branch):
603        """Checks if a branch is deemed OK by this instance's branch specs.
604
605        When called on a branch name, will return whether that branch
606        'fits' the specifications stored in self._bare_branches,
607        self._numeric_constraint, self._version_equal_constraint,
608        self._version_gte_constraint and self._version_lte_constraint.
609
610        @param branch: the branch to check.
611        @return True if b 'fits' with stored specs, False otherwise.
612        """
613        if branch in BARE_BRANCHES:
614            return branch in self._bare_branches
615        if self._numeric_constraint:
616            if self._version_equal_constraint:
617                return version.LooseVersion(branch) == self._numeric_constraint
618            elif self._version_gte_constraint:
619                return version.LooseVersion(branch) >= self._numeric_constraint
620            elif self._version_lte_constraint:
621                return version.LooseVersion(branch) <= self._numeric_constraint
622            else:
623                # Default to great or equal constraint.
624                return version.LooseVersion(branch) >= self._numeric_constraint
625        else:
626            return False
627
628
629    @property
630    def name(self):
631        """Name of this task, e.g. 'NightlyPower'."""
632        return self._name
633
634
635    @property
636    def suite(self):
637        """Name of the suite to run, e.g. 'bvt'."""
638        return self._suite
639
640
641    @property
642    def branch_specs(self):
643        """a pre-vetted iterable of branch specifiers,
644        e.g. ['>=R18', 'factory']."""
645        return self._branch_specs
646
647
648    @property
649    def pool(self):
650        """The pool of machines to use for scheduling purposes."""
651        return self._pool
652
653
654    @property
655    def num(self):
656        """The number of devices across which to shard the test suite.
657        Type: integer or None"""
658        return self._num
659
660
661    @property
662    def boards(self):
663        """The boards on which to run this suite.
664        Type: Iterable of strings"""
665        return self._boards
666
667
668    @property
669    def priority(self):
670        """The priority of the suite"""
671        return self._priority
672
673
674    @property
675    def timeout(self):
676        """The maximum lifetime of the suite in hours."""
677        return self._timeout
678
679
680    @property
681    def cros_build_spec(self):
682        """The build spec of ChromeOS to test with a firmware build."""
683        return self._cros_build_spec
684
685
686    @property
687    def firmware_rw_build_spec(self):
688        """The build spec of RW firmware to test with a ChromeOS build.
689
690        The value can be firmware or cros.
691        """
692        return self._firmware_rw_build_spec
693
694
695    @property
696    def firmware_ro_build_spec(self):
697        """The build spec of RO firmware to test with a ChromeOS build.
698
699        The value can be stable, firmware or cros, where stable is the stable
700        firmware build retrieved from stable_version table.
701        """
702        return self._firmware_ro_build_spec
703
704
705    @property
706    def test_source(self):
707        """Source of the test code, value can be `firmware_rw`, `firmware_ro` or
708        `cros`."""
709        return self._test_source
710
711
712    @property
713    def hour(self):
714        """An integer specifying the hour that a nightly run should be triggered
715        """
716        return self._hour
717
718
719    @property
720    def day(self):
721        """An integer specifying the day of a week that a weekly run should be
722        triggered"""
723        return self._day
724
725
726    @property
727    def os_type(self):
728        """Type of OS, e.g., cros, brillo, android."""
729        return self._os_type
730
731
732    @property
733    def launch_control_branches(self):
734        """A list of Launch Control builds."""
735        return self._launch_control_branches
736
737
738    @property
739    def launch_control_targets(self):
740        """A list of Launch Control targets."""
741        return self._launch_control_targets
742
743
744    def __str__(self):
745        return self._str
746
747
748    def __repr__(self):
749        return self._str
750
751
752    def __lt__(self, other):
753        return str(self) < str(other)
754
755
756    def __le__(self, other):
757        return str(self) <= str(other)
758
759
760    def __eq__(self, other):
761        return str(self) == str(other)
762
763
764    def __ne__(self, other):
765        return str(self) != str(other)
766
767
768    def __gt__(self, other):
769        return str(self) > str(other)
770
771
772    def __ge__(self, other):
773        return str(self) >= str(other)
774
775
776    def __hash__(self):
777        """Allows instances to be correctly deduped when used in a set."""
778        return hash(str(self))
779
780
781    def _GetCrOSBuild(self, mv, board):
782        """Get the ChromeOS build name to test with firmware build.
783
784        The ChromeOS build to be used is determined by `self.cros_build_spec`.
785        Its value can be:
786        tot: use the latest ToT build.
787        tot-x: use the latest build in x milestone before ToT.
788        Rxx: use the latest build on xx milestone.
789
790        @param board: the board against which to run self._suite.
791        @param mv: an instance of manifest_versions.ManifestVersions.
792
793        @return: The ChromeOS build name to test with firmware build.
794
795        """
796        if not self.cros_build_spec:
797            return None
798        if self.cros_build_spec.startswith('tot'):
799            milestone = TotMilestoneManager().ConvertTotSpec(
800                    self.cros_build_spec)[1:]
801        elif self.cros_build_spec.startswith('R'):
802            milestone = self.cros_build_spec[1:]
803        milestone, latest_manifest = mv.GetLatestManifest(
804                board, 'release', milestone=milestone)
805        latest_build = base_event.BuildName(board, 'release', milestone,
806                                            latest_manifest)
807        logging.debug('Found latest build of %s for spec %s: %s',
808                      board, self.cros_build_spec, latest_build)
809        return latest_build
810
811
812    def _GetFirmwareBuild(self, spec, mv, board):
813        """Get the firmware build name to test with ChromeOS build.
814
815        @param spec: build spec for RO or RW firmware, e.g., firmware, cros.
816                For RO firmware, the value can also be in the format of
817                released_ro_X, where X is the index of the list or RO builds
818                defined in global config RELEASED_RO_BUILDS_[board].
819                For example, for spec `released_ro_2`, and global config
820                CROS/RELEASED_RO_BUILDS_veyron_jerry: build1,build2
821                the return firmare RO build should be build2.
822        @param mv: an instance of manifest_versions.ManifestVersions.
823        @param board: the board against which to run self._suite.
824
825        @return: The firmware build name to test with ChromeOS build.
826        """
827        if spec == 'stable':
828            # TODO(crbug.com/577316): Query stable RO firmware.
829            raise NotImplementedError('`stable` RO firmware build is not '
830                                      'supported yet.')
831        if not spec:
832            return None
833
834        if spec.startswith('released_ro_'):
835            index = int(spec[12:])
836            released_ro_builds = CONFIG.get_config_value(
837                    'CROS', 'RELEASED_RO_BUILDS_%s' % board, type=str,
838                    default='').split(',')
839            if not released_ro_builds or len(released_ro_builds) < index:
840                return None
841            else:
842                return released_ro_builds[index-1]
843
844        # build_type is the build type of the firmware build, e.g., factory,
845        # firmware or release. If spec is set to cros, build type should be
846        # mapped to release.
847        build_type = 'release' if spec == 'cros' else spec
848        latest_milestone, latest_manifest = mv.GetLatestManifest(
849                board, build_type)
850        latest_build = base_event.BuildName(board, build_type, latest_milestone,
851                                            latest_manifest)
852        logging.debug('Found latest firmware build of %s for spec %s: %s',
853                      board, spec, latest_build)
854        return latest_build
855
856
857    def AvailableHosts(self, scheduler, board):
858        """Query what hosts are able to run a test on a board and pool
859        combination.
860
861        @param scheduler: an instance of DedupingScheduler, as defined in
862                          deduping_scheduler.py
863        @param board: the board against which one wants to run the test.
864        @return The list of hosts meeting the board and pool requirements,
865                or None if no hosts were found."""
866        if self._boards and board not in self._boards:
867            return []
868
869        board_label = Labels.BOARD_PREFIX + board
870        if self._testbed_dut_count:
871            board_label += '-%d' % self._testbed_dut_count
872        labels = [board_label]
873        if self._pool:
874            labels.append(Labels.POOL_PREFIX + self._pool)
875
876        return scheduler.CheckHostsExist(multiple_labels=labels)
877
878
879    def ShouldHaveAvailableHosts(self):
880        """As a sanity check, return true if we know for certain that
881        we should be able to schedule this test. If we claim this test
882        should be able to run, and it ends up not being scheduled, then
883        a warning will be reported.
884
885        @return True if this test should be able to run, False otherwise.
886        """
887        return self._pool == 'bvt'
888
889
890    def _ScheduleSuite(self, scheduler, cros_build, firmware_rw_build,
891                       firmware_ro_build, test_source_build,
892                       launch_control_build, board, force, run_prod_code=False):
893        """Try to schedule a suite with given build and board information.
894
895        @param scheduler: an instance of DedupingScheduler, as defined in
896                          deduping_scheduler.py
897        @oaran build: Build to run suite for, e.g., 'daisy-release/R18-1655.0.0'
898                      and 'git_mnc_release/shamu-eng/123'.
899        @param firmware_rw_build: Firmware RW build to run test with.
900        @param firmware_ro_build: Firmware RO build to run test with.
901        @param test_source_build: Test source build, used for server-side
902                                  packaging.
903        @param launch_control_build: Name of a Launch Control build, e.g.,
904                                     'git_mnc_release/shamu-eng/123'
905        @param board: the board against which to run self._suite.
906        @param force: Always schedule the suite.
907        @param run_prod_code: If True, the suite will run the test code that
908                              lives in prod aka the test code currently on the
909                              lab servers. If False, the control files and test
910                              code for this suite run will be retrieved from the
911                              build artifacts. Default is False.
912        """
913        test_source_build_msg = (
914                ' Test source build is %s.' % test_source_build
915                if test_source_build else '')
916        firmware_rw_build_msg = (
917                ' Firmware RW build is %s.' % firmware_rw_build
918                if firmware_rw_build else '')
919        firmware_ro_build_msg = (
920                ' Firmware RO build is %s.' % firmware_ro_build
921                if firmware_ro_build else '')
922        # If testbed_dut_count is set, the suite is for testbed. Update build
923        # and board with the dut count.
924        if self._testbed_dut_count:
925            launch_control_build = '%s#%d' % (launch_control_build,
926                                              self._testbed_dut_count)
927            test_source_build = launch_control_build
928            board = '%s-%d' % (board, self._testbed_dut_count)
929        build_string = cros_build or launch_control_build
930        logging.debug('Schedule %s for build %s.%s%s%s',
931                      self._suite, build_string, test_source_build_msg,
932                      firmware_rw_build_msg, firmware_ro_build_msg)
933
934        if not scheduler.ScheduleSuite(
935                self._suite, board, cros_build, self._pool, self._num,
936                self._priority, self._timeout, force,
937                file_bugs=self._file_bugs,
938                firmware_rw_build=firmware_rw_build,
939                firmware_ro_build=firmware_ro_build,
940                test_source_build=test_source_build,
941                job_retry=self._job_retry,
942                launch_control_build=launch_control_build,
943                run_prod_code=run_prod_code,
944                testbed_dut_count=self._testbed_dut_count,
945                no_delay=self._no_delay):
946            logging.info('Skipping scheduling %s on %s for %s',
947                         self._suite, build_string, board)
948
949
950    def _Run_CrOS_Builds(self, scheduler, branch_builds, board, force=False,
951                         mv=None):
952        """Run this task for CrOS builds. Returns False if it should be
953        destroyed.
954
955        Execute this task.  Attempt to schedule the associated suite.
956        Return True if this task should be kept around, False if it
957        should be destroyed.  This allows for one-shot Tasks.
958
959        @param scheduler: an instance of DedupingScheduler, as defined in
960                          deduping_scheduler.py
961        @param branch_builds: a dict mapping branch name to the build(s) to
962                              install for that branch, e.g.
963                              {'R18': ['x86-alex-release/R18-1655.0.0'],
964                               'R19': ['x86-alex-release/R19-2077.0.0']}
965        @param board: the board against which to run self._suite.
966        @param force: Always schedule the suite.
967        @param mv: an instance of manifest_versions.ManifestVersions.
968
969        @return True if the task should be kept, False if not
970
971        """
972        logging.info('Running %s on %s', self._name, board)
973        is_firmware_build = 'firmware' in self.branch_specs
974
975        # firmware_xx_build is only needed if firmware_xx_build_spec is given.
976        firmware_rw_build = None
977        firmware_ro_build = None
978        try:
979            if is_firmware_build:
980                # When build specified in branch_specs is a firmware build,
981                # we need a ChromeOS build to test with the firmware build.
982                cros_build = self._GetCrOSBuild(mv, board)
983            elif self.firmware_rw_build_spec or self.firmware_ro_build_spec:
984                # When firmware_xx_build_spec is specified, the test involves
985                # updating the RW firmware by firmware build specified in
986                # firmware_xx_build_spec.
987                firmware_rw_build = self._GetFirmwareBuild(
988                            self.firmware_rw_build_spec, mv, board)
989                firmware_ro_build = self._GetFirmwareBuild(
990                            self.firmware_ro_build_spec, mv, board)
991                # If RO firmware is specified, force to create suite, because
992                # dedupe based on test source build does not reflect the change
993                # of RO firmware.
994                if firmware_ro_build:
995                    force = True
996        except manifest_versions.QueryException as e:
997            logging.error(e)
998            logging.error('Running %s on %s is failed. Failed to find build '
999                          'required to run the suite.', self._name, board)
1000            return False
1001
1002        # Return if there is no firmware RO build found for given spec.
1003        if not firmware_ro_build and self.firmware_ro_build_spec:
1004            return True
1005
1006        builds = []
1007        for branch, build in branch_builds.iteritems():
1008            logging.info('Checking if %s fits spec %r',
1009                         branch, self.branch_specs)
1010            if self._FitsSpec(branch):
1011                logging.debug('Build %s fits the spec.', build)
1012                builds.extend(build)
1013        for build in builds:
1014            try:
1015                if is_firmware_build:
1016                    firmware_rw_build = build
1017                else:
1018                    cros_build = build
1019                if self.test_source == Builds.FIRMWARE_RW:
1020                    test_source_build = firmware_rw_build
1021                elif self.test_source == Builds.CROS:
1022                    test_source_build = cros_build
1023                else:
1024                    test_source_build = None
1025                self._ScheduleSuite(scheduler, cros_build, firmware_rw_build,
1026                                    firmware_ro_build, test_source_build,
1027                                    None, board, force)
1028            except deduping_scheduler.DedupingSchedulerException as e:
1029                logging.error(e)
1030        return True
1031
1032
1033    def _Run_LaunchControl_Builds(self, scheduler, launch_control_builds, board,
1034                                  force=False):
1035        """Run this task. Returns False if it should be destroyed.
1036
1037        Execute this task. Attempt to schedule the associated suite.
1038        Return True if this task should be kept around, False if it
1039        should be destroyed. This allows for one-shot Tasks.
1040
1041        @param scheduler: an instance of DedupingScheduler, as defined in
1042                          deduping_scheduler.py
1043        @param launch_control_builds: A list of Launch Control builds.
1044        @param board: the board against which to run self._suite.
1045        @param force: Always schedule the suite.
1046
1047        @return True if the task should be kept, False if not
1048
1049        """
1050        logging.info('Running %s on %s', self._name, board)
1051        for build in launch_control_builds:
1052            # Filter out builds don't match the branches setting.
1053            # Launch Control branches are merged in
1054            # BaseEvents.launch_control_branches_targets property. That allows
1055            # each event only query Launch Control once to get all latest
1056            # builds. However, when a task tries to run, it should only process
1057            # the builds matches the branches specified in task config.
1058            if not any([branch in build
1059                        for branch in self._launch_control_branches]):
1060                continue
1061            try:
1062                self._ScheduleSuite(scheduler, None, None, None,
1063                                    test_source_build=build,
1064                                    launch_control_build=build, board=board,
1065                                    force=force, run_prod_code=True)
1066            except deduping_scheduler.DedupingSchedulerException as e:
1067                logging.error(e)
1068        return True
1069
1070
1071    def Run(self, scheduler, branch_builds, board, force=False, mv=None,
1072            launch_control_builds=None):
1073        """Run this task.  Returns False if it should be destroyed.
1074
1075        Execute this task.  Attempt to schedule the associated suite.
1076        Return True if this task should be kept around, False if it
1077        should be destroyed.  This allows for one-shot Tasks.
1078
1079        @param scheduler: an instance of DedupingScheduler, as defined in
1080                          deduping_scheduler.py
1081        @param branch_builds: a dict mapping branch name to the build(s) to
1082                              install for that branch, e.g.
1083                              {'R18': ['x86-alex-release/R18-1655.0.0'],
1084                               'R19': ['x86-alex-release/R19-2077.0.0']}
1085        @param board: the board against which to run self._suite.
1086        @param force: Always schedule the suite.
1087        @param mv: an instance of manifest_versions.ManifestVersions.
1088        @param launch_control_builds: A list of Launch Control builds.
1089
1090        @return True if the task should be kept, False if not
1091
1092        """
1093        if ((self._os_type == OS_TYPE_CROS and not branch_builds) or
1094            (self._os_type != OS_TYPE_CROS and not launch_control_builds)):
1095            logging.debug('No build to run, skip running %s on %s.', self._name,
1096                          board)
1097            # Return True so the task will be kept, as the given build and board
1098            # do not match.
1099            return True
1100
1101        if self._os_type == OS_TYPE_CROS:
1102            return self._Run_CrOS_Builds(
1103                    scheduler, branch_builds, board, force, mv)
1104        else:
1105            return self._Run_LaunchControl_Builds(
1106                    scheduler, launch_control_builds, board, force)
1107
1108
1109class OneShotTask(Task):
1110    """A Task that can be run only once.  Can schedule itself."""
1111
1112
1113    def Run(self, scheduler, branch_builds, board, force=False, mv=None,
1114            launch_control_builds=None):
1115        """Run this task.  Returns False, indicating it should be destroyed.
1116
1117        Run this task.  Attempt to schedule the associated suite.
1118        Return False, indicating to the caller that it should discard this task.
1119
1120        @param scheduler: an instance of DedupingScheduler, as defined in
1121                          deduping_scheduler.py
1122        @param branch_builds: a dict mapping branch name to the build(s) to
1123                              install for that branch, e.g.
1124                              {'R18': ['x86-alex-release/R18-1655.0.0'],
1125                               'R19': ['x86-alex-release/R19-2077.0.0']}
1126        @param board: the board against which to run self._suite.
1127        @param force: Always schedule the suite.
1128        @param mv: an instance of manifest_versions.ManifestVersions.
1129        @param launch_control_builds: A list of Launch Control builds.
1130
1131        @return False
1132
1133        """
1134        super(OneShotTask, self).Run(scheduler, branch_builds, board, force,
1135                                     mv, launch_control_builds)
1136        return False
1137