site_utils.py revision 930ed9d0a30b546e936254a7c941caf263479781
1a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# Copyright (c) 2013 The Chromium Authors. All rights reserved. 2a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# Use of this source code is governed by a BSD-style license that can be 3a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# found in the LICENSE file. 4a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 5dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 620cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbsimport contextlib 718699fef3341507c6b0d415ced96f6e25e3c03eaFang Dengimport grp 83cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport httplib 93cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport json 10a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shiimport logging 1135d661e09666d315325f8942d06949ca7283666fMK Ryuimport os 12023afc65f9377db51ff6122977a9f529a32422d3beepsimport random 13dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Millerimport re 143cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport time 15bef578d9163c6574d626e8b98298c59a5ab79221Paul Drewsimport urllib2 16a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 173cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport common 18ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shifrom autotest_lib.client.common_lib import utils 19023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import error 20023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import global_config 210c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryufrom autotest_lib.client.common_lib import host_queue_entry_states 227756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basifrom autotest_lib.server.cros import provision 23a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shifrom autotest_lib.server.cros.dynamic_suite import constants 247e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shifrom autotest_lib.server.cros.dynamic_suite import job_status 25a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 2682997b9a9791105d81c9db14e75ed14946f78f94Dan Shitry: 2782997b9a9791105d81c9db14e75ed14946f78f94Dan Shi from chromite.lib import cros_build_lib 2820cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs from chromite.lib import ts_mon_config 2982997b9a9791105d81c9db14e75ed14946f78f94Dan Shiexcept ImportError: 3020cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs logging.warn('Unable to import chromite. Monarch is disabled.') 3182997b9a9791105d81c9db14e75ed14946f78f94Dan Shi # Init the module variable to None. Access to this module can check if it 3282997b9a9791105d81c9db14e75ed14946f78f94Dan Shi # is not None before making calls. 3382997b9a9791105d81c9db14e75ed14946f78f94Dan Shi cros_build_lib = None 3420cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs ts_mon_config = None 3582997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 36a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 37d37736b67febeebd991d0209dd030635f7ff5cfdDan ShiCONFIG = global_config.global_config 38d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 39d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi_SHERIFF_JS = CONFIG.get_config_value('NOTIFICATIONS', 'sheriffs', default='') 40d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi_LAB_SHERIFF_JS = CONFIG.get_config_value( 41d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 'NOTIFICATIONS', 'lab_sheriffs', default='') 42d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi_CHROMIUM_BUILD_URL = CONFIG.get_config_value( 43d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 'NOTIFICATIONS', 'chromium_build_url', default='') 44dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 453cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard BarnetteLAB_GOOD_STATES = ('open', 'throttled') 463cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 47d37736b67febeebd991d0209dd030635f7ff5cfdDan ShiENABLE_DRONE_IN_RESTRICTED_SUBNET = CONFIG.get_config_value( 48d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 'CROS', 'enable_drone_in_restricted_subnet', type=bool, 49d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi default=False) 503cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 51abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnetteclass TestLabException(Exception): 52abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette """Exception raised when the Test Lab blocks a test or suite.""" 533cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette pass 543cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 553cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 563cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteclass ParseBuildNameException(Exception): 573cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Raised when ParseBuildName() cannot parse a build name.""" 583cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette pass 593cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 603cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 61f08814a436cdff50ea26571001e72a09e3d57aafFang Dengclass Singleton(type): 62f08814a436cdff50ea26571001e72a09e3d57aafFang Deng """Enforce that only one client class is instantiated per process.""" 63f08814a436cdff50ea26571001e72a09e3d57aafFang Deng _instances = {} 64f08814a436cdff50ea26571001e72a09e3d57aafFang Deng 65f08814a436cdff50ea26571001e72a09e3d57aafFang Deng def __call__(cls, *args, **kwargs): 66f08814a436cdff50ea26571001e72a09e3d57aafFang Deng """Fetch the instance of a class to use for subsequent calls.""" 67f08814a436cdff50ea26571001e72a09e3d57aafFang Deng if cls not in cls._instances: 68f08814a436cdff50ea26571001e72a09e3d57aafFang Deng cls._instances[cls] = super(Singleton, cls).__call__( 69f08814a436cdff50ea26571001e72a09e3d57aafFang Deng *args, **kwargs) 70f08814a436cdff50ea26571001e72a09e3d57aafFang Deng return cls._instances[cls] 71f08814a436cdff50ea26571001e72a09e3d57aafFang Deng 7205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Chengclass EmptyAFEHost(object): 7305ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """Object to represent an AFE host object when there is no AFE.""" 7405ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 7505ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng def __init__(self): 7605ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """ 7705ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng We'll be setting the instance attributes as we use them. Right now 7805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng we only use attributes and labels but as time goes by and other 7905ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng attributes are used from an actual AFE Host object (check 8005ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng rpc_interfaces.get_hosts()), we'll add them in here so users won't be 8105ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng perplexed why their host's afe_host object complains that attribute 8205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng doesn't exist. 8305ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """ 8405ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng self.attributes = {} 8505ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng self.labels = [] 8605ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 87f08814a436cdff50ea26571001e72a09e3d57aafFang Deng 883cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnettedef ParseBuildName(name): 893cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Format a build name, given board, type, milestone, and manifest num. 903cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 91b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0' or a 92b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi relative build name, e.g. 'x86-alex-release/LATEST' 933cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 943cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return board: board the manifest is for, e.g. x86-alex. 953cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return type: one of 'release', 'factory', or 'firmware' 963cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return milestone: (numeric) milestone the manifest was associated with. 97b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi Will be None for relative build names. 98b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi @return manifest: manifest number, e.g. '2015.0.0'. 99b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi Will be None for relative build names. 1003cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 1013cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 102e02810d8efc58845d675176be5ef2356868967a4Dan Shi match = re.match(r'(trybot-)?(?P<board>[\w-]+?)(?:-chrome)?(?:-chromium)?' 103e02810d8efc58845d675176be5ef2356868967a4Dan Shi r'-(?P<type>\w+)/(R(?P<milestone>\d+)-' 104e02810d8efc58845d675176be5ef2356868967a4Dan Shi r'(?P<manifest>[\d.ab-]+)|LATEST)', 105f8f648e049ca2f8777fa55c0ea9c4f886efecea9Simran Basi name) 106f8f648e049ca2f8777fa55c0ea9c4f886efecea9Simran Basi if match and len(match.groups()) >= 5: 107b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi return (match.group('board'), match.group('type'), 108b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi match.group('milestone'), match.group('manifest')) 1093cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette raise ParseBuildNameException('%s is a malformed build name.' % name) 1103cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 111dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 1123d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shidef get_labels_from_afe(hostname, label_prefix, afe): 1133d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi """Retrieve a host's specific labels from the AFE. 1143d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1153d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi Looks for the host labels that have the form <label_prefix>:<value> 1163d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi and returns the "<value>" part of the label. None is returned 1173d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi if there is not a label matching the pattern 1183d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1193d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @param hostname: hostname of given DUT. 1203d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @param label_prefix: prefix of label to be matched, e.g., |board:| 1213d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @param afe: afe instance. 1223d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1233d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @returns A list of labels that match the prefix or 'None' 1243d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1253d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi """ 1263d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi labels = afe.get_labels(name__startswith=label_prefix, 1273d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi host__hostname__in=[hostname]) 1283d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi if labels: 1293d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi return [l.name.split(label_prefix, 1)[1] for l in labels] 1303d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1313d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 132a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_label_from_afe(hostname, label_prefix, afe): 133a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve a host's specific label from the AFE. 134a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 135a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks for a host label that has the form <label_prefix>:<value> 136a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi and returns the "<value>" part of the label. None is returned 137a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if there is not a label matching the pattern 138a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 139a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 140a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param label_prefix: prefix of label to be matched, e.g., |board:| 141a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 142a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns the label that matches the prefix or 'None' 143a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 144a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 1453d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi labels = get_labels_from_afe(hostname, label_prefix, afe) 146a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if labels and len(labels) == 1: 1473d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi return labels[0] 148a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 149a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 150a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_board_from_afe(hostname, afe): 151a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve given host's board from its labels in the AFE. 152a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 153a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks for a host label of the form "board:<board>", and 154a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi returns the "<board>" part of the label. `None` is returned 155a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if there is not a single, unique label matching the pattern. 156a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 157a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 158a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 159a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns board from label, or `None`. 160a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 161a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 162a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi return get_label_from_afe(hostname, constants.BOARD_PREFIX, afe) 163a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 164a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 165a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_build_from_afe(hostname, afe): 166a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve the current build for given host from the AFE. 167a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 168a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks through the host's labels in the AFE to determine its build. 169a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 170a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 171a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 172a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns The current build or None if it could not find it or if there 173a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi were multiple build labels assigned to this host. 174a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 175a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 1767756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi for prefix in [provision.CROS_VERSION_PREFIX, 1777756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi provision.ANDROID_BUILD_VERSION_PREFIX]: 1787756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi build = get_label_from_afe(hostname, prefix + ':', afe) 1797756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi if build: 1807756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi return build 1817756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi return None 182a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 183a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 1846a61239a702907ad1aedd56aa4ee700ca50bbab0Allen Li# TODO(fdeng): fix get_sheriffs crbug.com/483254 1853197b39f82eb92afff33c7d44b805afe120c7627Fang Dengdef get_sheriffs(lab_only=False): 186dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller """ 187dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller Polls the javascript file that holds the identity of the sheriff and 188dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller parses it's output to return a list of chromium sheriff email addresses. 189dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller The javascript file can contain the ldap of more than one sheriff, eg: 190dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller document.write('sheriff_one, sheriff_two'). 191dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 1923197b39f82eb92afff33c7d44b805afe120c7627Fang Deng @param lab_only: if True, only pulls lab sheriff. 1933197b39f82eb92afff33c7d44b805afe120c7627Fang Deng @return: A list of chroium.org sheriff email addresses to cc on the bug. 1943197b39f82eb92afff33c7d44b805afe120c7627Fang Deng An empty list if failed to parse the javascript. 195dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller """ 196dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller sheriff_ids = [] 1973197b39f82eb92afff33c7d44b805afe120c7627Fang Deng sheriff_js_list = _LAB_SHERIFF_JS.split(',') 1983197b39f82eb92afff33c7d44b805afe120c7627Fang Deng if not lab_only: 1993197b39f82eb92afff33c7d44b805afe120c7627Fang Deng sheriff_js_list.extend(_SHERIFF_JS.split(',')) 2003197b39f82eb92afff33c7d44b805afe120c7627Fang Deng 2013197b39f82eb92afff33c7d44b805afe120c7627Fang Deng for sheriff_js in sheriff_js_list: 202dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller try: 203ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi url_content = utils.urlopen('%s%s'% ( 204dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller _CHROMIUM_BUILD_URL, sheriff_js)).read() 205dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller except (ValueError, IOError) as e: 2064efdf03d1057f9d38e043b7c2affe842109805d2beeps logging.warning('could not parse sheriff from url %s%s: %s', 2074efdf03d1057f9d38e043b7c2affe842109805d2beeps _CHROMIUM_BUILD_URL, sheriff_js, str(e)) 208bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews except (urllib2.URLError, httplib.HTTPException) as e: 209bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews logging.warning('unexpected error reading from url "%s%s": %s', 210bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews _CHROMIUM_BUILD_URL, sheriff_js, str(e)) 211dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller else: 212dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller ldaps = re.search(r"document.write\('(.*)'\)", url_content) 213dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller if not ldaps: 2144efdf03d1057f9d38e043b7c2affe842109805d2beeps logging.warning('Could not retrieve sheriff ldaps for: %s', 2154efdf03d1057f9d38e043b7c2affe842109805d2beeps url_content) 216dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller continue 217dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller sheriff_ids += ['%s@chromium.org' % alias.replace(' ', '') 218dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller for alias in ldaps.group(1).split(',')] 219dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller return sheriff_ids 22046dadc9439355f72d394dcc4700902001bd797ffbeeps 22146dadc9439355f72d394dcc4700902001bd797ffbeeps 22246dadc9439355f72d394dcc4700902001bd797ffbeepsdef remote_wget(source_url, dest_path, ssh_cmd): 22346dadc9439355f72d394dcc4700902001bd797ffbeeps """wget source_url from localhost to dest_path on remote host using ssh. 22446dadc9439355f72d394dcc4700902001bd797ffbeeps 22546dadc9439355f72d394dcc4700902001bd797ffbeeps @param source_url: The complete url of the source of the package to send. 22646dadc9439355f72d394dcc4700902001bd797ffbeeps @param dest_path: The path on the remote host's file system where we would 22746dadc9439355f72d394dcc4700902001bd797ffbeeps like to store the package. 22846dadc9439355f72d394dcc4700902001bd797ffbeeps @param ssh_cmd: The ssh command to use in performing the remote wget. 22946dadc9439355f72d394dcc4700902001bd797ffbeeps """ 23046dadc9439355f72d394dcc4700902001bd797ffbeeps wget_cmd = ("wget -O - %s | %s 'cat >%s'" % 23146dadc9439355f72d394dcc4700902001bd797ffbeeps (source_url, ssh_cmd, dest_path)) 232ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi utils.run(wget_cmd) 23346dadc9439355f72d394dcc4700902001bd797ffbeeps 2343cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 235266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette_MAX_LAB_STATUS_ATTEMPTS = 5 236266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnettedef _get_lab_status(status_url): 2373cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Grabs the current lab status and message. 2383cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 239266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette @returns The JSON object obtained from the given URL. 240266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 2413cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 2423cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette retry_waittime = 1 243266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette for _ in range(_MAX_LAB_STATUS_ATTEMPTS): 2443cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette try: 2453cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette response = urllib2.urlopen(status_url) 2463cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette except IOError as e: 247266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette logging.debug('Error occurred when grabbing the lab status: %s.', 2483cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette e) 2493cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette time.sleep(retry_waittime) 2503cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette continue 2513cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # Check for successful response code. 2523cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette if response.getcode() == 200: 253266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return json.load(response) 2543cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette time.sleep(retry_waittime) 255266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return None 2563cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 2573cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 258abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnettedef _decode_lab_status(lab_status, build): 259266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette """Decode lab status, and report exceptions as needed. 2603cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 261abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette Take a deserialized JSON object from the lab status page, and 262abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette interpret it to determine the actual lab status. Raise 263266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette exceptions as required to report when the lab is down. 2643cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 265abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @param build: build name that we want to check the status of. 2663cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 267abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @raises TestLabException Raised if a request to test for the given 268abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette status and build should be blocked. 2693cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 2703cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # First check if the lab is up. 271266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette if not lab_status['general_state'] in LAB_GOOD_STATES: 272abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette raise TestLabException('Chromium OS Test Lab is closed: ' 273abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette '%s.' % lab_status['message']) 2743cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 275abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # Check if the build we wish to use is disabled. 2763cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # Lab messages should be in the format of: 277abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # Lab is 'status' [regex ...] (comment) 278abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # If the build name matches any regex, it will be blocked. 279abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette build_exceptions = re.search('\[(.*)\]', lab_status['message']) 280ae43721d4a11c6ecd2d502bba73b185b05bfce29Prashanth Balasubramanian if not build_exceptions or not build: 281abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette return 282abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette for build_pattern in build_exceptions.group(1).split(): 2837f215d3906ac204694d64b7eaa85155777f88f39J. Richard Barnette if re.match(build_pattern, build): 284abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette raise TestLabException('Chromium OS Test Lab is closed: ' 285abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette '%s matches %s.' % ( 286abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette build, build_pattern)) 2873cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette return 288266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 289266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 29094234cb747dc067b159789a4b63f21b23d706922Dan Shidef is_in_lab(): 29194234cb747dc067b159789a4b63f21b23d706922Dan Shi """Check if current Autotest instance is in lab 29294234cb747dc067b159789a4b63f21b23d706922Dan Shi 29394234cb747dc067b159789a4b63f21b23d706922Dan Shi @return: True if the Autotest instance is in lab. 29494234cb747dc067b159789a4b63f21b23d706922Dan Shi """ 295d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi test_server_name = CONFIG.get_config_value('SERVER', 'hostname') 29694234cb747dc067b159789a4b63f21b23d706922Dan Shi return test_server_name.startswith('cautotest') 29794234cb747dc067b159789a4b63f21b23d706922Dan Shi 29894234cb747dc067b159789a4b63f21b23d706922Dan Shi 299abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnettedef check_lab_status(build): 300abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette """Check if the lab status allows us to schedule for a build. 301266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 302abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette Checks if the lab is down, or if testing for the requested build 303abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette should be blocked. 304266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 305abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @param build: Name of the build to be scheduled for testing. 306266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 307abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @raises TestLabException Raised if a request to test for the given 308abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette status and build should be blocked. 309266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 310266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette """ 311266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # Ensure we are trying to schedule on the actual lab. 31294234cb747dc067b159789a4b63f21b23d706922Dan Shi if not is_in_lab(): 313266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return 314266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 315266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # Download the lab status from its home on the web. 316d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi status_url = CONFIG.get_config_value('CROS', 'lab_status_url') 317266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette json_status = _get_lab_status(status_url) 318266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette if json_status is None: 319266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # We go ahead and say the lab is open if we can't get the status. 32004be2bd5e4666a5c253e9c30ab20555e04286032Ilja H. Friedel logging.warning('Could not get a status from %s', status_url) 321266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return 322abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette _decode_lab_status(json_status, build) 323023afc65f9377db51ff6122977a9f529a32422d3beeps 324023afc65f9377db51ff6122977a9f529a32422d3beeps 325023afc65f9377db51ff6122977a9f529a32422d3beepsdef lock_host_with_labels(afe, lock_manager, labels): 326023afc65f9377db51ff6122977a9f529a32422d3beeps """Lookup and lock one host that matches the list of input labels. 327023afc65f9377db51ff6122977a9f529a32422d3beeps 328023afc65f9377db51ff6122977a9f529a32422d3beeps @param afe: An instance of the afe class, as defined in server.frontend. 329023afc65f9377db51ff6122977a9f529a32422d3beeps @param lock_manager: A lock manager capable of locking hosts, eg the 330023afc65f9377db51ff6122977a9f529a32422d3beeps one defined in server.cros.host_lock_manager. 331023afc65f9377db51ff6122977a9f529a32422d3beeps @param labels: A list of labels to look for on hosts. 332023afc65f9377db51ff6122977a9f529a32422d3beeps 333023afc65f9377db51ff6122977a9f529a32422d3beeps @return: The hostname of a host matching all labels, and locked through the 334023afc65f9377db51ff6122977a9f529a32422d3beeps lock_manager. The hostname will be as specified in the database the afe 335023afc65f9377db51ff6122977a9f529a32422d3beeps object is associated with, i.e if it exists in afe_hosts with a .cros 336023afc65f9377db51ff6122977a9f529a32422d3beeps suffix, the hostname returned will contain a .cros suffix. 337023afc65f9377db51ff6122977a9f529a32422d3beeps 338023afc65f9377db51ff6122977a9f529a32422d3beeps @raises: error.NoEligibleHostException: If no hosts matching the list of 339023afc65f9377db51ff6122977a9f529a32422d3beeps input labels are available. 340023afc65f9377db51ff6122977a9f529a32422d3beeps @raises: error.TestError: If unable to lock a host matching the labels. 341023afc65f9377db51ff6122977a9f529a32422d3beeps """ 342023afc65f9377db51ff6122977a9f529a32422d3beeps potential_hosts = afe.get_hosts(multiple_labels=labels) 343023afc65f9377db51ff6122977a9f529a32422d3beeps if not potential_hosts: 344023afc65f9377db51ff6122977a9f529a32422d3beeps raise error.NoEligibleHostException( 345023afc65f9377db51ff6122977a9f529a32422d3beeps 'No devices found with labels %s.' % labels) 346023afc65f9377db51ff6122977a9f529a32422d3beeps 347023afc65f9377db51ff6122977a9f529a32422d3beeps # This prevents errors where a fault might seem repeatable 348023afc65f9377db51ff6122977a9f529a32422d3beeps # because we lock, say, the same packet capturer for each test run. 349023afc65f9377db51ff6122977a9f529a32422d3beeps random.shuffle(potential_hosts) 350023afc65f9377db51ff6122977a9f529a32422d3beeps for host in potential_hosts: 351023afc65f9377db51ff6122977a9f529a32422d3beeps if lock_manager.lock([host.hostname]): 352023afc65f9377db51ff6122977a9f529a32422d3beeps logging.info('Locked device %s with labels %s.', 353023afc65f9377db51ff6122977a9f529a32422d3beeps host.hostname, labels) 354023afc65f9377db51ff6122977a9f529a32422d3beeps return host.hostname 355023afc65f9377db51ff6122977a9f529a32422d3beeps else: 356023afc65f9377db51ff6122977a9f529a32422d3beeps logging.info('Unable to lock device %s with labels %s.', 357023afc65f9377db51ff6122977a9f529a32422d3beeps host.hostname, labels) 358023afc65f9377db51ff6122977a9f529a32422d3beeps 359023afc65f9377db51ff6122977a9f529a32422d3beeps raise error.TestError('Could not lock a device with labels %s' % labels) 3607e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3617e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3627e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef get_test_views_from_tko(suite_job_id, tko): 3637e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Get test name and result for given suite job ID. 3647e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3657e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param suite_job_id: ID of suite job. 3667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param tko: an instance of TKO as defined in server/frontend.py. 3677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @return: A dictionary of test status keyed by test name, e.g., 3687e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'} 3697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @raise: Exception when there is no test view found. 3707e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3717e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 3727e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi views = tko.run('get_detailed_test_views', afe_job_id=suite_job_id) 3737e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi relevant_views = filter(job_status.view_is_relevant, views) 3747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not relevant_views: 3757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise Exception('Failed to retrieve job results.') 3767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views = {} 3787e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi for view in relevant_views: 3797e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views[view['test_name']] = view['status'] 3807e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3817e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi return test_views 38235d661e09666d315325f8942d06949ca7283666fMK Ryu 38335d661e09666d315325f8942d06949ca7283666fMK Ryu 384c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryudef get_data_key(prefix, suite, build, board): 385c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu """ 386c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu Constructs a key string from parameters. 387c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 388c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param prefix: Prefix for the generating key. 389c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param suite: a suite name. e.g., bvt-cq, bvt-inline, dummy 390c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param build: The build string. This string should have a consistent 391c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu format eg: x86-mario-release/R26-3570.0.0. If the format of this 392c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu string changes such that we can't determine build_type or branch 393c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu we give up and use the parametes we're sure of instead (suite, 394c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu board). eg: 395c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 1. build = x86-alex-pgo-release/R26-3570.0.0 396c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu branch = 26 397c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = pgo-release 398c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 2. build = lumpy-paladin/R28-3993.0.0-rc5 399c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu branch = 28 400c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = paladin 401c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param board: The board that this suite ran on. 402c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @return: The key string used for a dictionary. 403c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu """ 404c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu try: 405c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu _board, build_type, branch = ParseBuildName(build)[:3] 406c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu except ParseBuildNameException as e: 407c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu logging.error(str(e)) 408c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu branch = 'Unknown' 409c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = 'Unknown' 410c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu else: 411c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu embedded_str = re.search(r'x86-\w+-(.*)', _board) 412c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu if embedded_str: 413c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = embedded_str.group(1) + '-' + build_type 414c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 415c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu data_key_dict = { 416c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'prefix': prefix, 417c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'board': board, 418c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'branch': branch, 419c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'build_type': build_type, 420c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'suite': suite, 421c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu } 422c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu return ('%(prefix)s.%(board)s.%(build_type)s.%(branch)s.%(suite)s' 423c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu % data_key_dict) 42483184356b60f4352e46e69488d54222032d426c0MK Ryu 42583184356b60f4352e46e69488d54222032d426c0MK Ryu 4262d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryudef setup_logging(logfile=None, prefix=False): 42783184356b60f4352e46e69488d54222032d426c0MK Ryu """Setup basic logging with all logging info stripped. 42883184356b60f4352e46e69488d54222032d426c0MK Ryu 42983184356b60f4352e46e69488d54222032d426c0MK Ryu Calls to logging will only show the message. No severity is logged. 43083184356b60f4352e46e69488d54222032d426c0MK Ryu 43183184356b60f4352e46e69488d54222032d426c0MK Ryu @param logfile: If specified dump output to a file as well. 4322d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu @param prefix: Flag for log prefix. Set to True to add prefix to log 4332d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu entries to include timestamp and log level. Default is False. 43483184356b60f4352e46e69488d54222032d426c0MK Ryu """ 43583184356b60f4352e46e69488d54222032d426c0MK Ryu # Remove all existing handlers. client/common_lib/logging_config adds 43683184356b60f4352e46e69488d54222032d426c0MK Ryu # a StreamHandler to logger when modules are imported, e.g., 43783184356b60f4352e46e69488d54222032d426c0MK Ryu # autotest_lib.client.bin.utils. A new StreamHandler will be added here to 43883184356b60f4352e46e69488d54222032d426c0MK Ryu # log only messages, not severity. 43983184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().handlers = [] 44083184356b60f4352e46e69488d54222032d426c0MK Ryu 4412d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu if prefix: 4422d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu log_format = '%(asctime)s %(levelname)-5s| %(message)s' 4432d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu else: 4442d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu log_format = '%(message)s' 4452d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu 44683184356b60f4352e46e69488d54222032d426c0MK Ryu screen_handler = logging.StreamHandler() 4472d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu screen_handler.setFormatter(logging.Formatter(log_format)) 44883184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().addHandler(screen_handler) 44983184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().setLevel(logging.INFO) 45083184356b60f4352e46e69488d54222032d426c0MK Ryu if logfile: 45183184356b60f4352e46e69488d54222032d426c0MK Ryu file_handler = logging.FileHandler(logfile) 4522d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu file_handler.setFormatter(logging.Formatter(log_format)) 45383184356b60f4352e46e69488d54222032d426c0MK Ryu file_handler.setLevel(logging.DEBUG) 45483184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().addHandler(file_handler) 4558c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4568c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4578c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramaniandef is_shard(): 4588c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian """Determines if this instance is running as a shard. 4598c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4608c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian Reads the global_config value shard_hostname in the section SHARD. 4618c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4628c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian @return True, if shard_hostname is set, False otherwise. 4638c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian """ 464d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi hostname = CONFIG.get_config_value('SHARD', 'shard_hostname', default=None) 4650c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return bool(hostname) 4660c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 4670c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 4680cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Dengdef get_global_afe_hostname(): 4690cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Deng """Read the hostname of the global AFE from the global configuration.""" 470d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi return CONFIG.get_config_value('SERVER', 'global_afe_hostname') 4710cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Deng 4720cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Deng 47318699fef3341507c6b0d415ced96f6e25e3c03eaFang Dengdef is_restricted_user(username): 47418699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng """Determines if a user is in a restricted group. 47518699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 47618699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng User in restricted group only have access to master. 47718699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 47818699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng @param username: A string, representing a username. 47918699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 48018699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng @returns: True if the user is in a restricted group. 48118699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng """ 48218699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng if not username: 48318699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng return False 48418699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 485d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi restricted_groups = CONFIG.get_config_value( 48618699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 'AUTOTEST_WEB', 'restricted_groups', default='').split(',') 48718699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng for group in restricted_groups: 4885229c85219e09d6a7857701be20f286ba04809b3Fang Deng try: 4895229c85219e09d6a7857701be20f286ba04809b3Fang Deng if group and username in grp.getgrnam(group).gr_mem: 4905229c85219e09d6a7857701be20f286ba04809b3Fang Deng return True 4915229c85219e09d6a7857701be20f286ba04809b3Fang Deng except KeyError as e: 4925229c85219e09d6a7857701be20f286ba04809b3Fang Deng logging.debug("%s is not a valid group.", group) 49318699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng return False 49418699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 49518699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 4960c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_special_task_status(is_complete, success, is_active): 4970c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Get the status of a special task. 4980c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 4990c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu Emulate a host queue entry status for a special task 5000c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu Although SpecialTasks are not HostQueueEntries, it is helpful to 5010c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu the user to present similar statuses. 5020c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5030c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param is_complete Boolean if the task is completed. 5040c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param success Boolean if the task succeeded. 5050c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param is_active Boolean if the task is active. 5060c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5070c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @return The status of a special task. 5080c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5090c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if is_complete: 5100c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if success: 5110c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.COMPLETED 5120c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.FAILED 5130c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if is_active: 5140c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.RUNNING 5150c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.QUEUED 5160c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5170c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5180c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_special_task_exec_path(hostname, task_id, task_name, time_requested): 5190c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Get the execution path of the SpecialTask. 5200c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5210c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu This method returns different paths depending on where a 5220c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu the task ran: 5230c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu * Master: hosts/hostname/task_id-task_type 5240c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu * Shard: Master_path/time_created 5250c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu This is to work around the fact that a shard can fail independent 5260c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu of the master, and be replaced by another shard that has the same 5270c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu hosts. Without the time_created stamp the logs of the tasks running 5280c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu on the second shard will clobber the logs from the first in google 5290c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu storage, because task ids are not globally unique. 5300c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5310c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param hostname Hostname 5320c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param task_id Special task id 5330c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param task_name Special task name (e.g., Verify, Repair, etc) 5340c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param time_requested Special task requested time. 5350c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5360c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @return An execution path for the task. 5370c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5380c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu results_path = 'hosts/%s/%s-%s' % (hostname, task_id, task_name.lower()) 5390c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5400c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # If we do this on the master it will break backward compatibility, 5410c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # as there are tasks that currently don't have timestamps. If a host 5420c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # or job has been sent to a shard, the rpc for that host/job will 5430c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # be redirected to the shard, so this global_config check will happen 5440c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # on the shard the logs are on. 5450c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if not is_shard(): 5460c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return results_path 5470c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5480c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # Generate a uid to disambiguate special task result directories 5490c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # in case this shard fails. The simplest uid is the job_id, however 5500c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # in rare cases tasks do not have jobs associated with them (eg: 5510c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # frontend verify), so just use the creation timestamp. The clocks 5520c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # between a shard and master should always be in sync. Any discrepancies 5530c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # will be brought to our attention in the form of job timeouts. 5540c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu uid = time_requested.strftime('%Y%d%m%H%M%S') 5550c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5560c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # TODO: This is a hack, however it is the easiest way to achieve 5570c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # correctness. There is currently some debate over the future of 5580c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # tasks in our infrastructure and refactoring everything right 5590c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # now isn't worth the time. 5600c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return '%s/%s' % (results_path, uid) 5610c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5620c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5630c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_job_tag(id, owner): 5640c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Returns a string tag for a job. 5650c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5660c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param id Job id 5670c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param owner Job owner 5680c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5690c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5700c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return '%s-%s' % (id, owner) 5710c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5720c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5730c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_hqe_exec_path(tag, execution_subdir): 5740c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Returns a execution path to a HQE's results. 5750c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5760c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param tag Tag string for a job associated with a HQE. 5770c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param execution_subdir Execution sub-directory string of a HQE. 5780c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5790c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5800c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return os.path.join(tag, execution_subdir) 58182997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 58282997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 58382997b9a9791105d81c9db14e75ed14946f78f94Dan Shidef is_inside_chroot(): 58482997b9a9791105d81c9db14e75ed14946f78f94Dan Shi """Check if the process is running inside chroot. 58582997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 58682997b9a9791105d81c9db14e75ed14946f78f94Dan Shi This is a wrapper around chromite.lib.cros_build_lib.IsInsideChroot(). The 58782997b9a9791105d81c9db14e75ed14946f78f94Dan Shi method checks if cros_build_lib can be imported first. 58882997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 58982997b9a9791105d81c9db14e75ed14946f78f94Dan Shi @return: True if the process is running inside chroot or cros_build_lib 59082997b9a9791105d81c9db14e75ed14946f78f94Dan Shi cannot be imported. 59182997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 59282997b9a9791105d81c9db14e75ed14946f78f94Dan Shi """ 5937f215d3906ac204694d64b7eaa85155777f88f39J. Richard Barnette return not cros_build_lib or cros_build_lib.IsInsideChroot() 59470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 59570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 59670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shidef parse_job_name(name): 59770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi """Parse job name to get information including build, board and suite etc. 59870647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 59970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi Suite job created by run_suite follows the naming convention of: 60070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi [build]-test_suites/control.[suite] 60170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi For example: lumpy-release/R46-7272.0.0-test_suites/control.bvt 60270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi The naming convention is defined in site_rpc_interface.create_suite_job. 60370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 60470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi Test job created by suite job follows the naming convention of: 60570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi [build]/[suite]/[test name] 60670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi For example: lumpy-release/R46-7272.0.0/bvt/login_LoginSuccess 60770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi The naming convention is defined in 60870647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi server/cros/dynamic_suite/tools.create_job_name 60970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 61070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi Note that pgo and chrome-perf builds will fail the method. Since lab does 61170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi not run test for these builds, they can be ignored. 612ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi Also, tests for Launch Control builds have different naming convention. 613ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi The build ID will be used as build_version. 61470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 61570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi @param name: Name of the job. 61670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 61770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi @return: A dictionary containing the test information. The keyvals include: 61870647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi build: Name of the build, e.g., lumpy-release/R46-7272.0.0 61970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi build_version: The version of the build, e.g., R46-7272.0.0 62070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi board: Name of the board, e.g., lumpy 62170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi suite: Name of the test suite, e.g., bvt 62270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 62370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi """ 62470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info = {} 625ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi suite_job_regex = '([^/]*/[^/]*(?:/\d+)?)-test_suites/control\.(.*)' 626ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi test_job_regex = '([^/]*/[^/]*(?:/\d+)?)/([^/]+)/.*' 62770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi match = re.match(suite_job_regex, name) 62870647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi if not match: 62970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi match = re.match(test_job_regex, name) 63070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi if match: 63170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['build'] = match.groups()[0] 63270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['suite'] = match.groups()[1] 63370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['build_version'] = info['build'].split('/')[1] 63470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi try: 63570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['board'], _, _, _ = ParseBuildName(info['build']) 63670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi except ParseBuildNameException: 637ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi # Try to parse it as Launch Control build 638ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi # Launch Control builds have name format: 639ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi # branch/build_target-build_type/build_id. 640ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi try: 641ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi _, target, build_id = utils.parse_launch_control_build( 642ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi info['build']) 643ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi build_target, _ = utils.parse_launch_control_target(target) 644ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi if build_target: 645ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi info['board'] = build_target 646ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi info['build_version'] = build_id 647ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi except ValueError: 648ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi pass 64970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi return info 6503a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6513a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6523a4a57a7704ab15136286628f465dbb2d1a56171Kevin Chengdef add_label_detector(label_function_list, label_list=None, label=None): 6533a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """Decorator used to group functions together into the provided list. 6543a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6553a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng This is a helper function to automatically add label functions that have 6563a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng the label decorator. This is to help populate the class list of label 6573a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng functions to be retrieved by the get_labels class method. 6583a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6593a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param label_function_list: List of label detecting functions to add 6603a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng decorated function to. 6613a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param label_list: List of detectable labels to add detectable labels to. 6623a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng (Default: None) 6633a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param label: Label string that is detectable by this detection function 6643a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng (Default: None) 6653a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """ 6663a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng def add_func(func): 6673a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """ 6683a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param func: The function to be added as a detector. 6693a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """ 6703a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng label_function_list.append(func) 6713a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng if label and label_list is not None: 6723a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng label_list.append(label) 6733a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng return func 6743a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng return add_func 6759f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi 6769f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi 6779f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basidef verify_not_root_user(): 6789f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi """Simple function to error out if running with uid == 0""" 6799f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi if os.getuid() == 0: 6801bf60eb788365f083d0ee8045a6556f906149decSimran Basi raise error.IllegalUser('This script can not be ran as root.') 6811bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6821bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6831bf60eb788365f083d0ee8045a6556f906149decSimran Basidef get_hostname_from_machine(machine): 6841bf60eb788365f083d0ee8045a6556f906149decSimran Basi """Lookup hostname from a machine string or dict. 6851bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6861bf60eb788365f083d0ee8045a6556f906149decSimran Basi @returns: Machine hostname in string format. 6871bf60eb788365f083d0ee8045a6556f906149decSimran Basi """ 6881bf60eb788365f083d0ee8045a6556f906149decSimran Basi hostname, _ = get_host_info_from_machine(machine) 6891bf60eb788365f083d0ee8045a6556f906149decSimran Basi return hostname 6901bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6911bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6921bf60eb788365f083d0ee8045a6556f906149decSimran Basidef get_host_info_from_machine(machine): 6931bf60eb788365f083d0ee8045a6556f906149decSimran Basi """Lookup host information from a machine string or dict. 6941bf60eb788365f083d0ee8045a6556f906149decSimran Basi 69505ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng @returns: Tuple of (hostname, afe_host) 6961bf60eb788365f083d0ee8045a6556f906149decSimran Basi """ 6971bf60eb788365f083d0ee8045a6556f906149decSimran Basi if isinstance(machine, dict): 69805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return (machine['hostname'], machine['afe_host']) 6991bf60eb788365f083d0ee8045a6556f906149decSimran Basi else: 70005ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return (machine, EmptyAFEHost()) 70105ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 70205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 70305ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Chengdef get_afe_host_from_machine(machine): 70405ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """Return the afe_host from the machine dict if possible. 70505ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 70605ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng @returns: AFE host object. 70705ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """ 70805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng _, afe_host = get_host_info_from_machine(machine) 70905ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return afe_host 710f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 711f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 712f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Dengdef get_creds_abspath(creds_file): 713f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng """Returns the abspath of the credentials file. 714f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 715f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng If creds_file is already an absolute path, just return it. 716f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng Otherwise, assume it is located in the creds directory 717f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng specified in global_config and return the absolute path. 718f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 719f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng @param: creds_path, a path to the credentials. 720f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng @return: An absolute path to the credentials file. 721f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng """ 722f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng if not creds_file: 723f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng return None 724f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng if os.path.isabs(creds_file): 725f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng return creds_file 726d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi creds_dir = CONFIG.get_config_value('SERVER', 'creds_dir', default='') 727f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng if not creds_dir or not os.path.exists(creds_dir): 728f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng creds_dir = common.autotest_dir 729f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng return os.path.join(creds_dir, creds_file) 7303b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7313b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7323b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Chengdef machine_is_testbed(machine): 7333b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng """Checks if the machine is a testbed. 7343b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7353b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng The signal we use to determine if the machine is a testbed 7363b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng is if the host attributes contain more than 1 serial. 7373b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7383b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng @param machine: is a list of dicts 7393b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7403b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng @return: True if the machine is a testbed, False otherwise. 7413b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng """ 74205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng _, afe_host = get_host_info_from_machine(machine) 74305ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return len(afe_host.attributes.get('serials', '').split(',')) > 1 74420cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 74520cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 74620cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbsdef SetupTsMonGlobalState(*args, **kwargs): 74720cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs """Import-safe wrap around chromite.lib.ts_mon_config's setup function. 74820cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 74920cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs @param *args: Args to pass through. 75020cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs @param **kwargs: Kwargs to pass through. 75120cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs """ 75220cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs if ts_mon_config: 753930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs context = ts_mon_config.SetupTsMonGlobalState(*args, **kwargs) 754930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs if hasattr(context, '__exit__'): 755930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs return context 756930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs else: 757930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs return TrivialContextManager() 75820cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs else: 759930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs return TrivialContextManager() 76020cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 76120cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 76220cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs@contextlib.contextmanager 76320cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbsdef TrivialContextManager(): 76420cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs """Context manager that does nothing.""" 76520cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs yield 766