site_utils.py revision 16b46f8039a9e29fa036f1347dcadcacac6a7d77
1a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# Copyright (c) 2013 The Chromium Authors. All rights reserved. 2a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# Use of this source code is governed by a BSD-style license that can be 3a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# found in the LICENSE file. 4a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 5dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 620cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbsimport contextlib 718699fef3341507c6b0d415ced96f6e25e3c03eaFang Dengimport grp 83cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport httplib 93cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport json 10a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shiimport logging 1135d661e09666d315325f8942d06949ca7283666fMK Ryuimport os 12023afc65f9377db51ff6122977a9f529a32422d3beepsimport random 13dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Millerimport re 143cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport time 15bef578d9163c6574d626e8b98298c59a5ab79221Paul Drewsimport urllib2 16a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 173cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport common 18ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shifrom autotest_lib.client.common_lib import utils 19023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import error 20023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import global_config 210c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryufrom autotest_lib.client.common_lib import host_queue_entry_states 225f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Chengfrom autotest_lib.client.common_lib import host_states 237756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basifrom autotest_lib.server.cros import provision 24a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shifrom autotest_lib.server.cros.dynamic_suite import constants 257e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shifrom autotest_lib.server.cros.dynamic_suite import job_status 26a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 27a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 28d37736b67febeebd991d0209dd030635f7ff5cfdDan ShiCONFIG = global_config.global_config 29d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 30d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi_SHERIFF_JS = CONFIG.get_config_value('NOTIFICATIONS', 'sheriffs', default='') 31d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi_LAB_SHERIFF_JS = CONFIG.get_config_value( 32d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 'NOTIFICATIONS', 'lab_sheriffs', default='') 33d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi_CHROMIUM_BUILD_URL = CONFIG.get_config_value( 34d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 'NOTIFICATIONS', 'chromium_build_url', default='') 35dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 363cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard BarnetteLAB_GOOD_STATES = ('open', 'throttled') 373cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 38d37736b67febeebd991d0209dd030635f7ff5cfdDan ShiENABLE_DRONE_IN_RESTRICTED_SUBNET = CONFIG.get_config_value( 39d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi 'CROS', 'enable_drone_in_restricted_subnet', type=bool, 40d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi default=False) 413cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 425f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng# Wait at most 10 mins for duts to go idle. 435f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin ChengIDLE_DUT_WAIT_TIMEOUT = 600 445f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 4543274400d21331e2e725bbec429c7f39eaf23364Dan Shi# Mapping between board name and build target. This is for special case handling 4643274400d21331e2e725bbec429c7f39eaf23364Dan Shi# for certain Android board that the board name and build target name does not 4743274400d21331e2e725bbec429c7f39eaf23364Dan Shi# match. 4808fc62ea685fa40c913ed24f8099ba67785bcfd4tturneyANDROID_TARGET_TO_BOARD_MAP = { 4908fc62ea685fa40c913ed24f8099ba67785bcfd4tturney 'seed_l8150': 'gm4g_sprout', 5008fc62ea685fa40c913ed24f8099ba67785bcfd4tturney 'bat_land': 'bat' 5108fc62ea685fa40c913ed24f8099ba67785bcfd4tturney } 5208fc62ea685fa40c913ed24f8099ba67785bcfd4tturneyANDROID_BOARD_TO_TARGET_MAP = { 5308fc62ea685fa40c913ed24f8099ba67785bcfd4tturney 'gm4g_sprout': 'seed_l8150', 5408fc62ea685fa40c913ed24f8099ba67785bcfd4tturney 'bat': 'bat_land' 5508fc62ea685fa40c913ed24f8099ba67785bcfd4tturney } 5643274400d21331e2e725bbec429c7f39eaf23364Dan Shi 57abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnetteclass TestLabException(Exception): 58abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette """Exception raised when the Test Lab blocks a test or suite.""" 593cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette pass 603cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 613cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 623cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteclass ParseBuildNameException(Exception): 633cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Raised when ParseBuildName() cannot parse a build name.""" 643cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette pass 653cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 663cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 67f08814a436cdff50ea26571001e72a09e3d57aafFang Dengclass Singleton(type): 68f08814a436cdff50ea26571001e72a09e3d57aafFang Deng """Enforce that only one client class is instantiated per process.""" 69f08814a436cdff50ea26571001e72a09e3d57aafFang Deng _instances = {} 70f08814a436cdff50ea26571001e72a09e3d57aafFang Deng 71f08814a436cdff50ea26571001e72a09e3d57aafFang Deng def __call__(cls, *args, **kwargs): 72f08814a436cdff50ea26571001e72a09e3d57aafFang Deng """Fetch the instance of a class to use for subsequent calls.""" 73f08814a436cdff50ea26571001e72a09e3d57aafFang Deng if cls not in cls._instances: 74f08814a436cdff50ea26571001e72a09e3d57aafFang Deng cls._instances[cls] = super(Singleton, cls).__call__( 75f08814a436cdff50ea26571001e72a09e3d57aafFang Deng *args, **kwargs) 76f08814a436cdff50ea26571001e72a09e3d57aafFang Deng return cls._instances[cls] 77f08814a436cdff50ea26571001e72a09e3d57aafFang Deng 7805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Chengclass EmptyAFEHost(object): 7905ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """Object to represent an AFE host object when there is no AFE.""" 8005ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 8105ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng def __init__(self): 8205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """ 8305ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng We'll be setting the instance attributes as we use them. Right now 8405ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng we only use attributes and labels but as time goes by and other 8505ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng attributes are used from an actual AFE Host object (check 8605ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng rpc_interfaces.get_hosts()), we'll add them in here so users won't be 8705ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng perplexed why their host's afe_host object complains that attribute 8805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng doesn't exist. 8905ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """ 9005ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng self.attributes = {} 9105ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng self.labels = [] 9205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 93f08814a436cdff50ea26571001e72a09e3d57aafFang Deng 943cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnettedef ParseBuildName(name): 953cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Format a build name, given board, type, milestone, and manifest num. 963cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 97b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0' or a 98b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi relative build name, e.g. 'x86-alex-release/LATEST' 993cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 1003cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return board: board the manifest is for, e.g. x86-alex. 1013cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return type: one of 'release', 'factory', or 'firmware' 1023cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return milestone: (numeric) milestone the manifest was associated with. 103b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi Will be None for relative build names. 104b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi @return manifest: manifest number, e.g. '2015.0.0'. 105b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi Will be None for relative build names. 1063cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 1073cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 108e02810d8efc58845d675176be5ef2356868967a4Dan Shi match = re.match(r'(trybot-)?(?P<board>[\w-]+?)(?:-chrome)?(?:-chromium)?' 109e02810d8efc58845d675176be5ef2356868967a4Dan Shi r'-(?P<type>\w+)/(R(?P<milestone>\d+)-' 110e02810d8efc58845d675176be5ef2356868967a4Dan Shi r'(?P<manifest>[\d.ab-]+)|LATEST)', 111f8f648e049ca2f8777fa55c0ea9c4f886efecea9Simran Basi name) 112f8f648e049ca2f8777fa55c0ea9c4f886efecea9Simran Basi if match and len(match.groups()) >= 5: 113b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi return (match.group('board'), match.group('type'), 114b7d2116ad62dd09c52791c8f27dfa61b9df186b9Simran Basi match.group('milestone'), match.group('manifest')) 1153cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette raise ParseBuildNameException('%s is a malformed build name.' % name) 1163cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 117dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 1183d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shidef get_labels_from_afe(hostname, label_prefix, afe): 1193d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi """Retrieve a host's specific labels from the AFE. 1203d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1213d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi Looks for the host labels that have the form <label_prefix>:<value> 1223d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi and returns the "<value>" part of the label. None is returned 1233d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi if there is not a label matching the pattern 1243d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1253d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @param hostname: hostname of given DUT. 1263d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @param label_prefix: prefix of label to be matched, e.g., |board:| 1273d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @param afe: afe instance. 1283d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1293d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi @returns A list of labels that match the prefix or 'None' 1303d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1313d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi """ 1323d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi labels = afe.get_labels(name__startswith=label_prefix, 1333d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi host__hostname__in=[hostname]) 1343d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi if labels: 1353d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi return [l.name.split(label_prefix, 1)[1] for l in labels] 1363d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 1373d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi 138a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_label_from_afe(hostname, label_prefix, afe): 139a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve a host's specific label from the AFE. 140a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 141a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks for a host label that has the form <label_prefix>:<value> 142a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi and returns the "<value>" part of the label. None is returned 143a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if there is not a label matching the pattern 144a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 145a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 146a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param label_prefix: prefix of label to be matched, e.g., |board:| 147a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 148a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns the label that matches the prefix or 'None' 149a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 150a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 1513d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi labels = get_labels_from_afe(hostname, label_prefix, afe) 152a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if labels and len(labels) == 1: 1533d7a0e14e220d7ea2cbe00013959c6d967a6a41aDan Shi return labels[0] 154a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 155a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 156a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_board_from_afe(hostname, afe): 157a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve given host's board from its labels in the AFE. 158a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 159a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks for a host label of the form "board:<board>", and 160a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi returns the "<board>" part of the label. `None` is returned 161a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if there is not a single, unique label matching the pattern. 162a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 163a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 164a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 165a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns board from label, or `None`. 166a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 167a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 168a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi return get_label_from_afe(hostname, constants.BOARD_PREFIX, afe) 169a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 170a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 171a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_build_from_afe(hostname, afe): 172a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve the current build for given host from the AFE. 173a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 174a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks through the host's labels in the AFE to determine its build. 175a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 176a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 177a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 178a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns The current build or None if it could not find it or if there 179a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi were multiple build labels assigned to this host. 180a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 181a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 1827756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi for prefix in [provision.CROS_VERSION_PREFIX, 1837756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi provision.ANDROID_BUILD_VERSION_PREFIX]: 1847756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi build = get_label_from_afe(hostname, prefix + ':', afe) 1857756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi if build: 1867756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi return build 1877756a0b6949729cb5a632955ebd8fe5dbd308c08Simran Basi return None 188a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 189a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 1906a61239a702907ad1aedd56aa4ee700ca50bbab0Allen Li# TODO(fdeng): fix get_sheriffs crbug.com/483254 1913197b39f82eb92afff33c7d44b805afe120c7627Fang Dengdef get_sheriffs(lab_only=False): 192dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller """ 193dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller Polls the javascript file that holds the identity of the sheriff and 194dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller parses it's output to return a list of chromium sheriff email addresses. 195dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller The javascript file can contain the ldap of more than one sheriff, eg: 196dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller document.write('sheriff_one, sheriff_two'). 197dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 1983197b39f82eb92afff33c7d44b805afe120c7627Fang Deng @param lab_only: if True, only pulls lab sheriff. 1993197b39f82eb92afff33c7d44b805afe120c7627Fang Deng @return: A list of chroium.org sheriff email addresses to cc on the bug. 2003197b39f82eb92afff33c7d44b805afe120c7627Fang Deng An empty list if failed to parse the javascript. 201dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller """ 202dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller sheriff_ids = [] 2033197b39f82eb92afff33c7d44b805afe120c7627Fang Deng sheriff_js_list = _LAB_SHERIFF_JS.split(',') 2043197b39f82eb92afff33c7d44b805afe120c7627Fang Deng if not lab_only: 2053197b39f82eb92afff33c7d44b805afe120c7627Fang Deng sheriff_js_list.extend(_SHERIFF_JS.split(',')) 2063197b39f82eb92afff33c7d44b805afe120c7627Fang Deng 2073197b39f82eb92afff33c7d44b805afe120c7627Fang Deng for sheriff_js in sheriff_js_list: 208dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller try: 209ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi url_content = utils.urlopen('%s%s'% ( 210dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller _CHROMIUM_BUILD_URL, sheriff_js)).read() 211dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller except (ValueError, IOError) as e: 2124efdf03d1057f9d38e043b7c2affe842109805d2beeps logging.warning('could not parse sheriff from url %s%s: %s', 2134efdf03d1057f9d38e043b7c2affe842109805d2beeps _CHROMIUM_BUILD_URL, sheriff_js, str(e)) 214bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews except (urllib2.URLError, httplib.HTTPException) as e: 215bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews logging.warning('unexpected error reading from url "%s%s": %s', 216bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews _CHROMIUM_BUILD_URL, sheriff_js, str(e)) 217dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller else: 218dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller ldaps = re.search(r"document.write\('(.*)'\)", url_content) 219dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller if not ldaps: 2204efdf03d1057f9d38e043b7c2affe842109805d2beeps logging.warning('Could not retrieve sheriff ldaps for: %s', 2214efdf03d1057f9d38e043b7c2affe842109805d2beeps url_content) 222dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller continue 223dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller sheriff_ids += ['%s@chromium.org' % alias.replace(' ', '') 224dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller for alias in ldaps.group(1).split(',')] 225dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller return sheriff_ids 22646dadc9439355f72d394dcc4700902001bd797ffbeeps 22746dadc9439355f72d394dcc4700902001bd797ffbeeps 22846dadc9439355f72d394dcc4700902001bd797ffbeepsdef remote_wget(source_url, dest_path, ssh_cmd): 22946dadc9439355f72d394dcc4700902001bd797ffbeeps """wget source_url from localhost to dest_path on remote host using ssh. 23046dadc9439355f72d394dcc4700902001bd797ffbeeps 23146dadc9439355f72d394dcc4700902001bd797ffbeeps @param source_url: The complete url of the source of the package to send. 23246dadc9439355f72d394dcc4700902001bd797ffbeeps @param dest_path: The path on the remote host's file system where we would 23346dadc9439355f72d394dcc4700902001bd797ffbeeps like to store the package. 23446dadc9439355f72d394dcc4700902001bd797ffbeeps @param ssh_cmd: The ssh command to use in performing the remote wget. 23546dadc9439355f72d394dcc4700902001bd797ffbeeps """ 23646dadc9439355f72d394dcc4700902001bd797ffbeeps wget_cmd = ("wget -O - %s | %s 'cat >%s'" % 23746dadc9439355f72d394dcc4700902001bd797ffbeeps (source_url, ssh_cmd, dest_path)) 238ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi utils.run(wget_cmd) 23946dadc9439355f72d394dcc4700902001bd797ffbeeps 2403cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 241266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette_MAX_LAB_STATUS_ATTEMPTS = 5 242266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnettedef _get_lab_status(status_url): 2433cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Grabs the current lab status and message. 2443cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 245266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette @returns The JSON object obtained from the given URL. 246266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 2473cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 2483cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette retry_waittime = 1 249266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette for _ in range(_MAX_LAB_STATUS_ATTEMPTS): 2503cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette try: 2513cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette response = urllib2.urlopen(status_url) 2523cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette except IOError as e: 253266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette logging.debug('Error occurred when grabbing the lab status: %s.', 2543cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette e) 2553cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette time.sleep(retry_waittime) 2563cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette continue 2573cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # Check for successful response code. 2583cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette if response.getcode() == 200: 259266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return json.load(response) 2603cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette time.sleep(retry_waittime) 261266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return None 2623cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 2633cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 264abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnettedef _decode_lab_status(lab_status, build): 265266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette """Decode lab status, and report exceptions as needed. 2663cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 267abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette Take a deserialized JSON object from the lab status page, and 268abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette interpret it to determine the actual lab status. Raise 269266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette exceptions as required to report when the lab is down. 2703cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 271abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @param build: build name that we want to check the status of. 2723cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 273abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @raises TestLabException Raised if a request to test for the given 274abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette status and build should be blocked. 2753cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 2763cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # First check if the lab is up. 277266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette if not lab_status['general_state'] in LAB_GOOD_STATES: 278abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette raise TestLabException('Chromium OS Test Lab is closed: ' 279abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette '%s.' % lab_status['message']) 2803cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 281abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # Check if the build we wish to use is disabled. 2823cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # Lab messages should be in the format of: 283abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # Lab is 'status' [regex ...] (comment) 284abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # If the build name matches any regex, it will be blocked. 285abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette build_exceptions = re.search('\[(.*)\]', lab_status['message']) 286ae43721d4a11c6ecd2d502bba73b185b05bfce29Prashanth Balasubramanian if not build_exceptions or not build: 287abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette return 288abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette for build_pattern in build_exceptions.group(1).split(): 2897f215d3906ac204694d64b7eaa85155777f88f39J. Richard Barnette if re.match(build_pattern, build): 290abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette raise TestLabException('Chromium OS Test Lab is closed: ' 291abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette '%s matches %s.' % ( 292abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette build, build_pattern)) 2933cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette return 294266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 295266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 29694234cb747dc067b159789a4b63f21b23d706922Dan Shidef is_in_lab(): 29794234cb747dc067b159789a4b63f21b23d706922Dan Shi """Check if current Autotest instance is in lab 29894234cb747dc067b159789a4b63f21b23d706922Dan Shi 29994234cb747dc067b159789a4b63f21b23d706922Dan Shi @return: True if the Autotest instance is in lab. 30094234cb747dc067b159789a4b63f21b23d706922Dan Shi """ 301d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi test_server_name = CONFIG.get_config_value('SERVER', 'hostname') 30294234cb747dc067b159789a4b63f21b23d706922Dan Shi return test_server_name.startswith('cautotest') 30394234cb747dc067b159789a4b63f21b23d706922Dan Shi 30494234cb747dc067b159789a4b63f21b23d706922Dan Shi 305abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnettedef check_lab_status(build): 306abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette """Check if the lab status allows us to schedule for a build. 307266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 308abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette Checks if the lab is down, or if testing for the requested build 309abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette should be blocked. 310266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 311abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @param build: Name of the build to be scheduled for testing. 312266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 313abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @raises TestLabException Raised if a request to test for the given 314abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette status and build should be blocked. 315266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 316266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette """ 317266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # Ensure we are trying to schedule on the actual lab. 31894234cb747dc067b159789a4b63f21b23d706922Dan Shi if not is_in_lab(): 319266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return 320266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 321266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # Download the lab status from its home on the web. 322d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi status_url = CONFIG.get_config_value('CROS', 'lab_status_url') 323266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette json_status = _get_lab_status(status_url) 324266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette if json_status is None: 325266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # We go ahead and say the lab is open if we can't get the status. 32604be2bd5e4666a5c253e9c30ab20555e04286032Ilja H. Friedel logging.warning('Could not get a status from %s', status_url) 327266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return 328abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette _decode_lab_status(json_status, build) 329023afc65f9377db51ff6122977a9f529a32422d3beeps 330023afc65f9377db51ff6122977a9f529a32422d3beeps 331023afc65f9377db51ff6122977a9f529a32422d3beepsdef lock_host_with_labels(afe, lock_manager, labels): 332023afc65f9377db51ff6122977a9f529a32422d3beeps """Lookup and lock one host that matches the list of input labels. 333023afc65f9377db51ff6122977a9f529a32422d3beeps 334023afc65f9377db51ff6122977a9f529a32422d3beeps @param afe: An instance of the afe class, as defined in server.frontend. 335023afc65f9377db51ff6122977a9f529a32422d3beeps @param lock_manager: A lock manager capable of locking hosts, eg the 336023afc65f9377db51ff6122977a9f529a32422d3beeps one defined in server.cros.host_lock_manager. 337023afc65f9377db51ff6122977a9f529a32422d3beeps @param labels: A list of labels to look for on hosts. 338023afc65f9377db51ff6122977a9f529a32422d3beeps 339023afc65f9377db51ff6122977a9f529a32422d3beeps @return: The hostname of a host matching all labels, and locked through the 340023afc65f9377db51ff6122977a9f529a32422d3beeps lock_manager. The hostname will be as specified in the database the afe 341023afc65f9377db51ff6122977a9f529a32422d3beeps object is associated with, i.e if it exists in afe_hosts with a .cros 342023afc65f9377db51ff6122977a9f529a32422d3beeps suffix, the hostname returned will contain a .cros suffix. 343023afc65f9377db51ff6122977a9f529a32422d3beeps 344023afc65f9377db51ff6122977a9f529a32422d3beeps @raises: error.NoEligibleHostException: If no hosts matching the list of 345023afc65f9377db51ff6122977a9f529a32422d3beeps input labels are available. 346023afc65f9377db51ff6122977a9f529a32422d3beeps @raises: error.TestError: If unable to lock a host matching the labels. 347023afc65f9377db51ff6122977a9f529a32422d3beeps """ 348023afc65f9377db51ff6122977a9f529a32422d3beeps potential_hosts = afe.get_hosts(multiple_labels=labels) 349023afc65f9377db51ff6122977a9f529a32422d3beeps if not potential_hosts: 350023afc65f9377db51ff6122977a9f529a32422d3beeps raise error.NoEligibleHostException( 351023afc65f9377db51ff6122977a9f529a32422d3beeps 'No devices found with labels %s.' % labels) 352023afc65f9377db51ff6122977a9f529a32422d3beeps 353023afc65f9377db51ff6122977a9f529a32422d3beeps # This prevents errors where a fault might seem repeatable 354023afc65f9377db51ff6122977a9f529a32422d3beeps # because we lock, say, the same packet capturer for each test run. 355023afc65f9377db51ff6122977a9f529a32422d3beeps random.shuffle(potential_hosts) 356023afc65f9377db51ff6122977a9f529a32422d3beeps for host in potential_hosts: 357023afc65f9377db51ff6122977a9f529a32422d3beeps if lock_manager.lock([host.hostname]): 358023afc65f9377db51ff6122977a9f529a32422d3beeps logging.info('Locked device %s with labels %s.', 359023afc65f9377db51ff6122977a9f529a32422d3beeps host.hostname, labels) 360023afc65f9377db51ff6122977a9f529a32422d3beeps return host.hostname 361023afc65f9377db51ff6122977a9f529a32422d3beeps else: 362023afc65f9377db51ff6122977a9f529a32422d3beeps logging.info('Unable to lock device %s with labels %s.', 363023afc65f9377db51ff6122977a9f529a32422d3beeps host.hostname, labels) 364023afc65f9377db51ff6122977a9f529a32422d3beeps 365023afc65f9377db51ff6122977a9f529a32422d3beeps raise error.TestError('Could not lock a device with labels %s' % labels) 3667e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3677e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3687e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef get_test_views_from_tko(suite_job_id, tko): 3697e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Get test name and result for given suite job ID. 3707e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3717e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param suite_job_id: ID of suite job. 3727e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param tko: an instance of TKO as defined in server/frontend.py. 3737e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @return: A dictionary of test status keyed by test name, e.g., 3747e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'} 3757e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @raise: Exception when there is no test view found. 3767e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3777e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 3787e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi views = tko.run('get_detailed_test_views', afe_job_id=suite_job_id) 3797e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi relevant_views = filter(job_status.view_is_relevant, views) 3807e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not relevant_views: 3817e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise Exception('Failed to retrieve job results.') 3827e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3837e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views = {} 3847e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi for view in relevant_views: 3857e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views[view['test_name']] = view['status'] 3867e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3877e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi return test_views 38835d661e09666d315325f8942d06949ca7283666fMK Ryu 38935d661e09666d315325f8942d06949ca7283666fMK Ryu 390c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryudef get_data_key(prefix, suite, build, board): 391c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu """ 392c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu Constructs a key string from parameters. 393c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 394c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param prefix: Prefix for the generating key. 395c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param suite: a suite name. e.g., bvt-cq, bvt-inline, dummy 396c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param build: The build string. This string should have a consistent 397c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu format eg: x86-mario-release/R26-3570.0.0. If the format of this 398c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu string changes such that we can't determine build_type or branch 399c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu we give up and use the parametes we're sure of instead (suite, 400c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu board). eg: 401c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 1. build = x86-alex-pgo-release/R26-3570.0.0 402c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu branch = 26 403c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = pgo-release 404c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 2. build = lumpy-paladin/R28-3993.0.0-rc5 405c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu branch = 28 406c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = paladin 407c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @param board: The board that this suite ran on. 408c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu @return: The key string used for a dictionary. 409c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu """ 410c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu try: 411c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu _board, build_type, branch = ParseBuildName(build)[:3] 412c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu except ParseBuildNameException as e: 413c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu logging.error(str(e)) 414c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu branch = 'Unknown' 415c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = 'Unknown' 416c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu else: 417c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu embedded_str = re.search(r'x86-\w+-(.*)', _board) 418c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu if embedded_str: 419c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu build_type = embedded_str.group(1) + '-' + build_type 420c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 421c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu data_key_dict = { 422c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'prefix': prefix, 423c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'board': board, 424c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'branch': branch, 425c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'build_type': build_type, 426c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu 'suite': suite, 427c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu } 428c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu return ('%(prefix)s.%(board)s.%(build_type)s.%(branch)s.%(suite)s' 429c9c0c3ff2e8800bfad3f0c79fe5879be0ec78489MK Ryu % data_key_dict) 43083184356b60f4352e46e69488d54222032d426c0MK Ryu 43183184356b60f4352e46e69488d54222032d426c0MK Ryu 4322d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryudef setup_logging(logfile=None, prefix=False): 43383184356b60f4352e46e69488d54222032d426c0MK Ryu """Setup basic logging with all logging info stripped. 43483184356b60f4352e46e69488d54222032d426c0MK Ryu 43583184356b60f4352e46e69488d54222032d426c0MK Ryu Calls to logging will only show the message. No severity is logged. 43683184356b60f4352e46e69488d54222032d426c0MK Ryu 43783184356b60f4352e46e69488d54222032d426c0MK Ryu @param logfile: If specified dump output to a file as well. 4382d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu @param prefix: Flag for log prefix. Set to True to add prefix to log 4392d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu entries to include timestamp and log level. Default is False. 44083184356b60f4352e46e69488d54222032d426c0MK Ryu """ 44183184356b60f4352e46e69488d54222032d426c0MK Ryu # Remove all existing handlers. client/common_lib/logging_config adds 44283184356b60f4352e46e69488d54222032d426c0MK Ryu # a StreamHandler to logger when modules are imported, e.g., 44383184356b60f4352e46e69488d54222032d426c0MK Ryu # autotest_lib.client.bin.utils. A new StreamHandler will be added here to 44483184356b60f4352e46e69488d54222032d426c0MK Ryu # log only messages, not severity. 44583184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().handlers = [] 44683184356b60f4352e46e69488d54222032d426c0MK Ryu 4472d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu if prefix: 4482d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu log_format = '%(asctime)s %(levelname)-5s| %(message)s' 4492d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu else: 4502d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu log_format = '%(message)s' 4512d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu 45283184356b60f4352e46e69488d54222032d426c0MK Ryu screen_handler = logging.StreamHandler() 4532d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu screen_handler.setFormatter(logging.Formatter(log_format)) 45483184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().addHandler(screen_handler) 45583184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().setLevel(logging.INFO) 45683184356b60f4352e46e69488d54222032d426c0MK Ryu if logfile: 45783184356b60f4352e46e69488d54222032d426c0MK Ryu file_handler = logging.FileHandler(logfile) 4582d0a364b21297418bef4d7f32acee8831e4eeefeMK Ryu file_handler.setFormatter(logging.Formatter(log_format)) 45983184356b60f4352e46e69488d54222032d426c0MK Ryu file_handler.setLevel(logging.DEBUG) 46083184356b60f4352e46e69488d54222032d426c0MK Ryu logging.getLogger().addHandler(file_handler) 4618c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4628c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4638c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramaniandef is_shard(): 4648c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian """Determines if this instance is running as a shard. 4658c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4668c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian Reads the global_config value shard_hostname in the section SHARD. 4678c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian 4688c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian @return True, if shard_hostname is set, False otherwise. 4698c98ac10beaa08bfb975c412b0b3bda23178763aPrashanth Balasubramanian """ 470d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi hostname = CONFIG.get_config_value('SHARD', 'shard_hostname', default=None) 4710c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return bool(hostname) 4720c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 4730c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 4740cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Dengdef get_global_afe_hostname(): 4750cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Deng """Read the hostname of the global AFE from the global configuration.""" 476d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi return CONFIG.get_config_value('SERVER', 'global_afe_hostname') 4770cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Deng 4780cb2a3b1d2d86d70da06a3f45be9297139e48207Fang Deng 47918699fef3341507c6b0d415ced96f6e25e3c03eaFang Dengdef is_restricted_user(username): 48018699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng """Determines if a user is in a restricted group. 48118699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 48218699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng User in restricted group only have access to master. 48318699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 48418699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng @param username: A string, representing a username. 48518699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 48618699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng @returns: True if the user is in a restricted group. 48718699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng """ 48818699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng if not username: 48918699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng return False 49018699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 491d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi restricted_groups = CONFIG.get_config_value( 49218699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 'AUTOTEST_WEB', 'restricted_groups', default='').split(',') 49318699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng for group in restricted_groups: 4945229c85219e09d6a7857701be20f286ba04809b3Fang Deng try: 4955229c85219e09d6a7857701be20f286ba04809b3Fang Deng if group and username in grp.getgrnam(group).gr_mem: 4965229c85219e09d6a7857701be20f286ba04809b3Fang Deng return True 4975229c85219e09d6a7857701be20f286ba04809b3Fang Deng except KeyError as e: 4985229c85219e09d6a7857701be20f286ba04809b3Fang Deng logging.debug("%s is not a valid group.", group) 49918699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng return False 50018699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 50118699fef3341507c6b0d415ced96f6e25e3c03eaFang Deng 5020c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_special_task_status(is_complete, success, is_active): 5030c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Get the status of a special task. 5040c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5050c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu Emulate a host queue entry status for a special task 5060c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu Although SpecialTasks are not HostQueueEntries, it is helpful to 5070c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu the user to present similar statuses. 5080c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5090c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param is_complete Boolean if the task is completed. 5100c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param success Boolean if the task succeeded. 5110c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param is_active Boolean if the task is active. 5120c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5130c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @return The status of a special task. 5140c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5150c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if is_complete: 5160c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if success: 5170c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.COMPLETED 5180c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.FAILED 5190c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if is_active: 5200c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.RUNNING 5210c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return host_queue_entry_states.Status.QUEUED 5220c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5230c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5240c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_special_task_exec_path(hostname, task_id, task_name, time_requested): 5250c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Get the execution path of the SpecialTask. 5260c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5270c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu This method returns different paths depending on where a 5280c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu the task ran: 5290c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu * Master: hosts/hostname/task_id-task_type 5300c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu * Shard: Master_path/time_created 5310c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu This is to work around the fact that a shard can fail independent 5320c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu of the master, and be replaced by another shard that has the same 5330c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu hosts. Without the time_created stamp the logs of the tasks running 5340c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu on the second shard will clobber the logs from the first in google 5350c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu storage, because task ids are not globally unique. 5360c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5370c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param hostname Hostname 5380c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param task_id Special task id 5390c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param task_name Special task name (e.g., Verify, Repair, etc) 5400c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param time_requested Special task requested time. 5410c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5420c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @return An execution path for the task. 5430c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5440c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu results_path = 'hosts/%s/%s-%s' % (hostname, task_id, task_name.lower()) 5450c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5460c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # If we do this on the master it will break backward compatibility, 5470c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # as there are tasks that currently don't have timestamps. If a host 5480c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # or job has been sent to a shard, the rpc for that host/job will 5490c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # be redirected to the shard, so this global_config check will happen 5500c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # on the shard the logs are on. 5510c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu if not is_shard(): 5520c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return results_path 5530c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5540c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # Generate a uid to disambiguate special task result directories 5550c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # in case this shard fails. The simplest uid is the job_id, however 5560c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # in rare cases tasks do not have jobs associated with them (eg: 5570c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # frontend verify), so just use the creation timestamp. The clocks 5580c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # between a shard and master should always be in sync. Any discrepancies 5590c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # will be brought to our attention in the form of job timeouts. 5600c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu uid = time_requested.strftime('%Y%d%m%H%M%S') 5610c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5620c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # TODO: This is a hack, however it is the easiest way to achieve 5630c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # correctness. There is currently some debate over the future of 5640c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # tasks in our infrastructure and refactoring everything right 5650c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu # now isn't worth the time. 5660c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return '%s/%s' % (results_path, uid) 5670c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5680c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5690c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_job_tag(id, owner): 5700c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Returns a string tag for a job. 5710c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5720c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param id Job id 5730c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param owner Job owner 5740c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5750c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5760c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return '%s-%s' % (id, owner) 5770c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5780c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5790c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryudef get_hqe_exec_path(tag, execution_subdir): 5800c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """Returns a execution path to a HQE's results. 5810c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5820c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param tag Tag string for a job associated with a HQE. 5830c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu @param execution_subdir Execution sub-directory string of a HQE. 5840c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu 5850c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu """ 5860c1a37dd9b1237fe8d43c7f911ce601104806339MK Ryu return os.path.join(tag, execution_subdir) 58782997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 58882997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 58982997b9a9791105d81c9db14e75ed14946f78f94Dan Shidef is_inside_chroot(): 59082997b9a9791105d81c9db14e75ed14946f78f94Dan Shi """Check if the process is running inside chroot. 59182997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 59282997b9a9791105d81c9db14e75ed14946f78f94Dan Shi This is a wrapper around chromite.lib.cros_build_lib.IsInsideChroot(). The 59382997b9a9791105d81c9db14e75ed14946f78f94Dan Shi method checks if cros_build_lib can be imported first. 59482997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 59582997b9a9791105d81c9db14e75ed14946f78f94Dan Shi @return: True if the process is running inside chroot or cros_build_lib 59682997b9a9791105d81c9db14e75ed14946f78f94Dan Shi cannot be imported. 59782997b9a9791105d81c9db14e75ed14946f78f94Dan Shi 59882997b9a9791105d81c9db14e75ed14946f78f94Dan Shi """ 59916b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu try: 60016b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # TODO(crbug.com/739466) This module import is delayed because it adds 60116b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # 1-2 seconds to the module import time and most users of site_utils 60216b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # don't need it. The correct fix is to break apart site_utils into more 60316b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # meaningful chunks. 60416b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu from chromite.lib import cros_build_lib 60516b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu except ImportError: 60616b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu logging.warn('Unable to import chromite. Can not detect chroot. ' 60716b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu 'Defaulting to False') 60816b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu return False 60916b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu return cros_build_lib.IsInsideChroot() 61070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 61170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 61270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shidef parse_job_name(name): 61370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi """Parse job name to get information including build, board and suite etc. 61470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 61570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi Suite job created by run_suite follows the naming convention of: 61670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi [build]-test_suites/control.[suite] 61770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi For example: lumpy-release/R46-7272.0.0-test_suites/control.bvt 618cdd00f20ac7607ff89a97e83e2483a4c8feddb7bAllen Li The naming convention is defined in rpc_interface.create_suite_job. 61970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 62070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi Test job created by suite job follows the naming convention of: 62170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi [build]/[suite]/[test name] 62270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi For example: lumpy-release/R46-7272.0.0/bvt/login_LoginSuccess 62370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi The naming convention is defined in 62470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi server/cros/dynamic_suite/tools.create_job_name 62570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 62670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi Note that pgo and chrome-perf builds will fail the method. Since lab does 62770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi not run test for these builds, they can be ignored. 628ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi Also, tests for Launch Control builds have different naming convention. 629ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi The build ID will be used as build_version. 63070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 63170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi @param name: Name of the job. 63270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 63370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi @return: A dictionary containing the test information. The keyvals include: 63470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi build: Name of the build, e.g., lumpy-release/R46-7272.0.0 63570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi build_version: The version of the build, e.g., R46-7272.0.0 63670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi board: Name of the board, e.g., lumpy 63770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi suite: Name of the test suite, e.g., bvt 63870647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi 63970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi """ 64070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info = {} 641ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi suite_job_regex = '([^/]*/[^/]*(?:/\d+)?)-test_suites/control\.(.*)' 642ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi test_job_regex = '([^/]*/[^/]*(?:/\d+)?)/([^/]+)/.*' 64370647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi match = re.match(suite_job_regex, name) 64470647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi if not match: 64570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi match = re.match(test_job_regex, name) 64670647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi if match: 64770647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['build'] = match.groups()[0] 64870647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['suite'] = match.groups()[1] 64970647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['build_version'] = info['build'].split('/')[1] 65070647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi try: 65170647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi info['board'], _, _, _ = ParseBuildName(info['build']) 65270647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi except ParseBuildNameException: 653ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi # Try to parse it as Launch Control build 654ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi # Launch Control builds have name format: 655ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi # branch/build_target-build_type/build_id. 656ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi try: 657ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi _, target, build_id = utils.parse_launch_control_build( 658ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi info['build']) 659ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi build_target, _ = utils.parse_launch_control_target(target) 660ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi if build_target: 661ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi info['board'] = build_target 662ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi info['build_version'] = build_id 663ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi except ValueError: 664ef31f03d7347b40c431754a6280b5ff9eea2903fDan Shi pass 66570647cafbd061a7754ac304fd9dc067f2b6dbab4Dan Shi return info 6663a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6673a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6683a4a57a7704ab15136286628f465dbb2d1a56171Kevin Chengdef add_label_detector(label_function_list, label_list=None, label=None): 6693a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """Decorator used to group functions together into the provided list. 6703a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6713a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng This is a helper function to automatically add label functions that have 6723a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng the label decorator. This is to help populate the class list of label 6733a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng functions to be retrieved by the get_labels class method. 6743a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng 6753a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param label_function_list: List of label detecting functions to add 6763a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng decorated function to. 6773a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param label_list: List of detectable labels to add detectable labels to. 6783a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng (Default: None) 6793a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param label: Label string that is detectable by this detection function 6803a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng (Default: None) 6813a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """ 6823a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng def add_func(func): 6833a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """ 6843a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng @param func: The function to be added as a detector. 6853a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng """ 6863a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng label_function_list.append(func) 6873a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng if label and label_list is not None: 6883a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng label_list.append(label) 6893a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng return func 6903a4a57a7704ab15136286628f465dbb2d1a56171Kevin Cheng return add_func 6919f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi 6929f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi 6939f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basidef verify_not_root_user(): 6949f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi """Simple function to error out if running with uid == 0""" 6959f364a6fe1fedaecc652f111cd143ab2ee7ca825Simran Basi if os.getuid() == 0: 6961bf60eb788365f083d0ee8045a6556f906149decSimran Basi raise error.IllegalUser('This script can not be ran as root.') 6971bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6981bf60eb788365f083d0ee8045a6556f906149decSimran Basi 6991bf60eb788365f083d0ee8045a6556f906149decSimran Basidef get_hostname_from_machine(machine): 7001bf60eb788365f083d0ee8045a6556f906149decSimran Basi """Lookup hostname from a machine string or dict. 7011bf60eb788365f083d0ee8045a6556f906149decSimran Basi 7021bf60eb788365f083d0ee8045a6556f906149decSimran Basi @returns: Machine hostname in string format. 7031bf60eb788365f083d0ee8045a6556f906149decSimran Basi """ 7041bf60eb788365f083d0ee8045a6556f906149decSimran Basi hostname, _ = get_host_info_from_machine(machine) 7051bf60eb788365f083d0ee8045a6556f906149decSimran Basi return hostname 7061bf60eb788365f083d0ee8045a6556f906149decSimran Basi 7071bf60eb788365f083d0ee8045a6556f906149decSimran Basi 7081bf60eb788365f083d0ee8045a6556f906149decSimran Basidef get_host_info_from_machine(machine): 7091bf60eb788365f083d0ee8045a6556f906149decSimran Basi """Lookup host information from a machine string or dict. 7101bf60eb788365f083d0ee8045a6556f906149decSimran Basi 71105ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng @returns: Tuple of (hostname, afe_host) 7121bf60eb788365f083d0ee8045a6556f906149decSimran Basi """ 7131bf60eb788365f083d0ee8045a6556f906149decSimran Basi if isinstance(machine, dict): 71405ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return (machine['hostname'], machine['afe_host']) 7151bf60eb788365f083d0ee8045a6556f906149decSimran Basi else: 71605ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return (machine, EmptyAFEHost()) 71705ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 71805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 71905ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Chengdef get_afe_host_from_machine(machine): 72005ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """Return the afe_host from the machine dict if possible. 72105ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng 72205ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng @returns: AFE host object. 72305ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng """ 72405ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng _, afe_host = get_host_info_from_machine(machine) 72505ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return afe_host 726f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 727f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 728f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Dengdef get_creds_abspath(creds_file): 729f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng """Returns the abspath of the credentials file. 730f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 731f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng If creds_file is already an absolute path, just return it. 732f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng Otherwise, assume it is located in the creds directory 733f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng specified in global_config and return the absolute path. 734f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng 735f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng @param: creds_path, a path to the credentials. 736f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng @return: An absolute path to the credentials file. 737f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng """ 738f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng if not creds_file: 739f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng return None 740f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng if os.path.isabs(creds_file): 741f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng return creds_file 742d37736b67febeebd991d0209dd030635f7ff5cfdDan Shi creds_dir = CONFIG.get_config_value('SERVER', 'creds_dir', default='') 743f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng if not creds_dir or not os.path.exists(creds_dir): 744f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng creds_dir = common.autotest_dir 745f8a94e2adf24e017be4e9d0687d4ab8f46ac4fceFang Deng return os.path.join(creds_dir, creds_file) 7463b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7473b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7483b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Chengdef machine_is_testbed(machine): 7493b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng """Checks if the machine is a testbed. 7503b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7513b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng The signal we use to determine if the machine is a testbed 7523b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng is if the host attributes contain more than 1 serial. 7533b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7543b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng @param machine: is a list of dicts 7553b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng 7563b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng @return: True if the machine is a testbed, False otherwise. 7573b11181c9ea282d584663d362bcfe2d1b4f9ba51Kevin Cheng """ 75805ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng _, afe_host = get_host_info_from_machine(machine) 75905ae2a45397188dd7d7adbc49a61095ebf85e151Kevin Cheng return len(afe_host.attributes.get('serials', '').split(',')) > 1 76020cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 76120cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 76220cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbsdef SetupTsMonGlobalState(*args, **kwargs): 76320cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs """Import-safe wrap around chromite.lib.ts_mon_config's setup function. 76420cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 76520cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs @param *args: Args to pass through. 76620cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs @param **kwargs: Kwargs to pass through. 76720cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs """ 76816b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu try: 76916b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # TODO(crbug.com/739466) This module import is delayed because it adds 77016b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # 1-2 seconds to the module import time and most users of site_utils 77116b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # don't need it. The correct fix is to break apart site_utils into more 77216b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu # meaningful chunks. 77316b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu from chromite.lib import ts_mon_config 77416b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu except ImportError: 77516b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu logging.warn('Unable to import chromite. Monarch is disabled.') 776930ed9d0a30b546e936254a7c941caf263479781Paul Hobbs return TrivialContextManager() 77716b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu 77816b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu try: 77916b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu context = ts_mon_config.SetupTsMonGlobalState(*args, **kwargs) 78016b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu if hasattr(context, '__exit__'): 78116b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu return context 78216b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu except Exception as e: 78316b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu logging.warning('Caught an exception trying to setup ts_mon, ' 78416b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu 'monitoring is disabled: %s', e, exc_info=True) 78516b46f8039a9e29fa036f1347dcadcacac6a7d77Prathmesh Prabhu return TrivialContextManager() 78620cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 78720cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs 78820cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs@contextlib.contextmanager 789604fc875f5d910bb2683680af07adada427be0d4Paul Hobbsdef TrivialContextManager(*args, **kwargs): 790604fc875f5d910bb2683680af07adada427be0d4Paul Hobbs """Context manager that does nothing. 791604fc875f5d910bb2683680af07adada427be0d4Paul Hobbs 792604fc875f5d910bb2683680af07adada427be0d4Paul Hobbs @param *args: Ignored args 793604fc875f5d910bb2683680af07adada427be0d4Paul Hobbs @param **kwargs: Ignored kwargs. 794604fc875f5d910bb2683680af07adada427be0d4Paul Hobbs """ 79520cc72ac289b10e362e2865f4b662425f6ea1a6dPaul Hobbs yield 7965f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 7975f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 7985f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Chengdef wait_for_idle_duts(duts, afe, max_wait=IDLE_DUT_WAIT_TIMEOUT): 7995f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng """Wait for the hosts to all go idle. 8005f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8015f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @param duts: List of duts to check for idle state. 8025f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @param afe: afe instance. 8035f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @param max_wait: Max wait time in seconds. 8045f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8055f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @returns Boolean True if all hosts are idle or False if any hosts did not 8065f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng go idle within max_wait. 8075f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng """ 8085f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng start_time = time.time() 8095f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng # We make a shallow copy since we're going to be modifying active_dut_list. 8105f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng active_dut_list = duts[:] 8115f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng while active_dut_list: 8125f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng # Let's rate-limit how often we hit the AFE. 8135f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng time.sleep(1) 8145f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8155f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng # Check if we've waited too long. 8165f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng if (time.time() - start_time) > max_wait: 8175f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng return False 8185f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8195f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng idle_duts = [] 8205f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng # Get the status for the duts and see if they're in the idle state. 8215f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng afe_hosts = afe.get_hosts(active_dut_list) 8225f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng idle_duts = [afe_host.hostname for afe_host in afe_hosts 8235f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng if afe_host.status in host_states.IDLE_STATES] 8245f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8255f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng # Take out idle duts so we don't needlessly check them 8265f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng # next time around. 8275f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng for idle_dut in idle_duts: 8285f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng active_dut_list.remove(idle_dut) 8295f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8305f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng logging.info('still waiting for following duts to go idle: %s', 8315f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng active_dut_list) 8325f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng return True 8335f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8345f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8355f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng@contextlib.contextmanager 8365f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Chengdef lock_duts_and_wait(duts, afe, lock_msg='default lock message', 8375f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng max_wait=IDLE_DUT_WAIT_TIMEOUT): 8385f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng """Context manager to lock the duts and wait for them to go idle. 8395f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8405f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @param duts: List of duts to lock. 8415f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @param afe: afe instance. 8425f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng 8435f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng @returns Boolean lock_success where True if all duts locked successfully or 8445f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng False if we timed out waiting too long for hosts to go idle. 8455f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng """ 8465f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng try: 8475f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng locked_duts = [] 8485f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng duts.sort() 8495f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng for dut in duts: 8505f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng if afe.lock_host(dut, lock_msg, fail_if_locked=True): 8515f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng locked_duts.append(dut) 8525f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng else: 8535f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng logging.info('%s already locked', dut) 8545f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng yield wait_for_idle_duts(locked_duts, afe, max_wait) 8555f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng finally: 8565f2ba6cf0b4617abb5e029cd8210942f2c2ec36fKevin Cheng afe.unlock_hosts(locked_duts) 857b5b8b4f981036971c619b26956c8847140eabd35Dan Shi 858b5b8b4f981036971c619b26956c8847140eabd35Dan Shi 859b5b8b4f981036971c619b26956c8847140eabd35Dan Shidef board_labels_allowed(boards): 860b5b8b4f981036971c619b26956c8847140eabd35Dan Shi """Check if the list of board labels can be set to a single host. 861b5b8b4f981036971c619b26956c8847140eabd35Dan Shi 862b5b8b4f981036971c619b26956c8847140eabd35Dan Shi The only case multiple board labels can be set to a single host is for 863b5b8b4f981036971c619b26956c8847140eabd35Dan Shi testbed, which may have a list of board labels like 864b5b8b4f981036971c619b26956c8847140eabd35Dan Shi board:angler-1, board:angler-2, board:angler-3, board:marlin-1' 865b5b8b4f981036971c619b26956c8847140eabd35Dan Shi 866b5b8b4f981036971c619b26956c8847140eabd35Dan Shi @param boards: A list of board labels (may include platform label). 867b5b8b4f981036971c619b26956c8847140eabd35Dan Shi 868b5b8b4f981036971c619b26956c8847140eabd35Dan Shi @returns True if the the list of boards can be set to a single host. 869b5b8b4f981036971c619b26956c8847140eabd35Dan Shi """ 870b5b8b4f981036971c619b26956c8847140eabd35Dan Shi # Filter out any non-board labels 871b5b8b4f981036971c619b26956c8847140eabd35Dan Shi boards = [b for b in boards if re.match('board:.*', b)] 872b5b8b4f981036971c619b26956c8847140eabd35Dan Shi if len(boards) <= 1: 873b5b8b4f981036971c619b26956c8847140eabd35Dan Shi return True 874b5b8b4f981036971c619b26956c8847140eabd35Dan Shi for board in boards: 875b5b8b4f981036971c619b26956c8847140eabd35Dan Shi if not re.match('board:[^-]+-\d+', board): 876b5b8b4f981036971c619b26956c8847140eabd35Dan Shi return False 877b5b8b4f981036971c619b26956c8847140eabd35Dan Shi return True 878