site_utils.py revision 94234cb747dc067b159789a4b63f21b23d706922
1a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# Copyright (c) 2013 The Chromium Authors. All rights reserved. 2a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# Use of this source code is governed by a BSD-style license that can be 3a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi# found in the LICENSE file. 4a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 5dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 63cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport httplib 73cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport json 8a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shiimport logging 9023afc65f9377db51ff6122977a9f529a32422d3beepsimport random 10dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Millerimport re 113cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport time 12bef578d9163c6574d626e8b98298c59a5ab79221Paul Drewsimport urllib2 13a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 143cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteimport common 15023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import base_utils 16023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import error 17023afc65f9377db51ff6122977a9f529a32422d3beepsfrom autotest_lib.client.common_lib import global_config 18a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shifrom autotest_lib.server.cros.dynamic_suite import constants 197e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shifrom autotest_lib.server.cros.dynamic_suite import job_status 20a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 21a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 22dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller_SHERIFF_JS = global_config.global_config.get_config_value( 23dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 'NOTIFICATIONS', 'sheriffs', default='') 243197b39f82eb92afff33c7d44b805afe120c7627Fang Deng_LAB_SHERIFF_JS = global_config.global_config.get_config_value( 253197b39f82eb92afff33c7d44b805afe120c7627Fang Deng 'NOTIFICATIONS', 'lab_sheriffs', default='') 26dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller_CHROMIUM_BUILD_URL = global_config.global_config.get_config_value( 27dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 'NOTIFICATIONS', 'chromium_build_url', default='') 28dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 293cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard BarnetteLAB_GOOD_STATES = ('open', 'throttled') 303cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 313cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 32abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnetteclass TestLabException(Exception): 33abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette """Exception raised when the Test Lab blocks a test or suite.""" 343cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette pass 353cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 363cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 373cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnetteclass ParseBuildNameException(Exception): 383cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Raised when ParseBuildName() cannot parse a build name.""" 393cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette pass 403cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 413cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 423cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnettedef ParseBuildName(name): 433cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Format a build name, given board, type, milestone, and manifest num. 443cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 453cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0' 463cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 473cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return board: board the manifest is for, e.g. x86-alex. 483cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return type: one of 'release', 'factory', or 'firmware' 493cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return milestone: (numeric) milestone the manifest was associated with. 503cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette @return manifest: manifest number, e.g. '2015.0.0' 513cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 523cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 533cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette match = re.match(r'([\w-]+)-(\w+)/R(\d+)-([\d.ab-]+)', name) 543cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette if match and len(match.groups()) == 4: 553cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette return match.groups() 563cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette raise ParseBuildNameException('%s is a malformed build name.' % name) 573cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 58dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 59a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_label_from_afe(hostname, label_prefix, afe): 60a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve a host's specific label from the AFE. 61a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 62a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks for a host label that has the form <label_prefix>:<value> 63a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi and returns the "<value>" part of the label. None is returned 64a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if there is not a label matching the pattern 65a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 66a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 67a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param label_prefix: prefix of label to be matched, e.g., |board:| 68a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 69a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns the label that matches the prefix or 'None' 70a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 71a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 72a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi labels = afe.get_labels(name__startswith=label_prefix, 73a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi host__hostname__in=[hostname]) 74a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if labels and len(labels) == 1: 75a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi return labels[0].name.split(label_prefix, 1)[1] 76a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 77a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 78a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_board_from_afe(hostname, afe): 79a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve given host's board from its labels in the AFE. 80a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 81a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks for a host label of the form "board:<board>", and 82a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi returns the "<board>" part of the label. `None` is returned 83a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi if there is not a single, unique label matching the pattern. 84a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 85a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 86a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 87a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns board from label, or `None`. 88a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 89a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 90a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi return get_label_from_afe(hostname, constants.BOARD_PREFIX, afe) 91a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 92a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 93a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shidef get_build_from_afe(hostname, afe): 94a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """Retrieve the current build for given host from the AFE. 95a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 96a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi Looks through the host's labels in the AFE to determine its build. 97a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 98a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param hostname: hostname of given DUT. 99a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @param afe: afe instance. 100a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi @returns The current build or None if it could not find it or if there 101a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi were multiple build labels assigned to this host. 102a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 103a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi """ 104a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi return get_label_from_afe(hostname, constants.VERSION_PREFIX, afe) 105a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 106a1ecd5c903928f359cd6cbcff5c986652e109599Dan Shi 1073197b39f82eb92afff33c7d44b805afe120c7627Fang Dengdef get_sheriffs(lab_only=False): 108dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller """ 109dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller Polls the javascript file that holds the identity of the sheriff and 110dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller parses it's output to return a list of chromium sheriff email addresses. 111dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller The javascript file can contain the ldap of more than one sheriff, eg: 112dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller document.write('sheriff_one, sheriff_two'). 113dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller 1143197b39f82eb92afff33c7d44b805afe120c7627Fang Deng @param lab_only: if True, only pulls lab sheriff. 1153197b39f82eb92afff33c7d44b805afe120c7627Fang Deng @return: A list of chroium.org sheriff email addresses to cc on the bug. 1163197b39f82eb92afff33c7d44b805afe120c7627Fang Deng An empty list if failed to parse the javascript. 117dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller """ 118dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller sheriff_ids = [] 1193197b39f82eb92afff33c7d44b805afe120c7627Fang Deng sheriff_js_list = _LAB_SHERIFF_JS.split(',') 1203197b39f82eb92afff33c7d44b805afe120c7627Fang Deng if not lab_only: 1213197b39f82eb92afff33c7d44b805afe120c7627Fang Deng sheriff_js_list.extend(_SHERIFF_JS.split(',')) 1223197b39f82eb92afff33c7d44b805afe120c7627Fang Deng 1233197b39f82eb92afff33c7d44b805afe120c7627Fang Deng for sheriff_js in sheriff_js_list: 124dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller try: 125dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller url_content = base_utils.urlopen('%s%s'% ( 126dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller _CHROMIUM_BUILD_URL, sheriff_js)).read() 127dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller except (ValueError, IOError) as e: 1284efdf03d1057f9d38e043b7c2affe842109805d2beeps logging.warning('could not parse sheriff from url %s%s: %s', 1294efdf03d1057f9d38e043b7c2affe842109805d2beeps _CHROMIUM_BUILD_URL, sheriff_js, str(e)) 130bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews except (urllib2.URLError, httplib.HTTPException) as e: 131bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews logging.warning('unexpected error reading from url "%s%s": %s', 132bef578d9163c6574d626e8b98298c59a5ab79221Paul Drews _CHROMIUM_BUILD_URL, sheriff_js, str(e)) 133dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller else: 134dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller ldaps = re.search(r"document.write\('(.*)'\)", url_content) 135dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller if not ldaps: 1364efdf03d1057f9d38e043b7c2affe842109805d2beeps logging.warning('Could not retrieve sheriff ldaps for: %s', 1374efdf03d1057f9d38e043b7c2affe842109805d2beeps url_content) 138dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller continue 139dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller sheriff_ids += ['%s@chromium.org' % alias.replace(' ', '') 140dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller for alias in ldaps.group(1).split(',')] 141dadc2c21c0b0b904067ea6400e179a187c0247d4Alex Miller return sheriff_ids 14246dadc9439355f72d394dcc4700902001bd797ffbeeps 14346dadc9439355f72d394dcc4700902001bd797ffbeeps 14446dadc9439355f72d394dcc4700902001bd797ffbeepsdef remote_wget(source_url, dest_path, ssh_cmd): 14546dadc9439355f72d394dcc4700902001bd797ffbeeps """wget source_url from localhost to dest_path on remote host using ssh. 14646dadc9439355f72d394dcc4700902001bd797ffbeeps 14746dadc9439355f72d394dcc4700902001bd797ffbeeps @param source_url: The complete url of the source of the package to send. 14846dadc9439355f72d394dcc4700902001bd797ffbeeps @param dest_path: The path on the remote host's file system where we would 14946dadc9439355f72d394dcc4700902001bd797ffbeeps like to store the package. 15046dadc9439355f72d394dcc4700902001bd797ffbeeps @param ssh_cmd: The ssh command to use in performing the remote wget. 15146dadc9439355f72d394dcc4700902001bd797ffbeeps """ 15246dadc9439355f72d394dcc4700902001bd797ffbeeps wget_cmd = ("wget -O - %s | %s 'cat >%s'" % 15346dadc9439355f72d394dcc4700902001bd797ffbeeps (source_url, ssh_cmd, dest_path)) 15446dadc9439355f72d394dcc4700902001bd797ffbeeps base_utils.run(wget_cmd) 15546dadc9439355f72d394dcc4700902001bd797ffbeeps 1563cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 157266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette_MAX_LAB_STATUS_ATTEMPTS = 5 158266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnettedef _get_lab_status(status_url): 1593cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """Grabs the current lab status and message. 1603cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 161266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette @returns The JSON object obtained from the given URL. 162266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 1633cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 1643cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette retry_waittime = 1 165266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette for _ in range(_MAX_LAB_STATUS_ATTEMPTS): 1663cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette try: 1673cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette response = urllib2.urlopen(status_url) 1683cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette except IOError as e: 169266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette logging.debug('Error occurred when grabbing the lab status: %s.', 1703cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette e) 1713cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette time.sleep(retry_waittime) 1723cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette continue 1733cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # Check for successful response code. 1743cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette if response.getcode() == 200: 175266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return json.load(response) 1763cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette time.sleep(retry_waittime) 177266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return None 1783cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 1793cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 180abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnettedef _decode_lab_status(lab_status, build): 181266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette """Decode lab status, and report exceptions as needed. 1823cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 183abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette Take a deserialized JSON object from the lab status page, and 184abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette interpret it to determine the actual lab status. Raise 185266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette exceptions as required to report when the lab is down. 1863cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 187abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @param build: build name that we want to check the status of. 1883cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 189abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @raises TestLabException Raised if a request to test for the given 190abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette status and build should be blocked. 1913cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette """ 1923cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # First check if the lab is up. 193266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette if not lab_status['general_state'] in LAB_GOOD_STATES: 194abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette raise TestLabException('Chromium OS Test Lab is closed: ' 195abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette '%s.' % lab_status['message']) 1963cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette 197abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # Check if the build we wish to use is disabled. 1983cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette # Lab messages should be in the format of: 199abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # Lab is 'status' [regex ...] (comment) 200abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette # If the build name matches any regex, it will be blocked. 201abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette build_exceptions = re.search('\[(.*)\]', lab_status['message']) 202abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette if not build_exceptions: 203abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette return 204abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette for build_pattern in build_exceptions.group(1).split(): 205abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette if re.search(build_pattern, build): 206abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette raise TestLabException('Chromium OS Test Lab is closed: ' 207abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette '%s matches %s.' % ( 208abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette build, build_pattern)) 2093cbd76b06c383d8bc3e4ba959659a0bd04180761J. Richard Barnette return 210266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 211266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 21294234cb747dc067b159789a4b63f21b23d706922Dan Shidef is_in_lab(): 21394234cb747dc067b159789a4b63f21b23d706922Dan Shi """Check if current Autotest instance is in lab 21494234cb747dc067b159789a4b63f21b23d706922Dan Shi 21594234cb747dc067b159789a4b63f21b23d706922Dan Shi @return: True if the Autotest instance is in lab. 21694234cb747dc067b159789a4b63f21b23d706922Dan Shi """ 21794234cb747dc067b159789a4b63f21b23d706922Dan Shi test_server_name = global_config.global_config.get_config_value( 21894234cb747dc067b159789a4b63f21b23d706922Dan Shi 'SERVER', 'hostname') 21994234cb747dc067b159789a4b63f21b23d706922Dan Shi return test_server_name.startswith('cautotest') 22094234cb747dc067b159789a4b63f21b23d706922Dan Shi 22194234cb747dc067b159789a4b63f21b23d706922Dan Shi 222abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnettedef check_lab_status(build): 223abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette """Check if the lab status allows us to schedule for a build. 224266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 225abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette Checks if the lab is down, or if testing for the requested build 226abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette should be blocked. 227266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 228abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @param build: Name of the build to be scheduled for testing. 229266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 230abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette @raises TestLabException Raised if a request to test for the given 231abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette status and build should be blocked. 232266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 233266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette """ 234266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # Ensure we are trying to schedule on the actual lab. 23594234cb747dc067b159789a4b63f21b23d706922Dan Shi if not is_in_lab(): 236266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return 237266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 238266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # Download the lab status from its home on the web. 239266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette status_url = global_config.global_config.get_config_value( 240266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette 'CROS', 'lab_status_url') 241266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette json_status = _get_lab_status(status_url) 242266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette if json_status is None: 243266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette # We go ahead and say the lab is open if we can't get the status. 24404be2bd5e4666a5c253e9c30ab20555e04286032Ilja H. Friedel logging.warning('Could not get a status from %s', status_url) 245266da2a226391a7153df264d54dcb54635df7bfcJ. Richard Barnette return 246abbe09600d73d77af92157027a20661c035be01eJ. Richard Barnette _decode_lab_status(json_status, build) 247023afc65f9377db51ff6122977a9f529a32422d3beeps 248023afc65f9377db51ff6122977a9f529a32422d3beeps 249023afc65f9377db51ff6122977a9f529a32422d3beepsdef lock_host_with_labels(afe, lock_manager, labels): 250023afc65f9377db51ff6122977a9f529a32422d3beeps """Lookup and lock one host that matches the list of input labels. 251023afc65f9377db51ff6122977a9f529a32422d3beeps 252023afc65f9377db51ff6122977a9f529a32422d3beeps @param afe: An instance of the afe class, as defined in server.frontend. 253023afc65f9377db51ff6122977a9f529a32422d3beeps @param lock_manager: A lock manager capable of locking hosts, eg the 254023afc65f9377db51ff6122977a9f529a32422d3beeps one defined in server.cros.host_lock_manager. 255023afc65f9377db51ff6122977a9f529a32422d3beeps @param labels: A list of labels to look for on hosts. 256023afc65f9377db51ff6122977a9f529a32422d3beeps 257023afc65f9377db51ff6122977a9f529a32422d3beeps @return: The hostname of a host matching all labels, and locked through the 258023afc65f9377db51ff6122977a9f529a32422d3beeps lock_manager. The hostname will be as specified in the database the afe 259023afc65f9377db51ff6122977a9f529a32422d3beeps object is associated with, i.e if it exists in afe_hosts with a .cros 260023afc65f9377db51ff6122977a9f529a32422d3beeps suffix, the hostname returned will contain a .cros suffix. 261023afc65f9377db51ff6122977a9f529a32422d3beeps 262023afc65f9377db51ff6122977a9f529a32422d3beeps @raises: error.NoEligibleHostException: If no hosts matching the list of 263023afc65f9377db51ff6122977a9f529a32422d3beeps input labels are available. 264023afc65f9377db51ff6122977a9f529a32422d3beeps @raises: error.TestError: If unable to lock a host matching the labels. 265023afc65f9377db51ff6122977a9f529a32422d3beeps """ 266023afc65f9377db51ff6122977a9f529a32422d3beeps potential_hosts = afe.get_hosts(multiple_labels=labels) 267023afc65f9377db51ff6122977a9f529a32422d3beeps if not potential_hosts: 268023afc65f9377db51ff6122977a9f529a32422d3beeps raise error.NoEligibleHostException( 269023afc65f9377db51ff6122977a9f529a32422d3beeps 'No devices found with labels %s.' % labels) 270023afc65f9377db51ff6122977a9f529a32422d3beeps 271023afc65f9377db51ff6122977a9f529a32422d3beeps # This prevents errors where a fault might seem repeatable 272023afc65f9377db51ff6122977a9f529a32422d3beeps # because we lock, say, the same packet capturer for each test run. 273023afc65f9377db51ff6122977a9f529a32422d3beeps random.shuffle(potential_hosts) 274023afc65f9377db51ff6122977a9f529a32422d3beeps for host in potential_hosts: 275023afc65f9377db51ff6122977a9f529a32422d3beeps if lock_manager.lock([host.hostname]): 276023afc65f9377db51ff6122977a9f529a32422d3beeps logging.info('Locked device %s with labels %s.', 277023afc65f9377db51ff6122977a9f529a32422d3beeps host.hostname, labels) 278023afc65f9377db51ff6122977a9f529a32422d3beeps return host.hostname 279023afc65f9377db51ff6122977a9f529a32422d3beeps else: 280023afc65f9377db51ff6122977a9f529a32422d3beeps logging.info('Unable to lock device %s with labels %s.', 281023afc65f9377db51ff6122977a9f529a32422d3beeps host.hostname, labels) 282023afc65f9377db51ff6122977a9f529a32422d3beeps 283023afc65f9377db51ff6122977a9f529a32422d3beeps raise error.TestError('Could not lock a device with labels %s' % labels) 2847e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2857e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2867e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shidef get_test_views_from_tko(suite_job_id, tko): 2877e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """Get test name and result for given suite job ID. 2887e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2897e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param suite_job_id: ID of suite job. 2907e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @param tko: an instance of TKO as defined in server/frontend.py. 2917e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @return: A dictionary of test status keyed by test name, e.g., 2927e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'} 2937e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi @raise: Exception when there is no test view found. 2947e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 2957e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi """ 2967e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi views = tko.run('get_detailed_test_views', afe_job_id=suite_job_id) 2977e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi relevant_views = filter(job_status.view_is_relevant, views) 2987e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi if not relevant_views: 2997e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi raise Exception('Failed to retrieve job results.') 3007e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3017e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views = {} 3027e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi for view in relevant_views: 3037e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi test_views[view['test_name']] = view['status'] 3047e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi 3057e04fa8cf1f07512f52870cf4bffd9f8b0801088Dan Shi return test_views 306