dev_server.py revision ba31601061b723086f28b44eed2a9ee20ab57734
1adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Use of this source code is governed by a BSD-style license that can be 3adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# found in the LICENSE file. 4adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 5adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom distutils import version 6adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport cStringIO 7adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport HTMLParser 8adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport httplib 9adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport json 10adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport logging 11adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport multiprocessing 12adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport os 13adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport re 14adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport time 15adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport urllib2 16adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectimport urlparse 17adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 18adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.bin import utils as site_utils 19adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.common_lib import android_utils 20adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.common_lib import error 21adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.common_lib import global_config 22adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.common_lib import utils 23adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.common_lib.cros import retry 24adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectfrom autotest_lib.client.common_lib.cros.graphite import autotest_stats 25adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# TODO(cmasone): redo this class using requests module; http://crosbug.com/30107 26adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 27adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 28adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectCONFIG = global_config.global_config 29adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# This file is generated at build time and specifies, per suite and per test, 30adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# the DEPENDENCIES list specified in each control file. It's a dict of dicts: 31adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# {'bvt': {'/path/to/autotest/control/site_tests/test1/control': ['dep1']} 32adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# 'suite': {'/path/to/autotest/control/site_tests/test2/control': ['dep2']} 33adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# 'power': {'/path/to/autotest/control/site_tests/test1/control': ['dep1'], 34adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# '/path/to/autotest/control/site_tests/test3/control': ['dep3']} 35adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# } 36adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectDEPENDENCIES_FILE = 'test_suites/dependency_info' 37adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Number of seconds for caller to poll devserver's is_staged call to check if 38adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# artifacts are staged. 39adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project_ARTIFACT_STAGE_POLLING_INTERVAL = 5 40adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Artifacts that should be staged when client calls devserver RPC to stage an 41adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# image. 42adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project_ARTIFACTS_TO_BE_STAGED_FOR_IMAGE = 'full_payload,test_suites,stateful' 43adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Artifacts that should be staged when client calls devserver RPC to stage an 44adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# image with autotest artifact. 45adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project_ARTIFACTS_TO_BE_STAGED_FOR_IMAGE_WITH_AUTOTEST = ('full_payload,test_suites,' 46adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'control_files,stateful,' 47adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'autotest_packages') 48adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Artifacts that should be staged when client calls devserver RPC to stage an 49adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Android build. 50adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project_BRILLO_ARTIFACTS_TO_BE_STAGED_FOR_IMAGE = ('zip_images,vendor_partitions') 51adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectSKIP_DEVSERVER_HEALTH_CHECK = CONFIG.get_config_value( 52adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'CROS', 'skip_devserver_health_check', type=bool) 53adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Number of seconds for the call to get devserver load to time out. 54adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectTIMEOUT_GET_DEVSERVER_LOAD = 2.0 55adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 56adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Android artifact path in devserver 57adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectANDROID_BUILD_NAME_PATTERN = CONFIG.get_config_value( 58adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'CROS', 'android_build_name_pattern', type=str).replace('\\', '') 59adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 60adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# Return value from a devserver RPC indicating the call succeeded. 61adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectSUCCESS = 'Success' 62adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 63adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# The timeout minutes for a given devserver ssh call. 64adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectDEVSERVER_SSH_TIMEOUT_MINS = 1 65adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 66adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project# The timeout minutes for waiting a devserver staging. 67adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectDEVSERVER_IS_STAGING_RETRY_MIN = 100 68adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 69adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectPREFER_LOCAL_DEVSERVER = CONFIG.get_config_value( 70adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'CROS', 'prefer_local_devserver', type=bool, default=False) 71adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 72adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectENABLE_SSH_CONNECTION_FOR_DEVSERVER = CONFIG.get_config_value( 73adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'CROS', 'enable_ssh_connection_for_devserver', type=bool, 74adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project default=False) 75adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 76adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source ProjectDEFAULT_SUBNET_MASKBIT = 19 77adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 78adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectclass MarkupStripper(HTMLParser.HTMLParser): 79adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """HTML parser that strips HTML tags, coded characters like & 80adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 81adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project Works by, basically, not doing anything for any tags, and only recording 82adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project the content of text nodes in an internal data structure. 83adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """ 84adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project def __init__(self): 85adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project self.reset() 86adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project self.fed = [] 87adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 88adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 89adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project def handle_data(self, d): 90adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """Consume content of text nodes, store it away.""" 91adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project self.fed.append(d) 92adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 93adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 94adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project def get_data(self): 95adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """Concatenate and return all stored data.""" 96adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project return ''.join(self.fed) 97adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 98adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 99adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectdef _strip_http_message(message): 100adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """Strip the HTTP marker from the an HTTP message. 101adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 102adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project @param message: A string returned by an HTTP call. 103adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 104adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project @return: A string with HTTP marker being stripped. 105adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """ 106adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project strip = MarkupStripper() 107adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project try: 108adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project strip.feed(message.decode('utf_32')) 109adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project except UnicodeDecodeError: 110adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project strip.feed(message) 111adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project return strip.get_data() 112adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 113adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 114adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectdef _get_image_storage_server(): 115adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project return CONFIG.get_config_value('CROS', 'image_storage_server', type=str) 116adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 117adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 118adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectdef _get_canary_channel_server(): 119adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """ 120adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project Get the url of the canary-channel server, 121adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project eg: gsutil://chromeos-releases/canary-channel/<board>/<release> 122adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 123adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project @return: The url to the canary channel server. 124adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """ 125adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project return CONFIG.get_config_value('CROS', 'canary_channel_server', type=str) 126adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 127adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 128adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectdef _get_storage_server_for_artifacts(artifacts=None): 129adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """Gets the appropriate storage server for the given artifacts. 130adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 131adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project @param artifacts: A list of artifacts we need to stage. 132adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project @return: The address of the storage server that has these artifacts. 133adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project The default image storage server if no artifacts are specified. 134adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project """ 135adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project factory_artifact = global_config.global_config.get_config_value( 136adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 'CROS', 'factory_artifact', type=str, default='') 137adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project if artifacts and factory_artifact and factory_artifact in artifacts: 138adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project return _get_canary_channel_server() 139adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project return _get_image_storage_server() 140adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 141adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Project 142adc854b798c1cfe3bfd4c27d68d5cee38ca617daThe Android Open Source Projectdef _get_dev_server_list(): 143 return CONFIG.get_config_value('CROS', 'dev_server', type=list, default=[]) 144 145 146def _get_crash_server_list(): 147 return CONFIG.get_config_value('CROS', 'crash_server', type=list, 148 default=[]) 149 150 151def remote_devserver_call(timeout_min=DEVSERVER_IS_STAGING_RETRY_MIN): 152 """A decorator to use with remote devserver calls. 153 154 This decorator converts urllib2.HTTPErrors into DevServerExceptions 155 with any embedded error info converted into plain text. The method 156 retries on urllib2.URLError or error.CmdError to avoid devserver flakiness. 157 """ 158 #pylint: disable=C0111 159 def inner_decorator(method): 160 161 @retry.retry((urllib2.URLError, error.CmdError), 162 timeout_min=timeout_min, 163 exception_to_raise=DevServerException) 164 def wrapper(*args, **kwargs): 165 """This wrapper actually catches the HTTPError.""" 166 try: 167 return method(*args, **kwargs) 168 except urllib2.HTTPError as e: 169 error_markup = e.read() 170 raise DevServerException(_strip_http_message(error_markup)) 171 172 return wrapper 173 174 return inner_decorator 175 176 177class DevServerException(Exception): 178 """Raised when the dev server returns a non-200 HTTP response.""" 179 pass 180 181 182class DevServer(object): 183 """Base class for all DevServer-like server stubs. 184 185 This is the base class for interacting with all Dev Server-like servers. 186 A caller should instantiate a sub-class of DevServer with: 187 188 host = SubClassServer.resolve(build) 189 server = SubClassServer(host) 190 """ 191 _MIN_FREE_DISK_SPACE_GB = 20 192 _MAX_APACHE_CLIENT_COUNT = 75 193 # Threshold for the CPU load percentage for a devserver to be selected. 194 MAX_CPU_LOAD = 80.0 195 # Threshold for the network IO, set to 80MB/s 196 MAX_NETWORK_IO = 1024 * 1024 * 80 197 DISK_IO = 'disk_total_bytes_per_second' 198 NETWORK_IO = 'network_total_bytes_per_second' 199 CPU_LOAD = 'cpu_percent' 200 FREE_DISK = 'free_disk' 201 STAGING_THREAD_COUNT = 'staging_thread_count' 202 APACHE_CLIENT_COUNT = 'apache_client_count' 203 204 205 def __init__(self, devserver): 206 self._devserver = devserver 207 208 209 def url(self): 210 """Returns the url for this devserver.""" 211 return self._devserver 212 213 214 @staticmethod 215 def get_server_name(url): 216 """Strip the http:// prefix and port from a url. 217 218 @param url: A url of a server. 219 220 @return the server name without http:// prefix and port. 221 222 """ 223 return urlparse.urlparse(url).hostname 224 225 226 @staticmethod 227 def get_server_url(url): 228 """Get the devserver url from a repo url, which includes build info. 229 230 @param url: A job repo url. 231 232 @return A devserver url, e.g., http://127.0.0.10:8080 233 """ 234 res = urlparse.urlparse(url) 235 if res.netloc: 236 return res.scheme + '://' + res.netloc 237 238 239 @classmethod 240 def get_devserver_load_wrapper(cls, devserver, timeout_sec, output): 241 """A wrapper function to call get_devserver_load in parallel. 242 243 @param devserver: url of the devserver. 244 @param timeout_sec: Number of seconds before time out the devserver 245 call. 246 @param output: An output queue to save results to. 247 """ 248 load = cls.get_devserver_load(devserver, timeout_min=timeout_sec/60.0) 249 if load: 250 load['devserver'] = devserver 251 output.put(load) 252 253 254 @classmethod 255 def get_devserver_load(cls, devserver, timeout_min=0.1): 256 """Returns True if the |devserver| is healthy to stage build. 257 258 @param devserver: url of the devserver. 259 @param timeout_min: How long to wait in minutes before deciding the 260 the devserver is not up (float). 261 262 @return: A dictionary of the devserver's load. 263 264 """ 265 server_name = DevServer.get_server_name(devserver) 266 # statsd treats |.| as path separator. 267 server_name = server_name.replace('.', '_') 268 call = DevServer._build_call(devserver, 'check_health') 269 270 @remote_devserver_call(timeout_min=timeout_min) 271 def make_call(): 272 """Inner method that makes the call.""" 273 return cls.run_call(call, timeout=timeout_min*60) 274 try: 275 result_dict = json.load(cStringIO.StringIO(make_call())) 276 for key, val in result_dict.iteritems(): 277 try: 278 autotest_stats.Gauge(server_name).send(key, float(val)) 279 except ValueError: 280 # Ignore all non-numerical health data. 281 pass 282 283 return result_dict 284 except Exception as e: 285 logging.error('Devserver call failed: "%s", timeout: %s seconds,' 286 ' Error: %s', call, timeout_min * 60, e) 287 288 289 @staticmethod 290 def is_free_disk_ok(load): 291 """Check if a devserver has enough free disk. 292 293 @param load: A dict of the load of the devserver. 294 295 @return: True if the devserver has enough free disk or disk check is 296 skipped in global config. 297 298 """ 299 if SKIP_DEVSERVER_HEALTH_CHECK: 300 logging.debug('devserver health check is skipped.') 301 elif load[DevServer.FREE_DISK] < DevServer._MIN_FREE_DISK_SPACE_GB: 302 return False 303 304 return True 305 306 307 @staticmethod 308 def is_apache_client_count_ok(load): 309 """Check if a devserver has enough Apache connections available. 310 311 Apache server by default has maximum of 150 concurrent connections. If 312 a devserver has too many live connections, it likely indicates the 313 server is busy handling many long running download requests, e.g., 314 downloading stateful partitions. It is better not to add more requests 315 to it. 316 317 @param load: A dict of the load of the devserver. 318 319 @return: True if the devserver has enough Apache connections available, 320 or disk check is skipped in global config. 321 322 """ 323 if SKIP_DEVSERVER_HEALTH_CHECK: 324 logging.debug('devserver health check is skipped.') 325 elif DevServer.APACHE_CLIENT_COUNT not in load: 326 logging.debug('Apache client count is not collected from devserver.') 327 elif (load[DevServer.APACHE_CLIENT_COUNT] > 328 DevServer._MAX_APACHE_CLIENT_COUNT): 329 return False 330 331 return True 332 333 334 @classmethod 335 def devserver_healthy(cls, devserver, timeout_min=0.1): 336 """Returns True if the |devserver| is healthy to stage build. 337 338 @param devserver: url of the devserver. 339 @param timeout_min: How long to wait in minutes before deciding the 340 the devserver is not up (float). 341 342 @return: True if devserver is healthy. Return False otherwise. 343 344 """ 345 server_name = DevServer.get_server_name(devserver) 346 # statsd treats |.| as path separator. 347 server_name = server_name.replace('.', '_') 348 load = cls.get_devserver_load(devserver, timeout_min=timeout_min) 349 if not load: 350 # Failed to get the load of devserver. 351 autotest_stats.Counter(server_name + 352 '.devserver_not_healthy').increment() 353 return False 354 355 apache_ok = DevServer.is_apache_client_count_ok(load) 356 if not apache_ok: 357 logging.error('Devserver check_health failed. Live Apache client ' 358 'count is too high: %d.', 359 load[DevServer.APACHE_CLIENT_COUNT]) 360 autotest_stats.Counter(server_name + 361 '.devserver_not_healthy').increment() 362 return False 363 364 disk_ok = DevServer.is_free_disk_ok(load) 365 if not disk_ok: 366 logging.error('Devserver check_health failed. Free disk space is ' 367 'low. Only %dGB is available.', 368 load[DevServer.FREE_DISK]) 369 counter = '.devserver_healthy' if disk_ok else '.devserver_not_healthy' 370 # This counter indicates the load of a devserver. By comparing the 371 # value of this counter for all devservers, we can evaluate the 372 # load balancing across all devservers. 373 autotest_stats.Counter(server_name + counter).increment() 374 return disk_ok 375 376 377 @staticmethod 378 def _build_call(host, method, **kwargs): 379 """Build a URL to |host| that calls |method|, passing |kwargs|. 380 381 Builds a URL that calls |method| on the dev server defined by |host|, 382 passing a set of key/value pairs built from the dict |kwargs|. 383 384 @param host: a string that is the host basename e.g. http://server:90. 385 @param method: the dev server method to call. 386 @param kwargs: a dict mapping arg names to arg values. 387 @return the URL string. 388 """ 389 argstr = '&'.join(map(lambda x: "%s=%s" % x, kwargs.iteritems())) 390 return "%(host)s/%(method)s?%(argstr)s" % dict( 391 host=host, method=method, argstr=argstr) 392 393 394 def build_call(self, method, **kwargs): 395 """Builds a devserver RPC string that is used by 'run_call()'. 396 397 @param method: remote devserver method to call. 398 """ 399 return self._build_call(self._devserver, method, **kwargs) 400 401 402 @classmethod 403 def build_all_calls(cls, method, **kwargs): 404 """Builds a list of URLs that makes RPC calls on all devservers. 405 406 Build a URL that calls |method| on the dev server, passing a set 407 of key/value pairs built from the dict |kwargs|. 408 409 @param method: the dev server method to call. 410 @param kwargs: a dict mapping arg names to arg values 411 412 @return the URL string 413 """ 414 calls = [] 415 # Note we use cls.servers as servers is class specific. 416 for server in cls.servers(): 417 if cls.devserver_healthy(server): 418 calls.append(cls._build_call(server, method, **kwargs)) 419 420 return calls 421 422 423 @classmethod 424 def run_call(cls, call, readline=False, timeout=None): 425 """Invoke a given devserver call using urllib.open. 426 427 Open the URL with HTTP, and return the text of the response. Exceptions 428 may be raised as for urllib2.urlopen(). 429 430 @param call: a url string that calls a method to a devserver. 431 @param readline: whether read http response line by line. 432 @param timeout: The timeout seconds for this urlopen call. 433 434 @return the results of this call. 435 """ 436 if timeout is not None: 437 return utils.urlopen_socket_timeout( 438 call, timeout=timeout).read() 439 elif readline: 440 response = urllib2.urlopen(call) 441 return [line.rstrip() for line in response] 442 else: 443 return urllib2.urlopen(call).read() 444 445 446 @staticmethod 447 def servers(): 448 """Returns a list of servers that can serve as this type of server.""" 449 raise NotImplementedError() 450 451 452 @classmethod 453 def get_devservers_in_same_subnet(cls, ip, mask_bits=DEFAULT_SUBNET_MASKBIT, 454 unrestricted_only=False): 455 """Get the devservers in the same subnet of the given ip. 456 457 @param ip: The IP address of a dut to look for devserver. 458 @param mask_bits: Number of mask bits. Default is 19. 459 @param unrestricted_only: Set to True to select from devserver in 460 unrestricted subnet only. Default is False. 461 462 @return: A list of devservers in the same subnet of the given ip. 463 464 """ 465 # server from cls.servers() is a URL, e.g., http://10.1.1.10:8082, so 466 # we need a dict to return the full devserver path once the IPs are 467 # filtered in get_servers_in_same_subnet. 468 server_names = {} 469 all_devservers = [] 470 devservers = (cls.get_unrestricted_devservers() if unrestricted_only 471 else cls.servers()) 472 for server in devservers: 473 server_name = cls.get_server_name(server) 474 server_names[server_name] = server 475 all_devservers.append(server_name) 476 devservers = utils.get_servers_in_same_subnet(ip, mask_bits, 477 all_devservers) 478 return [server_names[s] for s in devservers] 479 480 481 @classmethod 482 def get_unrestricted_devservers( 483 cls, restricted_subnets=utils.RESTRICTED_SUBNETS): 484 """Get the devservers not in any restricted subnet specified in 485 restricted_subnets. 486 487 @param restricted_subnets: A list of restriected subnets. 488 489 @return: A list of devservers not in any restricted subnet. 490 491 """ 492 if not restricted_subnets: 493 return cls.servers() 494 495 devservers = [] 496 for server in cls.servers(): 497 server_name = cls.get_server_name(server) 498 if not utils.get_restricted_subnet(server_name, restricted_subnets): 499 devservers.append(server) 500 return devservers 501 502 503 @classmethod 504 def get_healthy_devserver(cls, build, devservers): 505 """"Get a healthy devserver instance from the list of devservers. 506 507 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514). 508 509 @return: A DevServer object of a healthy devserver. Return None if no 510 healthy devserver is found. 511 512 """ 513 while devservers: 514 hash_index = hash(build) % len(devservers) 515 devserver = devservers.pop(hash_index) 516 if cls.devserver_healthy(devserver): 517 return cls(devserver) 518 519 520 @classmethod 521 def get_available_devservers(cls, hostname=None, 522 prefer_local_devserver=PREFER_LOCAL_DEVSERVER, 523 restricted_subnets=utils.RESTRICTED_SUBNETS): 524 """Get devservers in the same subnet of the given hostname. 525 526 @param hostname: Hostname of a DUT to choose devserver for. 527 528 @return: A tuple of (devservers, can_retry), devservers is a list of 529 devservers that's available for the given hostname. can_retry 530 is a flag that indicate if caller can retry the selection of 531 devserver if no devserver in the returned devservers can be 532 used. For example, if hostname is in a restricted subnet, 533 can_retry will be False. 534 """ 535 host_ip = None 536 if hostname: 537 host_ip = site_utils.get_ip_address(hostname) 538 if not host_ip: 539 logging.error('Failed to get IP address of %s. Will pick a ' 540 'devserver without subnet constraint.', hostname) 541 542 if not host_ip: 543 return cls.get_unrestricted_devservers(restricted_subnets), False 544 545 # Go through all restricted subnet settings and check if the DUT is 546 # inside a restricted subnet. If so, only return the devservers in the 547 # restricted subnet and doesn't allow retry. 548 if host_ip and restricted_subnets: 549 for subnet_ip, mask_bits in restricted_subnets: 550 if utils.is_in_same_subnet(host_ip, subnet_ip, mask_bits): 551 logging.debug('The host %s (%s) is in a restricted subnet. ' 552 'Try to locate a devserver inside subnet ' 553 '%s:%d.', hostname, host_ip, subnet_ip, 554 mask_bits) 555 devservers = cls.get_devservers_in_same_subnet( 556 subnet_ip, mask_bits) 557 return devservers, False 558 559 # If prefer_local_devserver is set to True and the host is not in 560 # restricted subnet, pick a devserver in the same subnet if possible. 561 # Set can_retry to True so it can pick a different devserver if all 562 # devservers in the same subnet are down. 563 if prefer_local_devserver: 564 return (cls.get_devservers_in_same_subnet( 565 host_ip, DEFAULT_SUBNET_MASKBIT, True), True) 566 567 return cls.get_unrestricted_devservers(restricted_subnets), False 568 569 570 @classmethod 571 def resolve(cls, build, hostname=None): 572 """"Resolves a build to a devserver instance. 573 574 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514). 575 @param hostname: The hostname of dut that requests a devserver. It's 576 used to make sure a devserver in the same subnet is 577 preferred. 578 579 @raise DevServerException: If no devserver is available. 580 """ 581 devservers, can_retry = cls.get_available_devservers(hostname) 582 583 devserver = cls.get_healthy_devserver(build, devservers) 584 585 if not devserver and can_retry: 586 # Find available devservers without dut location constrain. 587 devservers, _ = cls.get_available_devservers() 588 devserver = cls.get_healthy_devserver(build, devservers) 589 if devserver: 590 return devserver 591 else: 592 error_msg = 'All devservers are currently down: %s' % devservers 593 logging.error(error_msg) 594 raise DevServerException(error_msg) 595 596 597 @classmethod 598 def random(cls): 599 """Return a random devserver that's available. 600 601 Devserver election in `resolve` method is based on a hash of the 602 build that a caller wants to stage. The purpose is that different 603 callers requesting for the same build can get the same devserver, 604 while the lab is able to distribute different builds across all 605 devservers. That helps to reduce the duplication of builds across 606 all devservers. 607 This function returns a random devserver, by passing a random 608 pseudo build name to `resolve `method. 609 """ 610 return cls.resolve(build=str(time.time())) 611 612 613class CrashServer(DevServer): 614 """Class of DevServer that symbolicates crash dumps.""" 615 616 @staticmethod 617 def servers(): 618 return _get_crash_server_list() 619 620 621 @remote_devserver_call() 622 def symbolicate_dump(self, minidump_path, build): 623 """Ask the devserver to symbolicate the dump at minidump_path. 624 625 Stage the debug symbols for |build| and, if that works, ask the 626 devserver to symbolicate the dump at |minidump_path|. 627 628 @param minidump_path: the on-disk path of the minidump. 629 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 630 whose debug symbols are needed for symbolication. 631 @return The contents of the stack trace 632 @raise DevServerException upon any return code that's not HTTP OK. 633 """ 634 try: 635 import requests 636 except ImportError: 637 logging.warning("Can't 'import requests' to connect to dev server.") 638 return '' 639 server_name = self.get_server_name(self.url()) 640 server_name = server_name.replace('.', '_') 641 stats_key = 'CrashServer.%s.symbolicate_dump' % server_name 642 autotest_stats.Counter(stats_key).increment() 643 timer = autotest_stats.Timer(stats_key) 644 timer.start() 645 # Symbolicate minidump. 646 call = self.build_call('symbolicate_dump', 647 archive_url=_get_image_storage_server() + build) 648 request = requests.post( 649 call, files={'minidump': open(minidump_path, 'rb')}) 650 if request.status_code == requests.codes.OK: 651 timer.stop() 652 return request.text 653 654 error_fd = cStringIO.StringIO(request.text) 655 raise urllib2.HTTPError( 656 call, request.status_code, request.text, request.headers, 657 error_fd) 658 659 660 @classmethod 661 def get_available_devservers(cls, hostname): 662 """Get all available crash servers. 663 664 Crash server election doesn't need to count the location of hostname. 665 666 @param hostname: Hostname of a DUT to choose devserver for. 667 668 @return: A tuple of (all crash servers, False). can_retry is set to 669 False, as all crash servers are returned. There is no point to 670 retry. 671 """ 672 return cls.servers(), False 673 674 675class ImageServerBase(DevServer): 676 """Base class for devservers used to stage builds. 677 678 CrOS and Android builds are staged in different ways as they have different 679 sets of artifacts. This base class abstracts the shared functions between 680 the two types of ImageServer. 681 """ 682 683 @classmethod 684 def servers(cls): 685 """Returns a list of servers that can serve as a desired type of 686 devserver. 687 """ 688 return _get_dev_server_list() 689 690 691 def _get_image_url(self, image): 692 """Returns the url of the directory for this image on the devserver. 693 694 @param image: the image that was fetched. 695 """ 696 image = self.translate(image) 697 url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', 698 type=str) 699 return (url_pattern % (self.url(), image)).replace('update', 'static') 700 701 702 @staticmethod 703 def create_stats_str(subname, server_name, artifacts): 704 """Create a graphite name given the staged items. 705 706 The resulting name will look like 707 'dev_server.subname.DEVSERVER_URL.artifact1_artifact2' 708 The name can be used to create a stats object like 709 stats.Timer, stats.Counter, etc. 710 711 @param subname: A name for the graphite sub path. 712 @param server_name: name of the devserver, e.g 172.22.33.44. 713 @param artifacts: A list of artifacts. 714 715 @return A name described above. 716 717 """ 718 staged_items = sorted(artifacts) if artifacts else [] 719 staged_items_str = '_'.join(staged_items).replace( 720 '.', '_') if staged_items else None 721 server_name = server_name.replace('.', '_') 722 stats_str = 'dev_server.%s.%s' % (subname, server_name) 723 if staged_items_str: 724 stats_str += '.%s' % staged_items_str 725 return stats_str 726 727 728 @staticmethod 729 def create_metadata(server_name, image, artifacts=None, files=None): 730 """Create a metadata dictionary given the staged items. 731 732 The metadata can be send to metadata db along with stats. 733 734 @param server_name: name of the devserver, e.g 172.22.33.44. 735 @param image: The name of the image. 736 @param artifacts: A list of artifacts. 737 @param files: A list of files. 738 739 @return A metadata dictionary. 740 741 """ 742 metadata = {'devserver': server_name, 743 'image': image, 744 '_type': 'devserver'} 745 if artifacts: 746 metadata['artifacts'] = ' '.join(artifacts) 747 if files: 748 metadata['files'] = ' '.join(files) 749 return metadata 750 751 752 @classmethod 753 def run_ssh_call(cls, call, readline=False, timeout=None): 754 """Construct an ssh-based rpc call, and execute it. 755 756 @param call: a url string that calls a method to a devserver. 757 @param readline: whether read http response line by line. 758 @param timeout: The timeout seconds for ssh call. 759 760 @return the results of this call. 761 """ 762 hostname = urlparse.urlparse(call).hostname 763 ssh_call = 'ssh %s \'curl "%s"\'' % (hostname, utils.sh_escape(call)) 764 timeout_seconds = timeout if timeout else DEVSERVER_SSH_TIMEOUT_MINS*60 765 try: 766 result = utils.run(ssh_call, timeout=timeout_seconds) 767 except error.CmdError as e: 768 logging.debug('Error occurred with exit_code %d when executing the ' 769 'ssh call: %s.', e.result_obj.exit_status, 770 e.result_obj.stderr) 771 stats_str = 'dev_server.%s.%s' % (hostname.replace('.', '_'), 772 'ssh_dev_server_failure') 773 autotest_stats.Counter(stats_str).increment() 774 raise 775 response = result.stdout 776 777 # If the curl command's returned HTTP response contains certain 778 # exception string, raise the DevServerException of the response. 779 if 'DownloaderException' in response: 780 raise DevServerException(_strip_http_message(response)) 781 782 if readline: 783 # Remove line terminators and trailing whitespace 784 response = response.splitlines() 785 return [line.rstrip() for line in response] 786 787 return response 788 789 790 @classmethod 791 def run_call(cls, call, readline=False, timeout=None): 792 """Invoke a given devserver call using urllib.open or ssh. 793 794 Open the URL with HTTP or SSH-based HTTP, and return the text of the 795 response. Exceptions may be raised as for urllib2.urlopen() or 796 utils.run(). 797 798 @param call: a url string that calls a method to a devserver. 799 @param readline: whether read http response line by line. 800 @param timeout: The timeout seconds for urlopen call or ssh call. 801 802 @return the results of this call. 803 """ 804 if not ENABLE_SSH_CONNECTION_FOR_DEVSERVER: 805 return super(ImageServerBase, cls).run_call( 806 call, readline=readline, timeout=timeout) 807 else: 808 return cls.run_ssh_call( 809 call, readline=readline, timeout=timeout) 810 811 812 def _poll_is_staged(self, **kwargs): 813 """Polling devserver.is_staged until all artifacts are staged. 814 815 @param kwargs: keyword arguments to make is_staged devserver call. 816 817 @return: True if all artifacts are staged in devserver. 818 """ 819 call = self.build_call('is_staged', **kwargs) 820 821 def all_staged(): 822 """Call devserver.is_staged rpc to check if all files are staged. 823 824 @return: True if all artifacts are staged in devserver. False 825 otherwise. 826 @rasies DevServerException, the exception is a wrapper of all 827 exceptions that were raised when devserver tried to download 828 the artifacts. devserver raises an HTTPError or a CmdError 829 when an exception was raised in the code. Such exception 830 should be re-raised here to stop the caller from waiting. 831 If the call to devserver failed for connection issue, a 832 URLError exception is raised, and caller should retry the 833 call to avoid such network flakiness. 834 835 """ 836 try: 837 return self.run_call(call) == 'True' 838 except urllib2.HTTPError as e: 839 error_markup = e.read() 840 raise DevServerException(_strip_http_message(error_markup)) 841 except urllib2.URLError: 842 # Could be connection issue, retry it. 843 # For example: <urlopen error [Errno 111] Connection refused> 844 return False 845 except error.CmdError: 846 # Retry if SSH failed to connect to the devserver. 847 logging.warning('CmdError: Retrying SSH connection to check is_stage.') 848 return False 849 850 site_utils.poll_for_condition( 851 all_staged, 852 exception=site_utils.TimeoutError(), 853 timeout=DEVSERVER_IS_STAGING_RETRY_MIN * 60, 854 sleep_interval=_ARTIFACT_STAGE_POLLING_INTERVAL) 855 856 return True 857 858 859 def _call_and_wait(self, call_name, error_message, 860 expected_response=SUCCESS, **kwargs): 861 """Helper method to make a urlopen call, and wait for artifacts staged. 862 863 @param call_name: name of devserver rpc call. 864 @param error_message: Error message to be thrown if response does not 865 match expected_response. 866 @param expected_response: Expected response from rpc, default to 867 |Success|. If it's set to None, do not compare 868 the actual response. Any response is consider 869 to be good. 870 @param kwargs: keyword arguments to make is_staged devserver call. 871 872 @return: The response from rpc. 873 @raise DevServerException upon any return code that's expected_response. 874 875 """ 876 call = self.build_call(call_name, async=True, **kwargs) 877 try: 878 response = self.run_call(call) 879 except httplib.BadStatusLine as e: 880 logging.error(e) 881 raise DevServerException('Received Bad Status line, Devserver %s ' 882 'might have gone down while handling ' 883 'the call: %s' % (self.url(), call)) 884 885 if expected_response and not response == expected_response: 886 raise DevServerException(error_message) 887 888 # `os_type` is needed in build a devserver call, but not needed for 889 # wait_for_artifacts_staged, since that method is implemented by 890 # each ImageServerBase child class. 891 if 'os_type' in kwargs: 892 del kwargs['os_type'] 893 self.wait_for_artifacts_staged(**kwargs) 894 return response 895 896 897 def _stage_artifacts(self, build, artifacts, files, archive_url, **kwargs): 898 """Tell the devserver to download and stage |artifacts| from |image| 899 specified by kwargs. 900 901 This is the main call point for staging any specific artifacts for a 902 given build. To see the list of artifacts one can stage see: 903 904 ~src/platfrom/dev/artifact_info.py. 905 906 This is maintained along with the actual devserver code. 907 908 @param artifacts: A list of artifacts. 909 @param files: A list of files to stage. 910 @param archive_url: Optional parameter that has the archive_url to stage 911 this artifact from. Default is specified in autotest config + 912 image. 913 @param kwargs: keyword arguments that specify the build information, to 914 make stage devserver call. 915 916 @raise DevServerException upon any return code that's not HTTP OK. 917 """ 918 if not archive_url: 919 archive_url = _get_storage_server_for_artifacts(artifacts) + build 920 921 artifacts_arg = ','.join(artifacts) if artifacts else '' 922 files_arg = ','.join(files) if files else '' 923 error_message = ("staging %s for %s failed;" 924 "HTTP OK not accompanied by 'Success'." % 925 ('artifacts=%s files=%s ' % (artifacts_arg, files_arg), 926 build)) 927 928 staging_info = ('build=%s, artifacts=%s, files=%s, archive_url=%s' % 929 (build, artifacts, files, archive_url)) 930 logging.info('Staging artifacts on devserver %s: %s', 931 self.url(), staging_info) 932 if artifacts: 933 server_name = self.get_server_name(self.url()) 934 timer_key = self.create_stats_str( 935 'stage_artifacts', server_name, artifacts) 936 counter_key = self.create_stats_str( 937 'stage_artifacts_count', server_name, artifacts) 938 metadata = self.create_metadata(server_name, build, artifacts, 939 files) 940 autotest_stats.Counter(counter_key, metadata=metadata).increment() 941 timer = autotest_stats.Timer(timer_key, metadata=metadata) 942 timer.start() 943 try: 944 arguments = {'archive_url': archive_url, 945 'artifacts': artifacts_arg, 946 'files': files_arg} 947 if kwargs: 948 arguments.update(kwargs) 949 self.call_and_wait(call_name='stage',error_message=error_message, 950 **arguments) 951 if artifacts: 952 timer.stop() 953 logging.info('Finished staging artifacts: %s', staging_info) 954 except (site_utils.TimeoutError, error.TimeoutException): 955 logging.error('stage_artifacts timed out: %s', staging_info) 956 if artifacts: 957 timeout_key = self.create_stats_str( 958 'stage_artifacts_timeout', server_name, artifacts) 959 autotest_stats.Counter(timeout_key, 960 metadata=metadata).increment() 961 raise DevServerException( 962 'stage_artifacts timed out: %s' % staging_info) 963 964 965 def call_and_wait(self, *args, **kwargs): 966 """Helper method to make a urlopen call, and wait for artifacts staged. 967 968 This method needs to be overridden in the subclass to implement the 969 logic to call _call_and_wait. 970 """ 971 raise NotImplementedError 972 973 974 def _trigger_download(self, build, artifacts, files, synchronous=True, 975 **kwargs_build_info): 976 """Tell the devserver to download and stage image specified in 977 kwargs_build_info. 978 979 Tells the devserver to fetch |image| from the image storage server 980 named by _get_image_storage_server(). 981 982 If |synchronous| is True, waits for the entire download to finish 983 staging before returning. Otherwise only the artifacts necessary 984 to start installing images onto DUT's will be staged before returning. 985 A caller can then call finish_download to guarantee the rest of the 986 artifacts have finished staging. 987 988 @param synchronous: if True, waits until all components of the image are 989 staged before returning. 990 @param kwargs_build_info: Dictionary of build information. 991 For CrOS, it is None as build is the CrOS image name. 992 For Android, it is {'target': target, 993 'build_id': build_id, 994 'branch': branch} 995 996 @raise DevServerException upon any return code that's not HTTP OK. 997 998 """ 999 if kwargs_build_info: 1000 archive_url = None 1001 else: 1002 archive_url = _get_image_storage_server() + build 1003 error_message = ("trigger_download for %s failed;" 1004 "HTTP OK not accompanied by 'Success'." % build) 1005 kwargs = {'archive_url': archive_url, 1006 'artifacts': artifacts, 1007 'files': files, 1008 'error_message': error_message} 1009 if kwargs_build_info: 1010 kwargs.update(kwargs_build_info) 1011 1012 logging.info('trigger_download starts for %s', build) 1013 server_name = self.get_server_name(self.url()) 1014 artifacts_list = artifacts.split(',') 1015 counter_key = self.create_stats_str( 1016 'trigger_download_count', server_name, artifacts_list) 1017 metadata = self.create_metadata(server_name, build, artifacts_list) 1018 autotest_stats.Counter(counter_key, metadata=metadata).increment() 1019 try: 1020 response = self.call_and_wait(call_name='stage', **kwargs) 1021 logging.info('trigger_download finishes for %s', build) 1022 except (site_utils.TimeoutError, error.TimeoutException): 1023 logging.error('trigger_download timed out for %s.', build) 1024 timeout_key = self.create_stats_str( 1025 'trigger_download_timeout', server_name, artifacts_list) 1026 autotest_stats.Counter(timeout_key, metadata=metadata).increment() 1027 raise DevServerException( 1028 'trigger_download timed out for %s.' % build) 1029 was_successful = response == SUCCESS 1030 if was_successful and synchronous: 1031 self._finish_download(build, artifacts, files, **kwargs_build_info) 1032 1033 1034 def _finish_download(self, build, artifacts, files, **kwargs_build_info): 1035 """Tell the devserver to finish staging image specified in 1036 kwargs_build_info. 1037 1038 If trigger_download is called with synchronous=False, it will return 1039 before all artifacts have been staged. This method contacts the 1040 devserver and blocks until all staging is completed and should be 1041 called after a call to trigger_download. 1042 1043 @param kwargs_build_info: Dictionary of build information. 1044 For CrOS, it is None as build is the CrOS image name. 1045 For Android, it is {'target': target, 1046 'build_id': build_id, 1047 'branch': branch} 1048 1049 @raise DevServerException upon any return code that's not HTTP OK. 1050 """ 1051 archive_url = _get_image_storage_server() + build 1052 error_message = ("finish_download for %s failed;" 1053 "HTTP OK not accompanied by 'Success'." % build) 1054 kwargs = {'archive_url': archive_url, 1055 'artifacts': artifacts, 1056 'files': files, 1057 'error_message': error_message} 1058 if kwargs_build_info: 1059 kwargs.update(kwargs_build_info) 1060 try: 1061 self.call_and_wait(call_name='stage', **kwargs) 1062 except (site_utils.TimeoutError, error.TimeoutException): 1063 logging.error('finish_download timed out for %s', build) 1064 server_name = self.get_server_name(self.url()) 1065 artifacts_list = artifacts.split(',') 1066 timeout_key = self.create_stats_str( 1067 'finish_download_timeout', server_name, artifacts_list) 1068 metadata = self.create_metadata(server_name, build, artifacts_list) 1069 autotest_stats.Counter(timeout_key, metadata=metadata).increment() 1070 raise DevServerException( 1071 'finish_download timed out for %s.' % build) 1072 1073 1074 @remote_devserver_call() 1075 def locate_file(self, file_name, artifacts, build, build_info): 1076 """Locate a file with the given file_name on devserver. 1077 1078 This method calls devserver RPC `locate_file` to look up a file with 1079 the given file name inside specified build artifacts. 1080 1081 @param file_name: Name of the file to look for a file. 1082 @param artifacts: A list of artifact names to search for the file. 1083 @param build: Name of the build. For Android, it's None as build_info 1084 should be used. 1085 @param build_info: Dictionary of build information. 1086 For CrOS, it is None as build is the CrOS image name. 1087 For Android, it is {'target': target, 1088 'build_id': build_id, 1089 'branch': branch} 1090 1091 @return: A devserver url to the file. 1092 @raise DevServerException upon any return code that's not HTTP OK. 1093 """ 1094 if not build and not build_info: 1095 raise DevServerException('You must specify build information to ' 1096 'look for file %s in artifacts %s.' % 1097 (file_name, artifacts)) 1098 kwargs = {'file_name': file_name, 1099 'artifacts': artifacts} 1100 if build_info: 1101 build_path = '%(branch)s/%(target)s/%(build_id)s' % build_info 1102 kwargs.update(build_info) 1103 # Devserver treats Android and Brillo build in the same way as they 1104 # are both retrieved from Launch Control and have similar build 1105 # artifacts. Therefore, os_type for devserver calls is `android` for 1106 # both Android and Brillo builds. 1107 kwargs['os_type'] = 'android' 1108 else: 1109 build_path = build 1110 kwargs['build'] = build 1111 call = self.build_call('locate_file', async=False, **kwargs) 1112 try: 1113 file_path = self.run_call(call) 1114 return os.path.join(self.url(), 'static', build_path, file_path) 1115 except httplib.BadStatusLine as e: 1116 logging.error(e) 1117 raise DevServerException('Received Bad Status line, Devserver %s ' 1118 'might have gone down while handling ' 1119 'the call: %s' % (self.url(), call)) 1120 1121 1122 @remote_devserver_call() 1123 def list_control_files(self, build, suite_name=''): 1124 """Ask the devserver to list all control files for |build|. 1125 1126 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 1127 whose control files the caller wants listed. 1128 @param suite_name: The name of the suite for which we require control 1129 files. 1130 @return None on failure, or a list of control file paths 1131 (e.g. server/site_tests/autoupdate/control) 1132 @raise DevServerException upon any return code that's not HTTP OK. 1133 """ 1134 build = self.translate(build) 1135 call = self.build_call('controlfiles', build=build, 1136 suite_name=suite_name) 1137 return self.run_call(call, readline=True) 1138 1139 1140 @remote_devserver_call() 1141 def get_control_file(self, build, control_path): 1142 """Ask the devserver for the contents of a control file. 1143 1144 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 1145 whose control file the caller wants to fetch. 1146 @param control_path: The file to fetch 1147 (e.g. server/site_tests/autoupdate/control) 1148 @return The contents of the desired file. 1149 @raise DevServerException upon any return code that's not HTTP OK. 1150 """ 1151 build = self.translate(build) 1152 call = self.build_call('controlfiles', build=build, 1153 control_path=control_path) 1154 return self.run_call(call) 1155 1156 1157 @remote_devserver_call() 1158 def list_suite_controls(self, build, suite_name=''): 1159 """Ask the devserver to list contents of all control files for |build|. 1160 1161 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 1162 whose control files' contents the caller wants returned. 1163 @param suite_name: The name of the suite for which we require control 1164 files. 1165 @return None on failure, or a dict of contents of all control files 1166 (e.g. {'path1': "#Copyright controls ***", ..., 1167 pathX': "#Copyright controls ***"} 1168 @raise DevServerException upon any return code that's not HTTP OK. 1169 """ 1170 build = self.translate(build) 1171 call = self.build_call('list_suite_controls', build=build, 1172 suite_name=suite_name) 1173 return json.load(cStringIO.StringIO(self.run_call(call))) 1174 1175 1176class ImageServer(ImageServerBase): 1177 """Class for DevServer that handles RPCs related to CrOS images. 1178 1179 The calls to devserver to stage artifacts, including stage and download, are 1180 made in async mode. That is, when caller makes an RPC |stage| to request 1181 devserver to stage certain artifacts, devserver handles the call and starts 1182 staging artifacts in a new thread, and return |Success| without waiting for 1183 staging being completed. When caller receives message |Success|, it polls 1184 devserver's is_staged call until all artifacts are staged. 1185 Such mechanism is designed to prevent cherrypy threads in devserver being 1186 running out, as staging artifacts might take long time, and cherrypy starts 1187 with a fixed number of threads that handle devserver rpc. 1188 """ 1189 1190 class ArtifactUrls(object): 1191 """A container for URLs of staged artifacts. 1192 1193 Attributes: 1194 full_payload: URL for downloading a staged full release update 1195 mton_payload: URL for downloading a staged M-to-N release update 1196 nton_payload: URL for downloading a staged N-to-N release update 1197 1198 """ 1199 def __init__(self, full_payload=None, mton_payload=None, 1200 nton_payload=None): 1201 self.full_payload = full_payload 1202 self.mton_payload = mton_payload 1203 self.nton_payload = nton_payload 1204 1205 1206 def wait_for_artifacts_staged(self, archive_url, artifacts='', files=''): 1207 """Polling devserver.is_staged until all artifacts are staged. 1208 1209 @param archive_url: Google Storage URL for the build. 1210 @param artifacts: Comma separated list of artifacts to download. 1211 @param files: Comma separated list of files to download. 1212 @return: True if all artifacts are staged in devserver. 1213 """ 1214 kwargs = {'archive_url': archive_url, 1215 'artifacts': artifacts, 1216 'files': files} 1217 return self._poll_is_staged(**kwargs) 1218 1219 1220 @remote_devserver_call() 1221 def call_and_wait(self, call_name, archive_url, artifacts, files, 1222 error_message, expected_response=SUCCESS): 1223 """Helper method to make a urlopen call, and wait for artifacts staged. 1224 1225 @param call_name: name of devserver rpc call. 1226 @param archive_url: Google Storage URL for the build.. 1227 @param artifacts: Comma separated list of artifacts to download. 1228 @param files: Comma separated list of files to download. 1229 @param expected_response: Expected response from rpc, default to 1230 |Success|. If it's set to None, do not compare 1231 the actual response. Any response is consider 1232 to be good. 1233 @param error_message: Error message to be thrown if response does not 1234 match expected_response. 1235 1236 @return: The response from rpc. 1237 @raise DevServerException upon any return code that's expected_response. 1238 1239 """ 1240 kwargs = {'archive_url': archive_url, 1241 'artifacts': artifacts, 1242 'files': files} 1243 return self._call_and_wait(call_name, error_message, 1244 expected_response, **kwargs) 1245 1246 1247 @remote_devserver_call() 1248 def stage_artifacts(self, image=None, artifacts=None, files='', 1249 archive_url=None): 1250 """Tell the devserver to download and stage |artifacts| from |image|. 1251 1252 This is the main call point for staging any specific artifacts for a 1253 given build. To see the list of artifacts one can stage see: 1254 1255 ~src/platfrom/dev/artifact_info.py. 1256 1257 This is maintained along with the actual devserver code. 1258 1259 @param image: the image to fetch and stage. 1260 @param artifacts: A list of artifacts. 1261 @param files: A list of files to stage. 1262 @param archive_url: Optional parameter that has the archive_url to stage 1263 this artifact from. Default is specified in autotest config + 1264 image. 1265 1266 @raise DevServerException upon any return code that's not HTTP OK. 1267 """ 1268 if not artifacts and not files: 1269 raise DevServerException('Must specify something to stage.') 1270 image = self.translate(image) 1271 self._stage_artifacts(image, artifacts, files, archive_url) 1272 1273 1274 @remote_devserver_call(timeout_min=0.5) 1275 def list_image_dir(self, image): 1276 """List the contents of the image stage directory, on the devserver. 1277 1278 @param image: The image name, eg: <board>-<branch>/<Milestone>-<build>. 1279 1280 @raise DevServerException upon any return code that's not HTTP OK. 1281 """ 1282 image = self.translate(image) 1283 logging.info('Requesting contents from devserver %s for image %s', 1284 self.url(), image) 1285 archive_url = _get_storage_server_for_artifacts() + image 1286 call = self.build_call('list_image_dir', archive_url=archive_url) 1287 response = self.run_call(call, readline=True) 1288 for line in response: 1289 logging.info(line) 1290 1291 1292 def trigger_download(self, image, synchronous=True): 1293 """Tell the devserver to download and stage |image|. 1294 1295 Tells the devserver to fetch |image| from the image storage server 1296 named by _get_image_storage_server(). 1297 1298 If |synchronous| is True, waits for the entire download to finish 1299 staging before returning. Otherwise only the artifacts necessary 1300 to start installing images onto DUT's will be staged before returning. 1301 A caller can then call finish_download to guarantee the rest of the 1302 artifacts have finished staging. 1303 1304 @param image: the image to fetch and stage. 1305 @param synchronous: if True, waits until all components of the image are 1306 staged before returning. 1307 1308 @raise DevServerException upon any return code that's not HTTP OK. 1309 1310 """ 1311 image = self.translate(image) 1312 artifacts = _ARTIFACTS_TO_BE_STAGED_FOR_IMAGE 1313 self._trigger_download(image, artifacts, files='', 1314 synchronous=synchronous) 1315 1316 1317 @remote_devserver_call() 1318 def setup_telemetry(self, build): 1319 """Tell the devserver to setup telemetry for this build. 1320 1321 The devserver will stage autotest and then extract the required files 1322 for telemetry. 1323 1324 @param build: the build to setup telemetry for. 1325 1326 @returns path on the devserver that telemetry is installed to. 1327 """ 1328 build = self.translate(build) 1329 archive_url = _get_image_storage_server() + build 1330 call = self.build_call('setup_telemetry', archive_url=archive_url) 1331 try: 1332 response = self.run_call(call) 1333 except httplib.BadStatusLine as e: 1334 logging.error(e) 1335 raise DevServerException('Received Bad Status line, Devserver %s ' 1336 'might have gone down while handling ' 1337 'the call: %s' % (self.url(), call)) 1338 return response 1339 1340 1341 def finish_download(self, image): 1342 """Tell the devserver to finish staging |image|. 1343 1344 If trigger_download is called with synchronous=False, it will return 1345 before all artifacts have been staged. This method contacts the 1346 devserver and blocks until all staging is completed and should be 1347 called after a call to trigger_download. 1348 1349 @param image: the image to fetch and stage. 1350 @raise DevServerException upon any return code that's not HTTP OK. 1351 """ 1352 image = self.translate(image) 1353 artifacts = _ARTIFACTS_TO_BE_STAGED_FOR_IMAGE_WITH_AUTOTEST 1354 self._finish_download(image, artifacts, files='') 1355 1356 1357 def get_update_url(self, image): 1358 """Returns the url that should be passed to the updater. 1359 1360 @param image: the image that was fetched. 1361 """ 1362 image = self.translate(image) 1363 url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', 1364 type=str) 1365 return (url_pattern % (self.url(), image)) 1366 1367 1368 def get_staged_file_url(self, filename, image): 1369 """Returns the url of a staged file for this image on the devserver.""" 1370 return '/'.join([self._get_image_url(image), filename]) 1371 1372 1373 def get_full_payload_url(self, image): 1374 """Returns a URL to a staged full payload. 1375 1376 @param image: the image that was fetched. 1377 1378 @return A fully qualified URL that can be used for downloading the 1379 payload. 1380 1381 """ 1382 return self._get_image_url(image) + '/update.gz' 1383 1384 1385 def get_test_image_url(self, image): 1386 """Returns a URL to a staged test image. 1387 1388 @param image: the image that was fetched. 1389 1390 @return A fully qualified URL that can be used for downloading the 1391 image. 1392 1393 """ 1394 return self._get_image_url(image) + '/chromiumos_test_image.bin' 1395 1396 1397 @remote_devserver_call() 1398 def get_dependencies_file(self, build): 1399 """Ask the dev server for the contents of the suite dependencies file. 1400 1401 Ask the dev server at |self._dev_server| for the contents of the 1402 pre-processed suite dependencies file (at DEPENDENCIES_FILE) 1403 for |build|. 1404 1405 @param build: The build (e.g. x86-mario-release/R21-2333.0.0) 1406 whose dependencies the caller is interested in. 1407 @return The contents of the dependencies file, which should eval to 1408 a dict of dicts, as per site_utils/suite_preprocessor.py. 1409 @raise DevServerException upon any return code that's not HTTP OK. 1410 """ 1411 build = self.translate(build) 1412 call = self.build_call('controlfiles', 1413 build=build, control_path=DEPENDENCIES_FILE) 1414 return self.run_call(call) 1415 1416 1417 @remote_devserver_call() 1418 def get_latest_build_in_gs(self, board): 1419 """Ask the devservers for the latest offical build in Google Storage. 1420 1421 @param board: The board for who we want the latest official build. 1422 @return A string of the returned build rambi-release/R37-5868.0.0 1423 @raise DevServerException upon any return code that's not HTTP OK. 1424 """ 1425 call = self.build_call( 1426 'xbuddy_translate/remote/%s/latest-official' % board, 1427 image_dir=_get_image_storage_server()) 1428 image_name = self.run_call(call) 1429 return os.path.dirname(image_name) 1430 1431 1432 def translate(self, build_name): 1433 """Translate the build name if it's in LATEST format. 1434 1435 If the build name is in the format [builder]/LATEST, return the latest 1436 build in Google Storage otherwise return the build name as is. 1437 1438 @param build_name: build_name to check. 1439 1440 @return The actual build name to use. 1441 """ 1442 match = re.match(r'([\w-]+)-(\w+)/LATEST', build_name) 1443 if not match: 1444 return build_name 1445 translated_build = self.get_latest_build_in_gs(match.groups()[0]) 1446 logging.debug('Translated relative build %s to %s', build_name, 1447 translated_build) 1448 return translated_build 1449 1450 1451 @classmethod 1452 @remote_devserver_call() 1453 def get_latest_build(cls, target, milestone=''): 1454 """Ask all the devservers for the latest build for a given target. 1455 1456 @param target: The build target, typically a combination of the board 1457 and the type of build e.g. x86-mario-release. 1458 @param milestone: For latest build set to '', for builds only in a 1459 specific milestone set to a str of format Rxx 1460 (e.g. R16). Default: ''. Since we are dealing with a 1461 webserver sending an empty string, '', ensures that 1462 the variable in the URL is ignored as if it was set 1463 to None. 1464 @return A string of the returned build e.g. R20-2226.0.0. 1465 @raise DevServerException upon any return code that's not HTTP OK. 1466 """ 1467 calls = cls.build_all_calls('latestbuild', target=target, 1468 milestone=milestone) 1469 latest_builds = [] 1470 for call in calls: 1471 latest_builds.append(cls.run_call(call)) 1472 1473 return max(latest_builds, key=version.LooseVersion) 1474 1475 1476class AndroidBuildServer(ImageServerBase): 1477 """Class for DevServer that handles RPCs related to Android builds. 1478 1479 The calls to devserver to stage artifacts, including stage and download, are 1480 made in async mode. That is, when caller makes an RPC |stage| to request 1481 devserver to stage certain artifacts, devserver handles the call and starts 1482 staging artifacts in a new thread, and return |Success| without waiting for 1483 staging being completed. When caller receives message |Success|, it polls 1484 devserver's is_staged call until all artifacts are staged. 1485 Such mechanism is designed to prevent cherrypy threads in devserver being 1486 running out, as staging artifacts might take long time, and cherrypy starts 1487 with a fixed number of threads that handle devserver rpc. 1488 """ 1489 1490 def wait_for_artifacts_staged(self, target, build_id, branch, 1491 archive_url=None, artifacts='', files=''): 1492 """Polling devserver.is_staged until all artifacts are staged. 1493 1494 @param target: Target of the android build to stage, e.g., 1495 shamu-userdebug. 1496 @param build_id: Build id of the android build to stage. 1497 @param branch: Branch of the android build to stage. 1498 @param archive_url: Google Storage URL for the build. 1499 @param artifacts: Comma separated list of artifacts to download. 1500 @param files: Comma separated list of files to download. 1501 1502 @return: True if all artifacts are staged in devserver. 1503 """ 1504 kwargs = {'target': target, 1505 'build_id': build_id, 1506 'branch': branch, 1507 'artifacts': artifacts, 1508 'files': files, 1509 'os_type': 'android'} 1510 if archive_url: 1511 kwargs['archive_url'] = archive_url 1512 return self._poll_is_staged(**kwargs) 1513 1514 1515 @remote_devserver_call() 1516 def call_and_wait(self, call_name, target, build_id, branch, archive_url, 1517 artifacts, files, error_message, 1518 expected_response=SUCCESS): 1519 """Helper method to make a urlopen call, and wait for artifacts staged. 1520 1521 @param call_name: name of devserver rpc call. 1522 @param target: Target of the android build to stage, e.g., 1523 shamu-userdebug. 1524 @param build_id: Build id of the android build to stage. 1525 @param branch: Branch of the android build to stage. 1526 @param archive_url: Google Storage URL for the CrOS build. 1527 @param artifacts: Comma separated list of artifacts to download. 1528 @param files: Comma separated list of files to download. 1529 @param expected_response: Expected response from rpc, default to 1530 |Success|. If it's set to None, do not compare 1531 the actual response. Any response is consider 1532 to be good. 1533 @param error_message: Error message to be thrown if response does not 1534 match expected_response. 1535 1536 @return: The response from rpc. 1537 @raise DevServerException upon any return code that's expected_response. 1538 1539 """ 1540 kwargs = {'target': target, 1541 'build_id': build_id, 1542 'branch': branch, 1543 'artifacts': artifacts, 1544 'files': files, 1545 'os_type': 'android'} 1546 if archive_url: 1547 kwargs['archive_url'] = archive_url 1548 return self._call_and_wait(call_name, error_message, expected_response, 1549 **kwargs) 1550 1551 1552 @remote_devserver_call() 1553 def stage_artifacts(self, target=None, build_id=None, branch=None, 1554 image=None, artifacts=None, files='', archive_url=None): 1555 """Tell the devserver to download and stage |artifacts| from |image|. 1556 1557 This is the main call point for staging any specific artifacts for a 1558 given build. To see the list of artifacts one can stage see: 1559 1560 ~src/platfrom/dev/artifact_info.py. 1561 1562 This is maintained along with the actual devserver code. 1563 1564 @param target: Target of the android build to stage, e.g., 1565 shamu-userdebug. 1566 @param build_id: Build id of the android build to stage. 1567 @param branch: Branch of the android build to stage. 1568 @param image: Name of a build to test, in the format of 1569 branch/target/build_id 1570 @param artifacts: A list of artifacts. 1571 @param files: A list of files to stage. 1572 @param archive_url: Optional parameter that has the archive_url to stage 1573 this artifact from. Default is specified in autotest config + 1574 image. 1575 1576 @raise DevServerException upon any return code that's not HTTP OK. 1577 """ 1578 if image and not target and not build_id and not branch: 1579 branch, target, build_id = utils.parse_launch_control_build(image) 1580 if not target or not build_id or not branch: 1581 raise DevServerException('Must specify all build info (target, ' 1582 'build_id and branch) to stage.') 1583 1584 android_build_info = {'target': target, 1585 'build_id': build_id, 1586 'branch': branch} 1587 if not artifacts and not files: 1588 raise DevServerException('Must specify something to stage.') 1589 if not all(android_build_info.values()): 1590 raise DevServerException( 1591 'To stage an Android build, must specify target, build id ' 1592 'and branch.') 1593 build = ANDROID_BUILD_NAME_PATTERN % android_build_info 1594 self._stage_artifacts(build, artifacts, files, archive_url, 1595 **android_build_info) 1596 1597 1598 def trigger_download(self, target, build_id, branch, artifacts=None, 1599 is_brillo=False, synchronous=True): 1600 """Tell the devserver to download and stage an Android build. 1601 1602 Tells the devserver to fetch an Android build from the image storage 1603 server named by _get_image_storage_server(). 1604 1605 If |synchronous| is True, waits for the entire download to finish 1606 staging before returning. Otherwise only the artifacts necessary 1607 to start installing images onto DUT's will be staged before returning. 1608 A caller can then call finish_download to guarantee the rest of the 1609 artifacts have finished staging. 1610 1611 @param target: Target of the android build to stage, e.g., 1612 shamu-userdebug. 1613 @param build_id: Build id of the android build to stage. 1614 @param branch: Branch of the android build to stage. 1615 @param artifacts: A string of artifacts separated by comma. If None, 1616 use the default artifacts for Android or Brillo build. 1617 @param is_brillo: Set to True if it's a Brillo build. Default is False. 1618 @param synchronous: if True, waits until all components of the image are 1619 staged before returning. 1620 1621 @raise DevServerException upon any return code that's not HTTP OK. 1622 1623 """ 1624 android_build_info = {'target': target, 1625 'build_id': build_id, 1626 'branch': branch} 1627 build = ANDROID_BUILD_NAME_PATTERN % android_build_info 1628 if not artifacts: 1629 if is_brillo: 1630 artifacts = _BRILLO_ARTIFACTS_TO_BE_STAGED_FOR_IMAGE 1631 else: 1632 board = target.split('-')[0] 1633 artifacts = ( 1634 android_utils.AndroidArtifacts.get_artifacts_for_reimage( 1635 board)) 1636 self._trigger_download(build, artifacts, files='', 1637 synchronous=synchronous, **android_build_info) 1638 1639 1640 def finish_download(self, target, build_id, branch, is_brillo=False): 1641 """Tell the devserver to finish staging an Android build. 1642 1643 If trigger_download is called with synchronous=False, it will return 1644 before all artifacts have been staged. This method contacts the 1645 devserver and blocks until all staging is completed and should be 1646 called after a call to trigger_download. 1647 1648 @param target: Target of the android build to stage, e.g., 1649 shamu-userdebug. 1650 @param build_id: Build id of the android build to stage. 1651 @param branch: Branch of the android build to stage. 1652 @param is_brillo: Set to True if it's a Brillo build. Default is False. 1653 1654 @raise DevServerException upon any return code that's not HTTP OK. 1655 """ 1656 android_build_info = {'target': target, 1657 'build_id': build_id, 1658 'branch': branch} 1659 build = ANDROID_BUILD_NAME_PATTERN % android_build_info 1660 if is_brillo: 1661 artifacts = _BRILLO_ARTIFACTS_TO_BE_STAGED_FOR_IMAGE 1662 else: 1663 board = target.split('-')[0] 1664 artifacts = ( 1665 android_utils.AndroidArtifacts.get_artifacts_for_reimage( 1666 board)) 1667 self._finish_download(build, artifacts, files='', **android_build_info) 1668 1669 1670 def get_staged_file_url(self, filename, target, build_id, branch): 1671 """Returns the url of a staged file for this image on the devserver. 1672 1673 @param filename: Name of the file. 1674 @param target: Target of the android build to stage, e.g., 1675 shamu-userdebug. 1676 @param build_id: Build id of the android build to stage. 1677 @param branch: Branch of the android build to stage. 1678 1679 @return: The url of a staged file for this image on the devserver. 1680 """ 1681 android_build_info = {'target': target, 1682 'build_id': build_id, 1683 'branch': branch, 1684 'os_type': 'android'} 1685 build = ANDROID_BUILD_NAME_PATTERN % android_build_info 1686 return '/'.join([self._get_image_url(build), filename]) 1687 1688 1689 @remote_devserver_call() 1690 def translate(self, build_name): 1691 """Translate the build name if it's in LATEST format. 1692 1693 If the build name is in the format [branch]/[target]/LATEST, return the 1694 latest build in Launch Control otherwise return the build name as is. 1695 1696 @param build_name: build_name to check. 1697 1698 @return The actual build name to use. 1699 """ 1700 branch, target, build_id = utils.parse_launch_control_build(build_name) 1701 if build_id != 'LATEST': 1702 return build_name 1703 call = self.build_call('latestbuild', branch=branch, target=target, 1704 os_type='android') 1705 translated_build_id = self.run_call(call) 1706 translated_build = (ANDROID_BUILD_NAME_PATTERN % 1707 {'branch': branch, 1708 'target': target, 1709 'build_id': translated_build_id}) 1710 logging.debug('Translated relative build %s to %s', build_name, 1711 translated_build) 1712 return translated_build 1713 1714 1715def _is_load_healthy(load): 1716 """Check if devserver's load meets the minimum threshold. 1717 1718 @param load: The devserver's load stats to check. 1719 1720 @return: True if the load meets the minimum threshold. Return False 1721 otherwise. 1722 1723 """ 1724 # Threshold checks, including CPU load. 1725 if load[DevServer.CPU_LOAD] > DevServer.MAX_CPU_LOAD: 1726 logging.debug('CPU load of devserver %s is at %s%%, which is higher ' 1727 'than the threshold of %s%%', load['devserver'], 1728 load[DevServer.CPU_LOAD], DevServer.MAX_CPU_LOAD) 1729 return False 1730 if load[DevServer.NETWORK_IO] > DevServer.MAX_NETWORK_IO: 1731 logging.debug('Network IO of devserver %s is at %i Bps, which is ' 1732 'higher than the threshold of %i bytes per second.', 1733 load['devserver'], load[DevServer.NETWORK_IO], 1734 DevServer.MAX_NETWORK_IO) 1735 return False 1736 return True 1737 1738 1739def _compare_load(devserver1, devserver2): 1740 """Comparator function to compare load between two devservers. 1741 1742 @param devserver1: A dictionary of devserver load stats to be compared. 1743 @param devserver2: A dictionary of devserver load stats to be compared. 1744 1745 @return: Negative value if the load of `devserver1` is less than the load 1746 of `devserver2`. Return positive value otherwise. 1747 1748 """ 1749 return int(devserver1[DevServer.DISK_IO] - devserver2[DevServer.DISK_IO]) 1750 1751 1752def get_least_loaded_devserver(devserver_type=ImageServer, hostname=None): 1753 """Get the devserver with the least load. 1754 1755 Iterate through all devservers and get the one with least load. 1756 1757 TODO(crbug.com/486278): Devserver with required build already staged should 1758 take higher priority. This will need check_health call to be able to verify 1759 existence of a given build/artifact. Also, in case all devservers are 1760 overloaded, the logic here should fall back to the old behavior that randomly 1761 selects a devserver based on the hash of the image name/url. 1762 1763 @param devserver_type: Type of devserver to select from. Default is set to 1764 ImageServer. 1765 @param hostname: Hostname of the dut that the devserver is used for. The 1766 picked devserver needs to respect the location of the host if 1767 `prefer_local_devserver` is set to True or `restricted_subnets` is 1768 set. 1769 1770 @return: Name of the devserver with the least load. 1771 1772 """ 1773 devservers, can_retry = devserver_type.get_available_devservers( 1774 hostname) 1775 # If no healthy devservers available and can_retry is False, return None. 1776 # Otherwise, relax the constrain on hostname, allow all devservers to be 1777 # available. 1778 if not devserver_type.get_healthy_devserver('', devservers): 1779 if not can_retry: 1780 return None 1781 else: 1782 devservers, _ = devserver_type.get_available_devservers() 1783 1784 # get_devserver_load call needs to be made in a new process to allow force 1785 # timeout using signal. 1786 output = multiprocessing.Queue() 1787 processes = [] 1788 for devserver in devservers: 1789 processes.append(multiprocessing.Process( 1790 target=devserver_type.get_devserver_load_wrapper, 1791 args=(devserver, TIMEOUT_GET_DEVSERVER_LOAD, output))) 1792 1793 for p in processes: 1794 p.start() 1795 for p in processes: 1796 p.join() 1797 loads = [output.get() for p in processes] 1798 # Filter out any load failed to be retrieved or does not support load check. 1799 loads = [load for load in loads if load and DevServer.CPU_LOAD in load and 1800 DevServer.is_free_disk_ok(load) and 1801 DevServer.is_apache_client_count_ok(load)] 1802 if not loads: 1803 logging.debug('Failed to retrieve load stats from any devserver. No ' 1804 'load balancing can be applied.') 1805 return None 1806 loads = [load for load in loads if _is_load_healthy(load)] 1807 if not loads: 1808 logging.error('No devserver has the capacity to be selected.') 1809 return None 1810 loads = sorted(loads, cmp=_compare_load) 1811 return loads[0]['devserver'] 1812 1813 1814def resolve(build, hostname=None): 1815 """Resolve a devserver can be used for given build and hostname. 1816 1817 @param build: Name of a build to stage on devserver, e.g., 1818 ChromeOS build: daisy-release/R50-1234.0.0 1819 Launch Control build: git_mnc_release/shamu-eng 1820 @param hostname: Hostname of a devserver for, default is None, which means 1821 devserver is not restricted by the network location of the host. 1822 1823 @return: A DevServer instance that can be used to stage given build for the 1824 given host. 1825 """ 1826 if utils.is_launch_control_build(build): 1827 return AndroidBuildServer.resolve(build, hostname) 1828 else: 1829 return ImageServer.resolve(build, hostname) 1830