dev_server.py revision 35476b958f33c6bada2932237a97f59b483b451d
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from distutils import version 6import json 7import logging 8import os 9import urllib2 10import HTMLParser 11import cStringIO 12import re 13 14from autotest_lib.client.common_lib import global_config 15from autotest_lib.client.common_lib import utils 16from autotest_lib.client.common_lib.cros import retry 17from autotest_lib.site_utils.graphite import stats 18# TODO(cmasone): redo this class using requests module; http://crosbug.com/30107 19 20 21CONFIG = global_config.global_config 22# This file is generated at build time and specifies, per suite and per test, 23# the DEPENDENCIES list specified in each control file. It's a dict of dicts: 24# {'bvt': {'/path/to/autotest/control/site_tests/test1/control': ['dep1']} 25# 'suite': {'/path/to/autotest/control/site_tests/test2/control': ['dep2']} 26# 'power': {'/path/to/autotest/control/site_tests/test1/control': ['dep1'], 27# '/path/to/autotest/control/site_tests/test3/control': ['dep3']} 28# } 29DEPENDENCIES_FILE = 'test_suites/dependency_info' 30 31 32class MarkupStripper(HTMLParser.HTMLParser): 33 """HTML parser that strips HTML tags, coded characters like & 34 35 Works by, basically, not doing anything for any tags, and only recording 36 the content of text nodes in an internal data structure. 37 """ 38 def __init__(self): 39 self.reset() 40 self.fed = [] 41 42 43 def handle_data(self, d): 44 """Consume content of text nodes, store it away.""" 45 self.fed.append(d) 46 47 48 def get_data(self): 49 """Concatenate and return all stored data.""" 50 return ''.join(self.fed) 51 52 53def _get_image_storage_server(): 54 return CONFIG.get_config_value('CROS', 'image_storage_server', type=str) 55 56 57def _get_dev_server_list(): 58 return CONFIG.get_config_value('CROS', 'dev_server', type=list, default=[]) 59 60 61def _get_crash_server_list(): 62 return CONFIG.get_config_value('CROS', 'crash_server', type=list, 63 default=[]) 64 65 66def remote_devserver_call(timeout_min=30): 67 """A decorator to use with remote devserver calls. 68 69 This decorator converts urllib2.HTTPErrors into DevServerExceptions with 70 any embedded error info converted into plain text. 71 """ 72 #pylint: disable=C0111 73 def inner_decorator(method): 74 75 @retry.retry(urllib2.URLError, timeout_min=timeout_min) 76 def wrapper(*args, **kwargs): 77 """This wrapper actually catches the HTTPError.""" 78 try: 79 return method(*args, **kwargs) 80 except urllib2.HTTPError as e: 81 error_markup = e.read() 82 strip = MarkupStripper() 83 try: 84 strip.feed(error_markup.decode('utf_32')) 85 except UnicodeDecodeError: 86 strip.feed(error_markup) 87 raise DevServerException(strip.get_data()) 88 89 return wrapper 90 91 return inner_decorator 92 93 94class DevServerException(Exception): 95 """Raised when the dev server returns a non-200 HTTP response.""" 96 pass 97 98 99class DevServer(object): 100 """Base class for all DevServer-like server stubs. 101 102 This is the base class for interacting with all Dev Server-like servers. 103 A caller should instantiate a sub-class of DevServer with: 104 105 host = SubClassServer.resolve(build) 106 server = SubClassServer(host) 107 """ 108 _MIN_FREE_DISK_SPACE_GB = 20 109 110 def __init__(self, devserver): 111 self._devserver = devserver 112 113 114 def url(self): 115 """Returns the url for this devserver.""" 116 return self._devserver 117 118 119 @staticmethod 120 def devserver_healthy(devserver, timeout_min=0.1): 121 """Returns True if the |devserver| is healthy to stage build. 122 123 @param devserver: url of the devserver. 124 @param timeout_min: How long to wait in minutes before deciding the 125 the devserver is not up (float). 126 """ 127 server_name = re.sub(r':\d+$', '', devserver.lstrip('http://')) 128 # statsd treats |.| as path separator. 129 server_name = server_name.replace('.', '_') 130 call = DevServer._build_call(devserver, 'check_health') 131 132 @remote_devserver_call(timeout_min=timeout_min) 133 def make_call(): 134 """Inner method that makes the call.""" 135 return utils.urlopen_socket_timeout(call, 136 timeout=timeout_min*60).read() 137 138 try: 139 result_dict = json.load(cStringIO.StringIO(make_call())) 140 free_disk = result_dict['free_disk'] 141 stats.Gauge(server_name).send('free_disk', free_disk) 142 143 skip_devserver_health_check = CONFIG.get_config_value('CROS', 144 'skip_devserver_health_check', 145 type=bool) 146 if skip_devserver_health_check: 147 logging.debug('devserver health check is skipped.') 148 elif (free_disk < DevServer._MIN_FREE_DISK_SPACE_GB): 149 logging.error('Devserver check_health failed. Free disk space ' 150 'is low. Only %dGB is available.', free_disk) 151 stats.Counter(server_name +'.devserver_not_healthy').increment() 152 return False 153 154 # This counter indicates the load of a devserver. By comparing the 155 # value of this counter for all devservers, we can evaluate the 156 # load balancing across all devservers. 157 stats.Counter(server_name + '.devserver_healthy').increment() 158 return True 159 except Exception as e: 160 logging.error('Devserver call failed: "%s", timeout: %s seconds,' 161 ' Error: %s', call, timeout_min*60, str(e)) 162 stats.Counter(server_name + '.devserver_not_healthy').increment() 163 return False 164 165 166 @staticmethod 167 def _build_call(host, method, **kwargs): 168 """Build a URL to |host| that calls |method|, passing |kwargs|. 169 170 Builds a URL that calls |method| on the dev server defined by |host|, 171 passing a set of key/value pairs built from the dict |kwargs|. 172 173 @param host: a string that is the host basename e.g. http://server:90. 174 @param method: the dev server method to call. 175 @param kwargs: a dict mapping arg names to arg values. 176 @return the URL string. 177 """ 178 argstr = '&'.join(map(lambda x: "%s=%s" % x, kwargs.iteritems())) 179 return "%(host)s/%(method)s?%(argstr)s" % dict( 180 host=host, method=method, argstr=argstr) 181 182 183 def build_call(self, method, **kwargs): 184 """Builds a devserver RPC string that can be invoked using urllib.open. 185 186 @param method: remote devserver method to call. 187 """ 188 return self._build_call(self._devserver, method, **kwargs) 189 190 191 @classmethod 192 def build_all_calls(cls, method, **kwargs): 193 """Builds a list of URLs that makes RPC calls on all devservers. 194 195 Build a URL that calls |method| on the dev server, passing a set 196 of key/value pairs built from the dict |kwargs|. 197 198 @param method: the dev server method to call. 199 @param kwargs: a dict mapping arg names to arg values 200 @return the URL string 201 """ 202 calls = [] 203 # Note we use cls.servers as servers is class specific. 204 for server in cls.servers(): 205 if cls.devserver_healthy(server): 206 calls.append(cls._build_call(server, method, **kwargs)) 207 208 return calls 209 210 211 @staticmethod 212 def servers(): 213 """Returns a list of servers that can serve as this type of server.""" 214 raise NotImplementedError() 215 216 217 @classmethod 218 def resolve(cls, build): 219 """"Resolves a build to a devserver instance. 220 221 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514). 222 """ 223 devservers = cls.servers() 224 while devservers: 225 hash_index = hash(build) % len(devservers) 226 devserver = devservers.pop(hash_index) 227 if cls.devserver_healthy(devserver): 228 return cls(devserver) 229 else: 230 logging.error('All devservers are currently down!!!') 231 raise DevServerException('All devservers are currently down!!!') 232 233 234class CrashServer(DevServer): 235 """Class of DevServer that symbolicates crash dumps.""" 236 @staticmethod 237 def servers(): 238 return _get_crash_server_list() 239 240 241 @remote_devserver_call() 242 def symbolicate_dump(self, minidump_path, build): 243 """Ask the devserver to symbolicate the dump at minidump_path. 244 245 Stage the debug symbols for |build| and, if that works, ask the 246 devserver to symbolicate the dump at |minidump_path|. 247 248 @param minidump_path: the on-disk path of the minidump. 249 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 250 whose debug symbols are needed for symbolication. 251 @return The contents of the stack trace 252 @raise DevServerException upon any return code that's not HTTP OK. 253 """ 254 try: 255 import requests 256 except ImportError: 257 logging.warning("Can't 'import requests' to connect to dev server.") 258 return '' 259 260 # Symbolicate minidump. 261 call = self.build_call('symbolicate_dump', 262 archive_url=_get_image_storage_server() + build) 263 request = requests.post( 264 call, files={'minidump': open(minidump_path, 'rb')}) 265 if request.status_code == requests.codes.OK: 266 return request.text 267 268 error_fd = cStringIO.StringIO(request.text) 269 raise urllib2.HTTPError( 270 call, request.status_code, request.text, request.headers, 271 error_fd) 272 273 274class ImageServer(DevServer): 275 """Class for DevServer that handles image-related RPCs.""" 276 @staticmethod 277 def servers(): 278 return _get_dev_server_list() 279 280 281 @classmethod 282 def devserver_url_for_servo(cls, board): 283 """Returns the devserver url for use with servo recovery. 284 285 @param board: The board (e.g. 'x86-mario'). 286 """ 287 # Ideally, for load balancing we'd select the server based 288 # on the board. For now, to simplify manual steps on the 289 # server side, we ignore the board type and hard-code the 290 # server as first in the list. 291 # 292 # TODO(jrbarnette) Once we have automated selection of the 293 # build for recovery, we should revisit this. 294 url_pattern = CONFIG.get_config_value('CROS', 295 'servo_url_pattern', 296 type=str) 297 return url_pattern % (cls.servers()[0], board) 298 299 300 class ArtifactUrls(object): 301 """A container for URLs of staged artifacts. 302 303 Attributes: 304 full_payload: URL for downloading a staged full release update 305 mton_payload: URL for downloading a staged M-to-N release update 306 nton_payload: URL for downloading a staged N-to-N release update 307 308 """ 309 def __init__(self, full_payload=None, mton_payload=None, 310 nton_payload=None): 311 self.full_payload = full_payload 312 self.mton_payload = mton_payload 313 self.nton_payload = nton_payload 314 315 316 @remote_devserver_call() 317 def stage_artifacts(self, image, artifacts): 318 """Tell the devserver to download and stage |artifacts| from |image|. 319 320 This is the main call point for staging any specific artifacts for a 321 given build. To see the list of artifacts one can stage see: 322 323 ~src/platfrom/dev/artifact_info.py. 324 325 This is maintained along with the actual devserver code. 326 327 @param image: the image to fetch and stage. 328 @param artifacts: A list of artifacts. 329 330 @raise DevServerException upon any return code that's not HTTP OK. 331 """ 332 call = self.build_call('stage', 333 archive_url=_get_image_storage_server() + image, 334 artifacts=','.join(artifacts)) 335 response = urllib2.urlopen(call) 336 if not response.read() == 'Success': 337 raise DevServerException("staging artifacts %s for %s failed;" 338 "HTTP OK not accompanied by 'Success'." % 339 (' '.join(artifacts), image)) 340 341 342 @remote_devserver_call() 343 def trigger_download(self, image, synchronous=True): 344 """Tell the devserver to download and stage |image|. 345 346 Tells the devserver to fetch |image| from the image storage server 347 named by _get_image_storage_server(). 348 349 If |synchronous| is True, waits for the entire download to finish 350 staging before returning. Otherwise only the artifacts necessary 351 to start installing images onto DUT's will be staged before returning. 352 A caller can then call finish_download to guarantee the rest of the 353 artifacts have finished staging. 354 355 @param image: the image to fetch and stage. 356 @param synchronous: if True, waits until all components of the image are 357 staged before returning. 358 359 @raise DevServerException upon any return code that's not HTTP OK. 360 361 """ 362 call = self.build_call( 363 'download', archive_url=_get_image_storage_server() + image) 364 response = urllib2.urlopen(call) 365 was_successful = response.read() == 'Success' 366 if was_successful and synchronous: 367 self.finish_download(image) 368 elif not was_successful: 369 raise DevServerException("trigger_download for %s failed;" 370 "HTTP OK not accompanied by 'Success'." % 371 image) 372 373 374 @remote_devserver_call() 375 def setup_telemetry(self, build): 376 """Tell the devserver to setup telemetry for this build. 377 378 The devserver will stage autotest and then extract the required files 379 for telemetry. 380 381 @param build: the build to setup telemetry for. 382 383 @returns path on the devserver that telemetry is installed to. 384 """ 385 call = self.build_call( 386 'setup_telemetry', 387 archive_url=_get_image_storage_server() + build) 388 response = urllib2.urlopen(call) 389 return response.read() 390 391 392 @remote_devserver_call() 393 def finish_download(self, image): 394 """Tell the devserver to finish staging |image|. 395 396 If trigger_download is called with synchronous=False, it will return 397 before all artifacts have been staged. This method contacts the 398 devserver and blocks until all staging is completed and should be 399 called after a call to trigger_download. 400 401 @param image: the image to fetch and stage. 402 @raise DevServerException upon any return code that's not HTTP OK. 403 """ 404 call = self.build_call('wait_for_status', 405 archive_url=_get_image_storage_server() + image) 406 if urllib2.urlopen(call).read() != 'Success': 407 raise DevServerException("finish_download for %s failed;" 408 "HTTP OK not accompanied by 'Success'." % 409 image) 410 411 def get_update_url(self, image): 412 """Returns the url that should be passed to the updater. 413 414 @param image: the image that was fetched. 415 """ 416 url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', 417 type=str) 418 return (url_pattern % (self.url(), image)) 419 420 421 def _get_image_url(self, image): 422 """Returns the url of the directory for this image on the devserver. 423 424 @param image: the image that was fetched. 425 """ 426 url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', 427 type=str) 428 return (url_pattern % (self.url(), image)).replace( 429 'update', 'static') 430 431 432 def get_delta_payload_url(self, payload_type, image): 433 """Returns a URL to a staged delta payload. 434 435 @param payload_type: either 'mton' or 'nton' 436 @param image: the image that was fetched. 437 438 @return A fully qualified URL that can be used for downloading the 439 payload. 440 441 @raise DevServerException if payload type argument is invalid. 442 443 """ 444 if payload_type not in ('mton', 'nton'): 445 raise DevServerException('invalid delta payload type: %s' % 446 payload_type) 447 version = os.path.basename(image) 448 base_url = self._get_image_url(image) 449 return base_url + '/au/%s_%s/update.gz' % (version, payload_type) 450 451 452 def get_full_payload_url(self, image): 453 """Returns a URL to a staged full payload. 454 455 @param image: the image that was fetched. 456 457 @return A fully qualified URL that can be used for downloading the 458 payload. 459 460 """ 461 return self._get_image_url(image) + '/update.gz' 462 463 464 def get_test_image_url(self, image): 465 """Returns a URL to a staged test image. 466 467 @param image: the image that was fetched. 468 469 @return A fully qualified URL that can be used for downloading the 470 image. 471 472 """ 473 return self._get_image_url(image) + '/chromiumos_test_image.bin' 474 475 476 @remote_devserver_call() 477 def list_control_files(self, build): 478 """Ask the devserver to list all control files for |build|. 479 480 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 481 whose control files the caller wants listed. 482 @return None on failure, or a list of control file paths 483 (e.g. server/site_tests/autoupdate/control) 484 @raise DevServerException upon any return code that's not HTTP OK. 485 """ 486 call = self.build_call('controlfiles', build=build) 487 response = urllib2.urlopen(call) 488 return [line.rstrip() for line in response] 489 490 491 @remote_devserver_call() 492 def get_control_file(self, build, control_path): 493 """Ask the devserver for the contents of a control file. 494 495 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 496 whose control file the caller wants to fetch. 497 @param control_path: The file to fetch 498 (e.g. server/site_tests/autoupdate/control) 499 @return The contents of the desired file. 500 @raise DevServerException upon any return code that's not HTTP OK. 501 """ 502 call = self.build_call('controlfiles', build=build, 503 control_path=control_path) 504 return urllib2.urlopen(call).read() 505 506 507 @remote_devserver_call() 508 def get_dependencies_file(self, build): 509 """Ask the dev server for the contents of the suite dependencies file. 510 511 Ask the dev server at |self._dev_server| for the contents of the 512 pre-processed suite dependencies file (at DEPENDENCIES_FILE) 513 for |build|. 514 515 @param build: The build (e.g. x86-mario-release/R21-2333.0.0) 516 whose dependencies the caller is interested in. 517 @return The contents of the dependencies file, which should eval to 518 a dict of dicts, as per site_utils/suite_preprocessor.py. 519 @raise DevServerException upon any return code that's not HTTP OK. 520 """ 521 call = self.build_call('controlfiles', 522 build=build, control_path=DEPENDENCIES_FILE) 523 return urllib2.urlopen(call).read() 524 525 526 @classmethod 527 @remote_devserver_call() 528 def get_latest_build(cls, target, milestone=''): 529 """Ask all the devservers for the latest build for a given target. 530 531 @param target: The build target, typically a combination of the board 532 and the type of build e.g. x86-mario-release. 533 @param milestone: For latest build set to '', for builds only in a 534 specific milestone set to a str of format Rxx 535 (e.g. R16). Default: ''. Since we are dealing with a 536 webserver sending an empty string, '', ensures that 537 the variable in the URL is ignored as if it was set 538 to None. 539 @return A string of the returned build e.g. R20-2226.0.0. 540 @raise DevServerException upon any return code that's not HTTP OK. 541 """ 542 calls = cls.build_all_calls('latestbuild', target=target, 543 milestone=milestone) 544 latest_builds = [] 545 for call in calls: 546 latest_builds.append(urllib2.urlopen(call).read()) 547 548 return max(latest_builds, key=version.LooseVersion) 549