dev_server.py revision 31b2e3196a0507f439ba130bdc53dea83288a5d9
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from distutils import version 6import logging 7import os 8import urllib2 9import HTMLParser 10import cStringIO 11 12from autotest_lib.client.common_lib import global_config 13from autotest_lib.client.common_lib.cros import retry 14# TODO(cmasone): redo this class using requests module; http://crosbug.com/30107 15 16 17CONFIG = global_config.global_config 18# This file is generated at build time and specifies, per suite and per test, 19# the DEPENDENCIES list specified in each control file. It's a dict of dicts: 20# {'bvt': {'/path/to/autotest/control/site_tests/test1/control': ['dep1']} 21# 'suite': {'/path/to/autotest/control/site_tests/test2/control': ['dep2']} 22# 'power': {'/path/to/autotest/control/site_tests/test1/control': ['dep1'], 23# '/path/to/autotest/control/site_tests/test3/control': ['dep3']} 24# } 25DEPENDENCIES_FILE = 'test_suites/dependency_info' 26 27 28class MarkupStripper(HTMLParser.HTMLParser): 29 """HTML parser that strips HTML tags, coded characters like & 30 31 Works by, basically, not doing anything for any tags, and only recording 32 the content of text nodes in an internal data structure. 33 """ 34 def __init__(self): 35 self.reset() 36 self.fed = [] 37 38 39 def handle_data(self, d): 40 """Consume content of text nodes, store it away.""" 41 self.fed.append(d) 42 43 44 def get_data(self): 45 """Concatenate and return all stored data.""" 46 return ''.join(self.fed) 47 48 49def _get_image_storage_server(): 50 return CONFIG.get_config_value('CROS', 'image_storage_server', type=str) 51 52 53def _get_dev_server_list(): 54 return CONFIG.get_config_value('CROS', 'dev_server', type=list, default=[]) 55 56 57def _get_crash_server_list(): 58 return CONFIG.get_config_value('CROS', 'crash_server', type=list, 59 default=[]) 60 61 62def remote_devserver_call(timeout_min=30): 63 """A decorator to use with remote devserver calls. 64 65 This decorator converts urllib2.HTTPErrors into DevServerExceptions with 66 any embedded error info converted into plain text. 67 """ 68 #pylint: disable=C0111 69 def inner_decorator(method): 70 71 @retry.retry(urllib2.URLError, timeout_min=timeout_min) 72 def wrapper(*args, **kwargs): 73 """This wrapper actually catches the HTTPError.""" 74 try: 75 return method(*args, **kwargs) 76 except urllib2.HTTPError as e: 77 error_markup = e.read() 78 strip = MarkupStripper() 79 try: 80 strip.feed(error_markup.decode('utf_32')) 81 except UnicodeDecodeError: 82 strip.feed(error_markup) 83 raise DevServerException(strip.get_data()) 84 85 return wrapper 86 87 return inner_decorator 88 89 90class DevServerException(Exception): 91 """Raised when the dev server returns a non-200 HTTP response.""" 92 pass 93 94 95 96class DevServer(object): 97 """Base class for all DevServer-like server stubs. 98 99 This is the base class for interacting with all Dev Server-like servers. 100 A caller should instantiate a sub-class of DevServer with: 101 102 host = SubClassServer.resolve(build) 103 server = SubClassServer(host) 104 """ 105 def __init__(self, devserver): 106 self._devserver = devserver 107 108 109 def url(self): 110 """Returns the url for this devserver.""" 111 return self._devserver 112 113 114 @staticmethod 115 def devserver_up(devserver, timeout_min=0.1): 116 """Returns True if the |devserver| is responding to calls. 117 118 @param devserver: url of the devserver. 119 @param timeout_min: How long to wait in minutes before deciding the 120 the devserver is not up (float). 121 """ 122 call = DevServer._build_call(devserver, 'index') 123 124 @remote_devserver_call(timeout_min=timeout_min) 125 def make_call(): 126 """Inner method that makes the call.""" 127 urllib2.urlopen(call) 128 129 try: 130 make_call() 131 return True 132 except (DevServerException, urllib2.URLError): 133 return False 134 135 136 @staticmethod 137 def _build_call(host, method, **kwargs): 138 """Build a URL to |host| that calls |method|, passing |kwargs|. 139 140 Builds a URL that calls |method| on the dev server defined by |host|, 141 passing a set of key/value pairs built from the dict |kwargs|. 142 143 @param host: a string that is the host basename e.g. http://server:90. 144 @param method: the dev server method to call. 145 @param kwargs: a dict mapping arg names to arg values. 146 @return the URL string. 147 """ 148 argstr = '&'.join(map(lambda x: "%s=%s" % x, kwargs.iteritems())) 149 return "%(host)s/%(method)s?%(argstr)s" % dict( 150 host=host, method=method, argstr=argstr) 151 152 153 def build_call(self, method, **kwargs): 154 """Builds a devserver RPC string that can be invoked using urllib.open. 155 156 @param method: remote devserver method to call. 157 """ 158 return self._build_call(self._devserver, method, **kwargs) 159 160 161 @classmethod 162 def build_all_calls(cls, method, **kwargs): 163 """Builds a list of URLs that makes RPC calls on all devservers. 164 165 Build a URL that calls |method| on the dev server, passing a set 166 of key/value pairs built from the dict |kwargs|. 167 168 @param method: the dev server method to call. 169 @param kwargs: a dict mapping arg names to arg values 170 @return the URL string 171 """ 172 calls = [] 173 # Note we use cls.servers as servers is class specific. 174 for server in cls.servers(): 175 if cls.devserver_up(server): 176 calls.append(cls._build_call(server, method, **kwargs)) 177 178 return calls 179 180 181 @staticmethod 182 def servers(): 183 """Returns a list of servers that can serve as this type of server.""" 184 raise NotImplementedError() 185 186 187 @classmethod 188 def resolve(cls, build): 189 """"Resolves a build to a devserver instance. 190 191 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514). 192 """ 193 devservers = cls.servers() 194 while devservers: 195 hash_index = hash(build) % len(devservers) 196 devserver = devservers.pop(hash_index) 197 if cls.devserver_up(devserver): 198 return cls(devserver) 199 else: 200 logging.error('All devservers are currently down!!!') 201 raise DevServerException('All devservers are currently down!!!') 202 203 204class CrashServer(DevServer): 205 """Class of DevServer that symbolicates crash dumps.""" 206 @staticmethod 207 def servers(): 208 return _get_crash_server_list() 209 210 211 @remote_devserver_call() 212 def symbolicate_dump(self, minidump_path, build): 213 """Ask the devserver to symbolicate the dump at minidump_path. 214 215 Stage the debug symbols for |build| and, if that works, ask the 216 devserver to symbolicate the dump at |minidump_path|. 217 218 @param minidump_path: the on-disk path of the minidump. 219 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 220 whose debug symbols are needed for symbolication. 221 @return The contents of the stack trace 222 @raise DevServerException upon any return code that's not HTTP OK. 223 """ 224 try: 225 import requests 226 except ImportError: 227 logging.warning("Can't 'import requests' to connect to dev server.") 228 return '' 229 230 # Symbolicate minidump. 231 call = self.build_call('symbolicate_dump', 232 archive_url=_get_image_storage_server() + build) 233 request = requests.post( 234 call, files={'minidump': open(minidump_path, 'rb')}) 235 if request.status_code == requests.codes.OK: 236 return request.text 237 238 error_fd = cStringIO.StringIO(request.text) 239 raise urllib2.HTTPError( 240 call, request.status_code, request.text, request.headers, 241 error_fd) 242 243 244class ImageServer(DevServer): 245 """Class for DevServer that handles image-related RPCs.""" 246 @staticmethod 247 def servers(): 248 return _get_dev_server_list() 249 250 251 @classmethod 252 def devserver_url_for_servo(cls, board): 253 """Returns the devserver url for use with servo recovery. 254 255 @param board: The board (e.g. 'x86-mario'). 256 """ 257 # Ideally, for load balancing we'd select the server based 258 # on the board. For now, to simplify manual steps on the 259 # server side, we ignore the board type and hard-code the 260 # server as first in the list. 261 # 262 # TODO(jrbarnette) Once we have automated selection of the 263 # build for recovery, we should revisit this. 264 url_pattern = CONFIG.get_config_value('CROS', 265 'servo_url_pattern', 266 type=str) 267 return url_pattern % (cls.servers()[0], board) 268 269 270 class ArtifactUrls(object): 271 """A container for URLs of staged artifacts. 272 273 Attributes: 274 full_payload: URL for downloading a staged full release update 275 mton_payload: URL for downloading a staged M-to-N release update 276 nton_payload: URL for downloading a staged N-to-N release update 277 278 """ 279 def __init__(self, full_payload=None, mton_payload=None, 280 nton_payload=None): 281 self.full_payload = full_payload 282 self.mton_payload = mton_payload 283 self.nton_payload = nton_payload 284 285 286 @remote_devserver_call() 287 def stage_artifacts(self, image, artifacts): 288 """Tell the devserver to download and stage |artifacts| from |image|. 289 290 This is the main call point for staging any specific artifacts for a 291 given build. To see the list of artifacts one can stage see: 292 293 ~src/platfrom/dev/artifact_info.py. 294 295 This is maintained along with the actual devserver code. 296 297 @param image: the image to fetch and stage. 298 @param artifacts: A list of artifacts. 299 300 @raise DevServerException upon any return code that's not HTTP OK. 301 """ 302 call = self.build_call('stage', 303 archive_url=_get_image_storage_server() + image, 304 artifacts=','.join(artifacts)) 305 response = urllib2.urlopen(call) 306 if not response.read() == 'Success': 307 raise DevServerException("staging artifacts %s for %s failed;" 308 "HTTP OK not accompanied by 'Success'." % 309 (' '.join(artifacts), image)) 310 311 312 @remote_devserver_call() 313 def trigger_download(self, image, synchronous=True): 314 """Tell the devserver to download and stage |image|. 315 316 Tells the devserver to fetch |image| from the image storage server 317 named by _get_image_storage_server(). 318 319 If |synchronous| is True, waits for the entire download to finish 320 staging before returning. Otherwise only the artifacts necessary 321 to start installing images onto DUT's will be staged before returning. 322 A caller can then call finish_download to guarantee the rest of the 323 artifacts have finished staging. 324 325 @param image: the image to fetch and stage. 326 @param synchronous: if True, waits until all components of the image are 327 staged before returning. 328 329 @raise DevServerException upon any return code that's not HTTP OK. 330 331 """ 332 call = self.build_call( 333 'download', archive_url=_get_image_storage_server() + image) 334 response = urllib2.urlopen(call) 335 was_successful = response.read() == 'Success' 336 if was_successful and synchronous: 337 self.finish_download(image) 338 elif not was_successful: 339 raise DevServerException("trigger_download for %s failed;" 340 "HTTP OK not accompanied by 'Success'." % 341 image) 342 343 344 @remote_devserver_call() 345 def setup_telemetry(self, build): 346 """Tell the devserver to setup telemetry for this build. 347 348 The devserver will stage autotest and then extract the required files 349 for telemetry. 350 351 @param build: the build to setup telemetry for. 352 353 @returns path on the devserver that telemetry is installed to. 354 """ 355 call = self.build_call( 356 'setup_telemetry', 357 archive_url=_get_image_storage_server() + build) 358 response = urllib2.urlopen(call) 359 return response.read() 360 361 362 @remote_devserver_call() 363 def finish_download(self, image): 364 """Tell the devserver to finish staging |image|. 365 366 If trigger_download is called with synchronous=False, it will return 367 before all artifacts have been staged. This method contacts the 368 devserver and blocks until all staging is completed and should be 369 called after a call to trigger_download. 370 371 @param image: the image to fetch and stage. 372 @raise DevServerException upon any return code that's not HTTP OK. 373 """ 374 call = self.build_call('wait_for_status', 375 archive_url=_get_image_storage_server() + image) 376 if urllib2.urlopen(call).read() != 'Success': 377 raise DevServerException("finish_download for %s failed;" 378 "HTTP OK not accompanied by 'Success'." % 379 image) 380 381 def _get_image_url(self, image): 382 """Returns the url of the directory for this image on the devserver. 383 384 @param image: the image that was fetched. 385 """ 386 url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', 387 type=str) 388 return (url_pattern % (self.url(), image)).replace( 389 'update', 'static/archive') 390 391 392 def get_delta_payload_url(self, payload_type, image): 393 """Returns a URL to a staged delta payload. 394 395 @param payload_type: either 'mton' or 'nton' 396 @param image: the image that was fetched. 397 398 @return A fully qualified URL that can be used for downloading the 399 payload. 400 401 @raise DevServerException if payload type argument is invalid. 402 403 """ 404 if payload_type not in ('mton', 'nton'): 405 raise DevServerException('invalid delta payload type: %s' % 406 payload_type) 407 version = os.path.basename(image) 408 base_url = self._get_image_url(image) 409 return base_url + '/%s_%s' % (version, payload_type) 410 411 412 def get_full_payload_url(self, image): 413 """Returns a URL to a staged full payload. 414 415 @param image: the image that was fetched. 416 417 @return A fully qualified URL that can be used for downloading the 418 payload. 419 420 """ 421 return self._get_image_url(image) + '/update.gz' 422 423 424 def get_test_image_url(self, image): 425 """Returns a URL to a staged test image. 426 427 @param image: the image that was fetched. 428 429 @return A fully qualified URL that can be used for downloading the 430 image. 431 432 """ 433 return self._get_image_url(image) + '/chromiumos_test_image.bin' 434 435 436 @remote_devserver_call() 437 def list_control_files(self, build): 438 """Ask the devserver to list all control files for |build|. 439 440 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 441 whose control files the caller wants listed. 442 @return None on failure, or a list of control file paths 443 (e.g. server/site_tests/autoupdate/control) 444 @raise DevServerException upon any return code that's not HTTP OK. 445 """ 446 call = self.build_call('controlfiles', build=build) 447 response = urllib2.urlopen(call) 448 return [line.rstrip() for line in response] 449 450 451 @remote_devserver_call() 452 def get_control_file(self, build, control_path): 453 """Ask the devserver for the contents of a control file. 454 455 @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) 456 whose control file the caller wants to fetch. 457 @param control_path: The file to fetch 458 (e.g. server/site_tests/autoupdate/control) 459 @return The contents of the desired file. 460 @raise DevServerException upon any return code that's not HTTP OK. 461 """ 462 call = self.build_call('controlfiles', build=build, 463 control_path=control_path) 464 return urllib2.urlopen(call).read() 465 466 467 @remote_devserver_call() 468 def get_dependencies_file(self, build): 469 """Ask the dev server for the contents of the suite dependencies file. 470 471 Ask the dev server at |self._dev_server| for the contents of the 472 pre-processed suite dependencies file (at DEPENDENCIES_FILE) 473 for |build|. 474 475 @param build: The build (e.g. x86-mario-release/R21-2333.0.0) 476 whose dependencies the caller is interested in. 477 @return The contents of the dependencies file, which should eval to 478 a dict of dicts, as per site_utils/suite_preprocessor.py. 479 @raise DevServerException upon any return code that's not HTTP OK. 480 """ 481 call = self.build_call('controlfiles', 482 build=build, control_path=DEPENDENCIES_FILE) 483 return urllib2.urlopen(call).read() 484 485 486 @classmethod 487 @remote_devserver_call() 488 def get_latest_build(cls, target, milestone=''): 489 """Ask all the devservers for the latest build for a given target. 490 491 @param target: The build target, typically a combination of the board 492 and the type of build e.g. x86-mario-release. 493 @param milestone: For latest build set to '', for builds only in a 494 specific milestone set to a str of format Rxx 495 (e.g. R16). Default: ''. Since we are dealing with a 496 webserver sending an empty string, '', ensures that 497 the variable in the URL is ignored as if it was set 498 to None. 499 @return A string of the returned build e.g. R20-2226.0.0. 500 @raise DevServerException upon any return code that's not HTTP OK. 501 """ 502 calls = cls.build_all_calls('latestbuild', target=target, 503 milestone=milestone) 504 latest_builds = [] 505 for call in calls: 506 latest_builds.append(urllib2.urlopen(call).read()) 507 508 return max(latest_builds, key=version.LooseVersion) 509