site_utils.py revision 5b35656e62acce8fd0fac341592670ad0145c8a9
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import inspect 6import logging 7import os 8import re 9import signal 10import socket 11import time 12import urllib2 13import uuid 14 15from autotest_lib.client.common_lib import base_utils 16from autotest_lib.client.common_lib import error 17from autotest_lib.client.common_lib import global_config 18from autotest_lib.client.common_lib import lsbrelease_utils 19from autotest_lib.client.cros import constants 20 21 22# Keep checking if the pid is alive every second until the timeout (in seconds) 23CHECK_PID_IS_ALIVE_TIMEOUT = 6 24 25_LOCAL_HOST_LIST = ('localhost', '127.0.0.1') 26 27# The default address of a vm gateway. 28DEFAULT_VM_GATEWAY = '10.0.2.2' 29 30# Google Storage bucket URI to store results in. 31DEFAULT_OFFLOAD_GSURI = global_config.global_config.get_config_value( 32 'CROS', 'results_storage_server', default=None) 33 34# Default Moblab Ethernet Interface. 35MOBLAB_ETH = 'eth0' 36 37def ping(host, deadline=None, tries=None, timeout=60): 38 """Attempt to ping |host|. 39 40 Shell out to 'ping' to try to reach |host| for |timeout| seconds. 41 Returns exit code of ping. 42 43 Per 'man ping', if you specify BOTH |deadline| and |tries|, ping only 44 returns 0 if we get responses to |tries| pings within |deadline| seconds. 45 46 Specifying |deadline| or |count| alone should return 0 as long as 47 some packets receive responses. 48 49 @param host: the host to ping. 50 @param deadline: seconds within which |tries| pings must succeed. 51 @param tries: number of pings to send. 52 @param timeout: number of seconds after which to kill 'ping' command. 53 @return exit code of ping command. 54 """ 55 args = [host] 56 if deadline: 57 args.append('-w%d' % deadline) 58 if tries: 59 args.append('-c%d' % tries) 60 return base_utils.run('ping', args=args, 61 ignore_status=True, timeout=timeout, 62 stdout_tee=base_utils.TEE_TO_LOGS, 63 stderr_tee=base_utils.TEE_TO_LOGS).exit_status 64 65 66def host_is_in_lab_zone(hostname): 67 """Check if the host is in the CLIENT.dns_zone. 68 69 @param hostname: The hostname to check. 70 @returns True if hostname.dns_zone resolves, otherwise False. 71 """ 72 host_parts = hostname.split('.') 73 dns_zone = global_config.global_config.get_config_value('CLIENT', 'dns_zone', 74 default=None) 75 fqdn = '%s.%s' % (host_parts[0], dns_zone) 76 try: 77 socket.gethostbyname(fqdn) 78 return True 79 except socket.gaierror: 80 return False 81 82 83def host_could_be_in_afe(hostname): 84 """Check if the host could be in Autotest Front End. 85 86 Report whether or not a host could be in AFE, without actually 87 consulting AFE. This method exists because some systems are in the 88 lab zone, but not actually managed by AFE. 89 90 @param hostname: The hostname to check. 91 @returns True if hostname is in lab zone, and does not match *-dev-* 92 """ 93 # Do the 'dev' check first, so that we skip DNS lookup if the 94 # hostname matches. This should give us greater resilience to lab 95 # failures. 96 return (hostname.find('-dev-') == -1) and host_is_in_lab_zone(hostname) 97 98 99def get_chrome_version(job_views): 100 """ 101 Retrieves the version of the chrome binary associated with a job. 102 103 When a test runs we query the chrome binary for it's version and drop 104 that value into a client keyval. To retrieve the chrome version we get all 105 the views associated with a test from the db, including those of the 106 server and client jobs, and parse the version out of the first test view 107 that has it. If we never ran a single test in the suite the job_views 108 dictionary will not contain a chrome version. 109 110 This method cannot retrieve the chrome version from a dictionary that 111 does not conform to the structure of an autotest tko view. 112 113 @param job_views: a list of a job's result views, as returned by 114 the get_detailed_test_views method in rpc_interface. 115 @return: The chrome version string, or None if one can't be found. 116 """ 117 118 # Aborted jobs have no views. 119 if not job_views: 120 return None 121 122 for view in job_views: 123 if (view.get('attributes') 124 and constants.CHROME_VERSION in view['attributes'].keys()): 125 126 return view['attributes'].get(constants.CHROME_VERSION) 127 128 logging.warning('Could not find chrome version for failure.') 129 return None 130 131 132def get_interface_mac_address(interface): 133 """Return the MAC address of a given interface. 134 135 @param interface: Interface to look up the MAC address of. 136 """ 137 interface_link = base_utils.run( 138 'ip addr show %s | grep link/ether' % interface).stdout 139 # The output will be in the format of: 140 # 'link/ether <mac> brd ff:ff:ff:ff:ff:ff' 141 return interface_link.split()[1] 142 143 144def get_offload_gsuri(): 145 """Return the GSURI to offload test results to. 146 147 For the normal use case this is the results_storage_server in the 148 global_config. 149 150 However partners using Moblab will be offloading their results to a 151 subdirectory of their image storage buckets. The subdirectory is 152 determined by the MAC Address of the Moblab device. 153 154 @returns gsuri to offload test results to. 155 """ 156 if not lsbrelease_utils.is_moblab(): 157 return DEFAULT_OFFLOAD_GSURI 158 moblab_id_filepath = '/home/moblab/.moblab_id' 159 if os.path.exists(moblab_id_filepath): 160 with open(moblab_id_filepath, 'r') as moblab_id_file: 161 random_id = moblab_id_file.read() 162 else: 163 random_id = uuid.uuid1() 164 with open(moblab_id_filepath, 'w') as moblab_id_file: 165 moblab_id_file.write('%s' % random_id) 166 return '%sresults/%s/%s/' % ( 167 global_config.global_config.get_config_value( 168 'CROS', 'image_storage_server'), 169 get_interface_mac_address(MOBLAB_ETH), random_id) 170 171 172# TODO(petermayo): crosbug.com/31826 Share this with _GsUpload in 173# //chromite.git/buildbot/prebuilt.py somewhere/somehow 174def gs_upload(local_file, remote_file, acl, result_dir=None, 175 transfer_timeout=300, acl_timeout=300): 176 """Upload to GS bucket. 177 178 @param local_file: Local file to upload 179 @param remote_file: Remote location to upload the local_file to. 180 @param acl: name or file used for controlling access to the uploaded 181 file. 182 @param result_dir: Result directory if you want to add tracing to the 183 upload. 184 @param transfer_timeout: Timeout for this upload call. 185 @param acl_timeout: Timeout for the acl call needed to confirm that 186 the uploader has permissions to execute the upload. 187 188 @raise CmdError: the exit code of the gsutil call was not 0. 189 190 @returns True/False - depending on if the upload succeeded or failed. 191 """ 192 # https://developers.google.com/storage/docs/accesscontrol#extension 193 CANNED_ACLS = ['project-private', 'private', 'public-read', 194 'public-read-write', 'authenticated-read', 195 'bucket-owner-read', 'bucket-owner-full-control'] 196 _GSUTIL_BIN = 'gsutil' 197 acl_cmd = None 198 if acl in CANNED_ACLS: 199 cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file) 200 else: 201 # For private uploads we assume that the overlay board is set up 202 # properly and a googlestore_acl.xml is present, if not this script 203 # errors 204 cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file) 205 if not os.path.exists(acl): 206 logging.error('Unable to find ACL File %s.', acl) 207 return False 208 acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file) 209 if not result_dir: 210 base_utils.run(cmd, timeout=transfer_timeout, verbose=True) 211 if acl_cmd: 212 base_utils.run(acl_cmd, timeout=acl_timeout, verbose=True) 213 return True 214 with open(os.path.join(result_dir, 'tracing'), 'w') as ftrace: 215 ftrace.write('Preamble\n') 216 base_utils.run(cmd, timeout=transfer_timeout, verbose=True, 217 stdout_tee=ftrace, stderr_tee=ftrace) 218 if acl_cmd: 219 ftrace.write('\nACL setting\n') 220 # Apply the passed in ACL xml file to the uploaded object. 221 base_utils.run(acl_cmd, timeout=acl_timeout, verbose=True, 222 stdout_tee=ftrace, stderr_tee=ftrace) 223 ftrace.write('Postamble\n') 224 return True 225 226 227def gs_ls(uri_pattern): 228 """Returns a list of URIs that match a given pattern. 229 230 @param uri_pattern: a GS URI pattern, may contain wildcards 231 232 @return A list of URIs matching the given pattern. 233 234 @raise CmdError: the gsutil command failed. 235 236 """ 237 gs_cmd = ' '.join(['gsutil', 'ls', uri_pattern]) 238 result = base_utils.system_output(gs_cmd).splitlines() 239 return [path.rstrip() for path in result if path] 240 241 242def nuke_pids(pid_list, signal_queue=[signal.SIGTERM, signal.SIGKILL]): 243 """ 244 Given a list of pid's, kill them via an esclating series of signals. 245 246 @param pid_list: List of PID's to kill. 247 @param signal_queue: Queue of signals to send the PID's to terminate them. 248 249 @return: A mapping of the signal name to the number of processes it 250 was sent to. 251 """ 252 sig_count = {} 253 # Though this is slightly hacky it beats hardcoding names anyday. 254 sig_names = dict((k, v) for v, k in signal.__dict__.iteritems() 255 if v.startswith('SIG')) 256 for sig in signal_queue: 257 logging.debug('Sending signal %s to the following pids:', sig) 258 sig_count[sig_names.get(sig, 'unknown_signal')] = len(pid_list) 259 for pid in pid_list: 260 logging.debug('Pid %d', pid) 261 try: 262 os.kill(pid, sig) 263 except OSError: 264 # The process may have died from a previous signal before we 265 # could kill it. 266 pass 267 if sig == signal.SIGKILL: 268 return sig_count 269 pid_list = [pid for pid in pid_list if base_utils.pid_is_alive(pid)] 270 if not pid_list: 271 break 272 time.sleep(CHECK_PID_IS_ALIVE_TIMEOUT) 273 failed_list = [] 274 for pid in pid_list: 275 if base_utils.pid_is_alive(pid): 276 failed_list.append('Could not kill %d for process name: %s.' % pid, 277 base_utils.get_process_name(pid)) 278 if failed_list: 279 raise error.AutoservRunError('Following errors occured: %s' % 280 failed_list, None) 281 return sig_count 282 283 284def externalize_host(host): 285 """Returns an externally accessible host name. 286 287 @param host: a host name or address (string) 288 289 @return An externally visible host name or address 290 291 """ 292 return socket.gethostname() if host in _LOCAL_HOST_LIST else host 293 294 295def urlopen_socket_timeout(url, data=None, timeout=5): 296 """ 297 Wrapper to urllib2.urlopen with a socket timeout. 298 299 This method will convert all socket timeouts to 300 TimeoutExceptions, so we can use it in conjunction 301 with the rpc retry decorator and continue to handle 302 other URLErrors as we see fit. 303 304 @param url: The url to open. 305 @param data: The data to send to the url (eg: the urlencoded dictionary 306 used with a POST call). 307 @param timeout: The timeout for this urlopen call. 308 309 @return: The response of the urlopen call. 310 311 @raises: error.TimeoutException when a socket timeout occurs. 312 urllib2.URLError for errors that not caused by timeout. 313 urllib2.HTTPError for errors like 404 url not found. 314 """ 315 old_timeout = socket.getdefaulttimeout() 316 socket.setdefaulttimeout(timeout) 317 try: 318 return urllib2.urlopen(url, data=data) 319 except urllib2.URLError as e: 320 if type(e.reason) is socket.timeout: 321 raise error.TimeoutException(str(e)) 322 raise 323 finally: 324 socket.setdefaulttimeout(old_timeout) 325 326 327def parse_chrome_version(version_string): 328 """ 329 Parse a chrome version string and return version and milestone. 330 331 Given a chrome version of the form "W.X.Y.Z", return "W.X.Y.Z" as 332 the version and "W" as the milestone. 333 334 @param version_string: Chrome version string. 335 @return: a tuple (chrome_version, milestone). If the incoming version 336 string is not of the form "W.X.Y.Z", chrome_version will 337 be set to the incoming "version_string" argument and the 338 milestone will be set to the empty string. 339 """ 340 match = re.search('(\d+)\.\d+\.\d+\.\d+', version_string) 341 ver = match.group(0) if match else version_string 342 milestone = match.group(1) if match else '' 343 return ver, milestone 344 345 346def is_localhost(server): 347 """Check if server is equivalent to localhost. 348 349 @param server: Name of the server to check. 350 351 @return: True if given server is equivalent to localhost. 352 353 @raise socket.gaierror: If server name failed to be resolved. 354 """ 355 if server in _LOCAL_HOST_LIST: 356 return True 357 try: 358 return (socket.gethostbyname(socket.gethostname()) == 359 socket.gethostbyname(server)) 360 except socket.gaierror: 361 logging.error('Failed to resolve server name %s.', server) 362 return False 363 364 365def is_puppylab_vm(server): 366 """Check if server is a virtual machine in puppylab. 367 368 In the virtual machine testing environment (i.e., puppylab), each 369 shard VM has a hostname like localhost:<port>. 370 371 @param server: Server name to check. 372 373 @return True if given server is a virtual machine in puppylab. 374 375 """ 376 # TODO(mkryu): This is a puppylab specific hack. Please update 377 # this method if you have a better solution. 378 regex = re.compile(r'(.+):\d+') 379 m = regex.match(server) 380 if m: 381 return m.group(1) in _LOCAL_HOST_LIST 382 return False 383 384 385def get_function_arg_value(func, arg_name, args, kwargs): 386 """Get the value of the given argument for the function. 387 388 @param func: Function being called with given arguments. 389 @param arg_name: Name of the argument to look for value. 390 @param args: arguments for function to be called. 391 @param kwargs: keyword arguments for function to be called. 392 393 @return: The value of the given argument for the function. 394 395 @raise ValueError: If the argument is not listed function arguemnts. 396 @raise KeyError: If no value is found for the given argument. 397 """ 398 if arg_name in kwargs: 399 return kwargs[arg_name] 400 401 argspec = inspect.getargspec(func) 402 index = argspec.args.index(arg_name) 403 try: 404 return args[index] 405 except IndexError: 406 try: 407 # The argument can use a default value. Reverse the default value 408 # so argument with default value can be counted from the last to 409 # the first. 410 return argspec.defaults[::-1][len(argspec.args) - index - 1] 411 except IndexError: 412 raise KeyError('Argument %s is not given a value. argspec: %s, ' 413 'args:%s, kwargs:%s' % 414 (arg_name, argspec, args, kwargs)) 415 416 417def version_match(build_version, release_version, update_url=''): 418 """Compare release versino from lsb-release with cros-version label. 419 420 build_version is a string based on build name. It is prefixed with builder 421 info and branch ID, e.g., lumpy-release/R43-6809.0.0. It may not include 422 builder info, e.g., lumpy-release, in which case, update_url shall be passed 423 in to determine if the build is a trybot or pgo-generate build. 424 release_version is retrieved from lsb-release. 425 These two values might not match exactly. 426 427 The method is designed to compare version for following 6 scenarios with 428 samples of build version and expected release version: 429 1. trybot paladin build. 430 build version: trybot-lumpy-paladin/R27-3837.0.0-b123 431 release version: 3837.0.2013_03_21_1340 432 433 2. trybot release build. 434 build version: trybot-lumpy-release/R27-3837.0.0-b456 435 release version: 3837.0.0 436 437 3. buildbot official release build. 438 build version: lumpy-release/R27-3837.0.0 439 release version: 3837.0.0 440 441 4. non-official paladin rc build. 442 build version: lumpy-paladin/R27-3878.0.0-rc7 443 release version: 3837.0.0-rc7 444 445 5. chrome-perf build. 446 build version: lumpy-chrome-perf/R28-3837.0.0-b2996 447 release version: 3837.0.0 448 449 6. pgo-generate build. 450 build version: lumpy-release-pgo-generate/R28-3837.0.0-b2996 451 release version: 3837.0.0-pgo-generate 452 453 TODO: This logic has a bug if a trybot paladin build failed to be 454 installed in a DUT running an older trybot paladin build with same 455 platform number, but different build number (-b###). So to conclusively 456 determine if a tryjob paladin build is imaged successfully, we may need 457 to find out the date string from update url. 458 459 @param build_version: Build name for cros version, e.g. 460 peppy-release/R43-6809.0.0 or R43-6809.0.0 461 @param release_version: Release version retrieved from lsb-release, 462 e.g., 6809.0.0 463 @param update_url: Update url which include the full builder information. 464 Default is set to empty string. 465 466 @return: True if the values match, otherwise returns False. 467 """ 468 # If the build is from release, CQ or PFQ builder, cros-version label must 469 # be ended with release version in lsb-release. 470 if build_version.endswith(release_version): 471 return True 472 473 # Remove R#- and -b# at the end of build version 474 stripped_version = re.sub(r'(R\d+-|-b\d+)', '', build_version) 475 # Trim the builder info, e.g., trybot-lumpy-paladin/ 476 stripped_version = stripped_version.split('/')[-1] 477 478 is_trybot_paladin_build = ( 479 re.match(r'.*trybot-.+-paladin', build_version) or 480 re.match(r'.*trybot-.+-paladin', update_url)) 481 482 # Replace date string with 0 in release_version 483 release_version_no_date = re.sub(r'\d{4}_\d{2}_\d{2}_\d+', '0', 484 release_version) 485 has_date_string = release_version != release_version_no_date 486 487 is_pgo_generate_build = ( 488 re.match(r'.+-pgo-generate', build_version) or 489 re.match(r'.+-pgo-generate', update_url)) 490 491 # Remove |-pgo-generate| in release_version 492 release_version_no_pgo = release_version.replace('-pgo-generate', '') 493 has_pgo_generate = release_version != release_version_no_pgo 494 495 if is_trybot_paladin_build: 496 if not has_date_string: 497 logging.error('A trybot paladin build is expected. Version ' 498 '"%s" is not a paladin build.', release_version) 499 return False 500 return stripped_version == release_version_no_date 501 elif is_pgo_generate_build: 502 if not has_pgo_generate: 503 logging.error('A pgo-generate build is expected. Version ' 504 '"%s" is not a pgo-generate build.', 505 release_version) 506 return False 507 return stripped_version == release_version_no_pgo 508 else: 509 if has_date_string: 510 logging.error('Unexpected date found in a non trybot paladin ' 511 'build.') 512 return False 513 # Versioned build, i.e., rc or release build. 514 return stripped_version == release_version 515 516 517def get_real_user(): 518 """Get the real user that runs the script. 519 520 The function check environment variable SUDO_USER for the user if the 521 script is run with sudo. Otherwise, it returns the value of environment 522 variable USER. 523 524 @return: The user name that runs the script. 525 526 """ 527 user = os.environ.get('SUDO_USER') 528 if not user: 529 user = os.environ.get('USER') 530 return user 531 532 533def sudo_require_password(): 534 """Test if the process can run sudo command without using password. 535 536 @return: True if the process needs password to run sudo command. 537 538 """ 539 try: 540 base_utils.run('sudo -n true') 541 return False 542 except error.CmdError: 543 logging.warn('sudo command requires password.') 544 return True 545 546 547def is_in_container(): 548 """Check if the process is running inside a container. 549 550 @return: True if the process is running inside a container, otherwise False. 551 """ 552 result = base_utils.run('grep -q "/lxc/" /proc/1/cgroup', 553 verbose=False, ignore_status=True) 554 return result.exit_status == 0 555 556 557def is_flash_installed(): 558 """ 559 The Adobe Flash binary is only distributed with internal builds. 560 """ 561 return (os.path.exists('/opt/google/chrome/pepper/libpepflashplayer.so') 562 and os.path.exists('/opt/google/chrome/pepper/pepper-flash.info')) 563 564 565def verify_flash_installed(): 566 """ 567 The Adobe Flash binary is only distributed with internal builds. 568 Warn users of public builds of the extra dependency. 569 """ 570 if not is_flash_installed(): 571 raise error.TestNAError('No Adobe Flash binary installed.') 572