site_utils.py revision cf731e31a5a63f1d795286330afb87038093f795
1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import glob 6import logging 7import os 8import re 9import signal 10import socket 11import sys 12import time 13import urllib2 14 15from autotest_lib.client.common_lib import base_utils, error, global_config 16from autotest_lib.client.cros import constants 17 18 19# Keep checking if the pid is alive every second until the timeout (in seconds) 20CHECK_PID_IS_ALIVE_TIMEOUT = 6 21 22_LOCAL_HOST_LIST = ('localhost', '127.0.0.1') 23 24 25def ping(host, deadline=None, tries=None, timeout=60): 26 """Attempt to ping |host|. 27 28 Shell out to 'ping' to try to reach |host| for |timeout| seconds. 29 Returns exit code of ping. 30 31 Per 'man ping', if you specify BOTH |deadline| and |tries|, ping only 32 returns 0 if we get responses to |tries| pings within |deadline| seconds. 33 34 Specifying |deadline| or |count| alone should return 0 as long as 35 some packets receive responses. 36 37 @param host: the host to ping. 38 @param deadline: seconds within which |tries| pings must succeed. 39 @param tries: number of pings to send. 40 @param timeout: number of seconds after which to kill 'ping' command. 41 @return exit code of ping command. 42 """ 43 args = [host] 44 if deadline: 45 args.append('-w%d' % deadline) 46 if tries: 47 args.append('-c%d' % tries) 48 return base_utils.run('ping', args=args, 49 ignore_status=True, timeout=timeout, 50 stdout_tee=base_utils.TEE_TO_LOGS, 51 stderr_tee=base_utils.TEE_TO_LOGS).exit_status 52 53 54def host_is_in_lab_zone(hostname): 55 """Check if the host is in the CROS.dns_zone. 56 57 @param hostname: The hostname to check. 58 @returns True if hostname.dns_zone resolves, otherwise False. 59 """ 60 host_parts = hostname.split('.') 61 dns_zone = global_config.global_config.get_config_value('CROS', 'dns_zone', 62 default=None) 63 fqdn = '%s.%s' % (host_parts[0], dns_zone) 64 try: 65 socket.gethostbyname(fqdn) 66 return True 67 except socket.gaierror: 68 return False 69 70 71def get_chrome_version(job_views): 72 """ 73 Retrieves the version of the chrome binary associated with a job. 74 75 When a test runs we query the chrome binary for it's version and drop 76 that value into a client keyval. To retrieve the chrome version we get all 77 the views associated with a test from the db, including those of the 78 server and client jobs, and parse the version out of the first test view 79 that has it. If we never ran a single test in the suite the job_views 80 dictionary will not contain a chrome version. 81 82 This method cannot retrieve the chrome version from a dictionary that 83 does not conform to the structure of an autotest tko view. 84 85 @param job_views: a list of a job's result views, as returned by 86 the get_detailed_test_views method in rpc_interface. 87 @return: The chrome version string, or None if one can't be found. 88 """ 89 90 # Aborted jobs have no views. 91 if not job_views: 92 return None 93 94 for view in job_views: 95 if (view.get('attributes') 96 and constants.CHROME_VERSION in view['attributes'].keys()): 97 98 return view['attributes'].get(constants.CHROME_VERSION) 99 100 logging.warning('Could not find chrome version for failure.') 101 return None 102 103 104def _lsbrelease_search(regex, group_id=0): 105 """Searches /etc/lsb-release for a regex match. 106 107 @param regex: Regex to match. 108 @param group_id: The group in the regex we are searching for. 109 Default is group 0. 110 111 @returns the string in the specified group if there is a match or None if 112 not found. 113 114 @raises IOError if /etc/lsb-release can not be accessed. 115 """ 116 with open(constants.LSB_RELEASE) as lsb_release_file: 117 for line in lsb_release_file: 118 m = re.match(regex, line) 119 if m: 120 return m.group(group_id) 121 return None 122 123 124def get_current_board(): 125 """Return the current board name. 126 127 @return current board name, e.g "lumpy", None on fail. 128 """ 129 return _lsbrelease_search(r'^CHROMEOS_RELEASE_BOARD=(.+)$', group_id=1) 130 131 132def get_chromeos_release_version(): 133 """ 134 @return chromeos version in device under test as string. None on fail. 135 """ 136 return _lsbrelease_search(r'^CHROMEOS_RELEASE_VERSION=(.+)$', group_id=1) 137 138 139def is_moblab(): 140 """Return if we are running on a Moblab system or not. 141 142 @return the board string if this is a Moblab device or None if it is not. 143 """ 144 try: 145 return _lsbrelease_search(r'.*moblab') 146 except IOError as e: 147 logging.error('Unable to determine if this is a moblab system: %s', e) 148 149# TODO(petermayo): crosbug.com/31826 Share this with _GsUpload in 150# //chromite.git/buildbot/prebuilt.py somewhere/somehow 151def gs_upload(local_file, remote_file, acl, result_dir=None, 152 transfer_timeout=300, acl_timeout=300): 153 """Upload to GS bucket. 154 155 @param local_file: Local file to upload 156 @param remote_file: Remote location to upload the local_file to. 157 @param acl: name or file used for controlling access to the uploaded 158 file. 159 @param result_dir: Result directory if you want to add tracing to the 160 upload. 161 @param transfer_timeout: Timeout for this upload call. 162 @param acl_timeout: Timeout for the acl call needed to confirm that 163 the uploader has permissions to execute the upload. 164 165 @raise CmdError: the exit code of the gsutil call was not 0. 166 167 @returns True/False - depending on if the upload succeeded or failed. 168 """ 169 # https://developers.google.com/storage/docs/accesscontrol#extension 170 CANNED_ACLS = ['project-private', 'private', 'public-read', 171 'public-read-write', 'authenticated-read', 172 'bucket-owner-read', 'bucket-owner-full-control'] 173 _GSUTIL_BIN = 'gsutil' 174 acl_cmd = None 175 if acl in CANNED_ACLS: 176 cmd = '%s cp -a %s %s %s' % (_GSUTIL_BIN, acl, local_file, remote_file) 177 else: 178 # For private uploads we assume that the overlay board is set up 179 # properly and a googlestore_acl.xml is present, if not this script 180 # errors 181 cmd = '%s cp -a private %s %s' % (_GSUTIL_BIN, local_file, remote_file) 182 if not os.path.exists(acl): 183 logging.error('Unable to find ACL File %s.', acl) 184 return False 185 acl_cmd = '%s setacl %s %s' % (_GSUTIL_BIN, acl, remote_file) 186 if not result_dir: 187 base_utils.run(cmd, timeout=transfer_timeout, verbose=True) 188 if acl_cmd: 189 base_utils.run(acl_cmd, timeout=acl_timeout, verbose=True) 190 return True 191 with open(os.path.join(result_dir, 'tracing'), 'w') as ftrace: 192 ftrace.write('Preamble\n') 193 base_utils.run(cmd, timeout=transfer_timeout, verbose=True, 194 stdout_tee=ftrace, stderr_tee=ftrace) 195 if acl_cmd: 196 ftrace.write('\nACL setting\n') 197 # Apply the passed in ACL xml file to the uploaded object. 198 base_utils.run(acl_cmd, timeout=acl_timeout, verbose=True, 199 stdout_tee=ftrace, stderr_tee=ftrace) 200 ftrace.write('Postamble\n') 201 return True 202 203 204def gs_ls(uri_pattern): 205 """Returns a list of URIs that match a given pattern. 206 207 @param uri_pattern: a GS URI pattern, may contain wildcards 208 209 @return A list of URIs matching the given pattern. 210 211 @raise CmdError: the gsutil command failed. 212 213 """ 214 gs_cmd = ' '.join(['gsutil', 'ls', uri_pattern]) 215 result = base_utils.system_output(gs_cmd).splitlines() 216 return [path.rstrip() for path in result if path] 217 218 219def nuke_pids(pid_list, signal_queue=[signal.SIGTERM, signal.SIGKILL]): 220 """ 221 Given a list of pid's, kill them via an esclating series of signals. 222 223 @param pid_list: List of PID's to kill. 224 @param signal_queue: Queue of signals to send the PID's to terminate them. 225 226 @return: A mapping of the signal name to the number of processes it 227 was sent to. 228 """ 229 sig_count = {} 230 # Though this is slightly hacky it beats hardcoding names anyday. 231 sig_names = dict((k, v) for v, k in signal.__dict__.iteritems() 232 if v.startswith('SIG')) 233 for sig in signal_queue: 234 logging.debug('Sending signal %s to the following pids:', sig) 235 sig_count[sig_names.get(sig, 'unknown_signal')] = len(pid_list) 236 for pid in pid_list: 237 logging.debug('Pid %d', pid) 238 try: 239 os.kill(pid, sig) 240 except OSError: 241 # The process may have died from a previous signal before we 242 # could kill it. 243 pass 244 pid_list = [pid for pid in pid_list if base_utils.pid_is_alive(pid)] 245 if not pid_list: 246 break 247 time.sleep(CHECK_PID_IS_ALIVE_TIMEOUT) 248 failed_list = [] 249 if signal.SIGKILL in signal_queue: 250 return sig_count 251 for pid in pid_list: 252 if base_utils.pid_is_alive(pid): 253 failed_list.append('Could not kill %d for process name: %s.' % pid, 254 base_utils.get_process_name(pid)) 255 if failed_list: 256 raise error.AutoservRunError('Following errors occured: %s' % 257 failed_list, None) 258 return sig_count 259 260 261def externalize_host(host): 262 """Returns an externally accessible host name. 263 264 @param host: a host name or address (string) 265 266 @return An externally visible host name or address 267 268 """ 269 return socket.gethostname() if host in _LOCAL_HOST_LIST else host 270 271 272def urlopen_socket_timeout(url, data=None, timeout=5): 273 """ 274 Wrapper to urllib2.urlopen with a socket timeout. 275 276 This method will convert all socket timeouts to 277 TimeoutExceptions, so we can use it in conjunction 278 with the rpc retry decorator and continue to handle 279 other URLErrors as we see fit. 280 281 @param url: The url to open. 282 @param data: The data to send to the url (eg: the urlencoded dictionary 283 used with a POST call). 284 @param timeout: The timeout for this urlopen call. 285 286 @return: The response of the urlopen call. 287 288 @raises: error.TimeoutException when a socket timeout occurs. 289 urllib2.URLError for errors that not caused by timeout. 290 urllib2.HTTPError for errors like 404 url not found. 291 """ 292 old_timeout = socket.getdefaulttimeout() 293 socket.setdefaulttimeout(timeout) 294 try: 295 return urllib2.urlopen(url, data=data) 296 except urllib2.URLError as e: 297 if type(e.reason) is socket.timeout: 298 raise error.TimeoutException(str(e)) 299 raise 300 finally: 301 socket.setdefaulttimeout(old_timeout) 302 303 304def parse_chrome_version(version_string): 305 """ 306 Parse a chrome version string and return version and milestone. 307 308 Given a chrome version of the form "W.X.Y.Z", return "W.X.Y.Z" as 309 the version and "W" as the milestone. 310 311 @param version_string: Chrome version string. 312 @return: a tuple (chrome_version, milestone). If the incoming version 313 string is not of the form "W.X.Y.Z", chrome_version will 314 be set to the incoming "version_string" argument and the 315 milestone will be set to the empty string. 316 """ 317 match = re.search('(\d+)\.\d+\.\d+\.\d+', version_string) 318 ver = match.group(0) if match else version_string 319 milestone = match.group(1) if match else '' 320 return ver, milestone 321 322 323def take_screenshot(dest_dir, fname_prefix, format='png'): 324 """ 325 Take screenshot and save to a new file in the dest_dir. 326 327 @param dest_dir: path, destination directory to save the screenshot. 328 @param fname_prefix: string, prefix for output filename. 329 @param format: string, file format ('png', 'jpg', etc) to use. 330 331 @returns complete path to saved screenshot file. 332 333 """ 334 if not _is_x_running(): 335 return 336 337 next_index = len(glob.glob( 338 os.path.join(dest_dir, '%s-*.%s' % (fname_prefix, format)))) 339 screenshot_file = os.path.join( 340 dest_dir, '%s-%d.%s' % (fname_prefix, next_index, format)) 341 logging.info('Saving screenshot to %s.', screenshot_file) 342 343 import_cmd = ('/usr/local/bin/import -window root -depth 8 %s' % 344 screenshot_file) 345 346 _execute_screenshot_capture_command(import_cmd) 347 348 return screenshot_file 349 350 351def take_screen_shot_crop_by_height(fullpath, final_height, x_offset_pixels, 352 y_offset_pixels): 353 """ 354 Take a screenshot, crop to final height starting at given (x, y) coordinate. 355 356 Image width will be adjusted to maintain original aspect ratio). 357 358 @param fullpath: path, fullpath of the file that will become the image file. 359 @param final_height: integer, height in pixels of resulting image. 360 @param x_offset_pixels: integer, number of pixels from left margin 361 to begin cropping. 362 @param y_offset_pixels: integer, number of pixels from top margin 363 to begin cropping. 364 365 """ 366 367 params = {'height': final_height, 'x_offset': x_offset_pixels, 368 'y_offset': y_offset_pixels, 'path': fullpath} 369 370 import_cmd = ('/usr/local/bin/import -window root -depth 8 -crop ' 371 'x%(height)d+%(x_offset)d+%(y_offset)d %(path)s' % params) 372 373 _execute_screenshot_capture_command(import_cmd) 374 375 return fullpath 376 377 378def get_dut_display_resolution(): 379 """ 380 Parses output of xrandr to determine the display resolution of the dut. 381 382 @return: tuple, (w,h) resolution of device under test. 383 """ 384 385 env_vars = 'DISPLAY=:0.0 XAUTHORITY=/home/chronos/.Xauthority' 386 cmd = '%s xrandr | egrep -o "current [0-9]* x [0-9]*"' % env_vars 387 output = base_utils.system_output(cmd) 388 389 m = re.search('(\d+) x (\d+)', output) 390 391 if len(m.groups()) == 2: 392 return int(m.group(1)), int(m.group(2)) 393 else: 394 return None 395 396 397def _execute_screenshot_capture_command(import_cmd_string): 398 """ 399 Executes command to capture a screenshot. 400 401 Provides safe execution of command to capture screenshot by wrapping 402 the command around a try-catch construct. 403 404 @param import_cmd_string: string, screenshot capture command. 405 406 """ 407 408 old_exc_type = sys.exc_info()[0] 409 full_cmd = ('DISPLAY=:0.0 XAUTHORITY=/home/chronos/.Xauthority %s' % 410 import_cmd_string) 411 try: 412 base_utils.system(full_cmd) 413 except Exception as err: 414 # Do not raise an exception if the screenshot fails while processing 415 # another exception. 416 if old_exc_type is None: 417 raise 418 logging.error(err) 419 420 421def _is_x_running(): 422 try: 423 return int(base_utils.system_output('pgrep -o ^X$')) > 0 424 except Exception: 425 return False 426