utils.py revision 6f27d4f22a1ba5063968b8c322fa0845f3279ade
1# 2# Copyright 2008 Google Inc. Released under the GPL v2 3 4import os, pickle, random, re, resource, select, shutil, signal, StringIO 5import socket, struct, subprocess, sys, time, textwrap, urlparse 6import warnings, smtplib, logging, urllib2 7try: 8 import hashlib 9except ImportError: 10 import md5, sha 11from autotest_lib.client.common_lib import error, logging_manager 12 13def deprecated(func): 14 """This is a decorator which can be used to mark functions as deprecated. 15 It will result in a warning being emmitted when the function is used.""" 16 def new_func(*args, **dargs): 17 warnings.warn("Call to deprecated function %s." % func.__name__, 18 category=DeprecationWarning) 19 return func(*args, **dargs) 20 new_func.__name__ = func.__name__ 21 new_func.__doc__ = func.__doc__ 22 new_func.__dict__.update(func.__dict__) 23 return new_func 24 25 26class _NullStream(object): 27 def write(self, data): 28 pass 29 30 31 def flush(self): 32 pass 33 34 35TEE_TO_LOGS = object() 36_the_null_stream = _NullStream() 37 38DEFAULT_STDOUT_LEVEL = logging.DEBUG 39DEFAULT_STDERR_LEVEL = logging.ERROR 40 41# prefixes for logging stdout/stderr of commands 42STDOUT_PREFIX = '[stdout] ' 43STDERR_PREFIX = '[stderr] ' 44 45 46def get_stream_tee_file(stream, level, prefix=''): 47 if stream is None: 48 return _the_null_stream 49 if stream is TEE_TO_LOGS: 50 return logging_manager.LoggingFile(level=level, prefix=prefix) 51 return stream 52 53 54class BgJob(object): 55 def __init__(self, command, stdout_tee=None, stderr_tee=None, verbose=True, 56 stdin=None, stderr_level=DEFAULT_STDERR_LEVEL): 57 self.command = command 58 self.stdout_tee = get_stream_tee_file(stdout_tee, DEFAULT_STDOUT_LEVEL, 59 prefix=STDOUT_PREFIX) 60 self.stderr_tee = get_stream_tee_file(stderr_tee, stderr_level, 61 prefix=STDERR_PREFIX) 62 self.result = CmdResult(command) 63 64 # allow for easy stdin input by string, we'll let subprocess create 65 # a pipe for stdin input and we'll write to it in the wait loop 66 if isinstance(stdin, basestring): 67 self.string_stdin = stdin 68 stdin = subprocess.PIPE 69 else: 70 self.string_stdin = None 71 72 if verbose: 73 logging.debug("Running '%s'" % command) 74 self.sp = subprocess.Popen(command, stdout=subprocess.PIPE, 75 stderr=subprocess.PIPE, 76 preexec_fn=self._reset_sigpipe, shell=True, 77 executable="/bin/bash", 78 stdin=stdin) 79 80 81 def output_prepare(self, stdout_file=None, stderr_file=None): 82 self.stdout_file = stdout_file 83 self.stderr_file = stderr_file 84 85 86 def process_output(self, stdout=True, final_read=False): 87 """output_prepare must be called prior to calling this""" 88 if stdout: 89 pipe, buf, tee = self.sp.stdout, self.stdout_file, self.stdout_tee 90 else: 91 pipe, buf, tee = self.sp.stderr, self.stderr_file, self.stderr_tee 92 93 if final_read: 94 # read in all the data we can from pipe and then stop 95 data = [] 96 while select.select([pipe], [], [], 0)[0]: 97 data.append(os.read(pipe.fileno(), 1024)) 98 if len(data[-1]) == 0: 99 break 100 data = "".join(data) 101 else: 102 # perform a single read 103 data = os.read(pipe.fileno(), 1024) 104 buf.write(data) 105 tee.write(data) 106 107 108 def cleanup(self): 109 self.stdout_tee.flush() 110 self.stderr_tee.flush() 111 self.sp.stdout.close() 112 self.sp.stderr.close() 113 self.result.stdout = self.stdout_file.getvalue() 114 self.result.stderr = self.stderr_file.getvalue() 115 116 117 def _reset_sigpipe(self): 118 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 119 120 121def ip_to_long(ip): 122 # !L is a long in network byte order 123 return struct.unpack('!L', socket.inet_aton(ip))[0] 124 125 126def long_to_ip(number): 127 # See above comment. 128 return socket.inet_ntoa(struct.pack('!L', number)) 129 130 131def create_subnet_mask(bits): 132 return (1 << 32) - (1 << 32-bits) 133 134 135def format_ip_with_mask(ip, mask_bits): 136 masked_ip = ip_to_long(ip) & create_subnet_mask(mask_bits) 137 return "%s/%s" % (long_to_ip(masked_ip), mask_bits) 138 139 140def normalize_hostname(alias): 141 ip = socket.gethostbyname(alias) 142 return socket.gethostbyaddr(ip)[0] 143 144 145def get_ip_local_port_range(): 146 match = re.match(r'\s*(\d+)\s*(\d+)\s*$', 147 read_one_line('/proc/sys/net/ipv4/ip_local_port_range')) 148 return (int(match.group(1)), int(match.group(2))) 149 150 151def set_ip_local_port_range(lower, upper): 152 write_one_line('/proc/sys/net/ipv4/ip_local_port_range', 153 '%d %d\n' % (lower, upper)) 154 155 156 157def send_email(mail_from, mail_to, subject, body): 158 """ 159 Sends an email via smtp 160 161 mail_from: string with email address of sender 162 mail_to: string or list with email address(es) of recipients 163 subject: string with subject of email 164 body: (multi-line) string with body of email 165 """ 166 if isinstance(mail_to, str): 167 mail_to = [mail_to] 168 msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (mail_from, ','.join(mail_to), 169 subject, body) 170 try: 171 mailer = smtplib.SMTP('localhost') 172 try: 173 mailer.sendmail(mail_from, mail_to, msg) 174 finally: 175 mailer.quit() 176 except Exception, e: 177 # Emails are non-critical, not errors, but don't raise them 178 print "Sending email failed. Reason: %s" % repr(e) 179 180 181def read_one_line(filename): 182 return open(filename, 'r').readline().rstrip('\n') 183 184 185def read_file(filename): 186 f = open(filename) 187 try: 188 return f.read() 189 finally: 190 f.close() 191 192 193def write_one_line(filename, line): 194 open_write_close(filename, line.rstrip('\n') + '\n') 195 196 197def open_write_close(filename, data): 198 f = open(filename, 'w') 199 try: 200 f.write(data) 201 finally: 202 f.close() 203 204 205def matrix_to_string(matrix, header=None): 206 """ 207 Return a pretty, aligned string representation of a nxm matrix. 208 209 This representation can be used to print any tabular data, such as 210 database results. It works by scanning the lengths of each element 211 in each column, and determining the format string dynamically. 212 213 @param matrix: Matrix representation (list with n rows of m elements). 214 @param header: Optional tuple with header elements to be displayed. 215 """ 216 lengths = [] 217 for row in matrix: 218 for column in row: 219 i = row.index(column) 220 cl = len(column) 221 try: 222 ml = lengths[i] 223 if cl > ml: 224 lengths[i] = cl 225 except IndexError: 226 lengths.append(cl) 227 228 lengths = tuple(lengths) 229 format_string = "" 230 for length in lengths: 231 format_string += "%-" + str(length) + "s " 232 format_string += "\n" 233 234 matrix_str = "" 235 if header: 236 matrix_str += format_string % header 237 for row in matrix: 238 matrix_str += format_string % tuple(row) 239 240 return matrix_str 241 242 243def read_keyval(path): 244 """ 245 Read a key-value pair format file into a dictionary, and return it. 246 Takes either a filename or directory name as input. If it's a 247 directory name, we assume you want the file to be called keyval. 248 """ 249 if os.path.isdir(path): 250 path = os.path.join(path, 'keyval') 251 keyval = {} 252 if os.path.exists(path): 253 for line in open(path): 254 line = re.sub('#.*', '', line).rstrip() 255 if not re.search(r'^[-\.\w]+=', line): 256 raise ValueError('Invalid format line: %s' % line) 257 key, value = line.split('=', 1) 258 if re.search('^\d+$', value): 259 value = int(value) 260 elif re.search('^(\d+\.)?\d+$', value): 261 value = float(value) 262 keyval[key] = value 263 return keyval 264 265 266def write_keyval(path, dictionary, type_tag=None): 267 """ 268 Write a key-value pair format file out to a file. This uses append 269 mode to open the file, so existing text will not be overwritten or 270 reparsed. 271 272 If type_tag is None, then the key must be composed of alphanumeric 273 characters (or dashes+underscores). However, if type-tag is not 274 null then the keys must also have "{type_tag}" as a suffix. At 275 the moment the only valid values of type_tag are "attr" and "perf". 276 """ 277 if os.path.isdir(path): 278 path = os.path.join(path, 'keyval') 279 keyval = open(path, 'a') 280 281 if type_tag is None: 282 key_regex = re.compile(r'^[-\.\w]+$') 283 else: 284 if type_tag not in ('attr', 'perf'): 285 raise ValueError('Invalid type tag: %s' % type_tag) 286 escaped_tag = re.escape(type_tag) 287 key_regex = re.compile(r'^[-\.\w]+\{%s\}$' % escaped_tag) 288 try: 289 for key in sorted(dictionary.keys()): 290 if not key_regex.search(key): 291 raise ValueError('Invalid key: %s' % key) 292 keyval.write('%s=%s\n' % (key, dictionary[key])) 293 finally: 294 keyval.close() 295 296 297def is_url(path): 298 """Return true if path looks like a URL""" 299 # for now, just handle http and ftp 300 url_parts = urlparse.urlparse(path) 301 return (url_parts[0] in ('http', 'ftp')) 302 303 304def urlopen(url, data=None, timeout=5): 305 """Wrapper to urllib2.urlopen with timeout addition.""" 306 307 # Save old timeout 308 old_timeout = socket.getdefaulttimeout() 309 socket.setdefaulttimeout(timeout) 310 try: 311 return urllib2.urlopen(url, data=data) 312 finally: 313 socket.setdefaulttimeout(old_timeout) 314 315 316def urlretrieve(url, filename, data=None, timeout=300): 317 """Retrieve a file from given url.""" 318 logging.debug('Fetching %s -> %s', url, filename) 319 320 src_file = urlopen(url, data=data, timeout=timeout) 321 try: 322 dest_file = open(filename, 'wb') 323 try: 324 shutil.copyfileobj(src_file, dest_file) 325 finally: 326 dest_file.close() 327 finally: 328 src_file.close() 329 330 331def hash(type, input=None): 332 """ 333 Returns an hash object of type md5 or sha1. This function is implemented in 334 order to encapsulate hash objects in a way that is compatible with python 335 2.4 and python 2.6 without warnings. 336 337 Note that even though python 2.6 hashlib supports hash types other than 338 md5 and sha1, we are artificially limiting the input values in order to 339 make the function to behave exactly the same among both python 340 implementations. 341 342 @param input: Optional input string that will be used to update the hash. 343 """ 344 if type not in ['md5', 'sha1']: 345 raise ValueError("Unsupported hash type: %s" % type) 346 347 try: 348 hash = hashlib.new(type) 349 except NameError: 350 if type == 'md5': 351 hash = md5.new() 352 elif type == 'sha1': 353 hash = sha.new() 354 355 if input: 356 hash.update(input) 357 358 return hash 359 360 361def get_file(src, dest, permissions=None): 362 """Get a file from src, which can be local or a remote URL""" 363 if src == dest: 364 return 365 366 if is_url(src): 367 urlretrieve(src, dest) 368 else: 369 shutil.copyfile(src, dest) 370 371 if permissions: 372 os.chmod(dest, permissions) 373 return dest 374 375 376def unmap_url(srcdir, src, destdir='.'): 377 """ 378 Receives either a path to a local file or a URL. 379 returns either the path to the local file, or the fetched URL 380 381 unmap_url('/usr/src', 'foo.tar', '/tmp') 382 = '/usr/src/foo.tar' 383 unmap_url('/usr/src', 'http://site/file', '/tmp') 384 = '/tmp/file' 385 (after retrieving it) 386 """ 387 if is_url(src): 388 url_parts = urlparse.urlparse(src) 389 filename = os.path.basename(url_parts[2]) 390 dest = os.path.join(destdir, filename) 391 return get_file(src, dest) 392 else: 393 return os.path.join(srcdir, src) 394 395 396def update_version(srcdir, preserve_srcdir, new_version, install, 397 *args, **dargs): 398 """ 399 Make sure srcdir is version new_version 400 401 If not, delete it and install() the new version. 402 403 In the preserve_srcdir case, we just check it's up to date, 404 and if not, we rerun install, without removing srcdir 405 """ 406 versionfile = os.path.join(srcdir, '.version') 407 install_needed = True 408 409 if os.path.exists(versionfile): 410 old_version = pickle.load(open(versionfile)) 411 if old_version == new_version: 412 install_needed = False 413 414 if install_needed: 415 if not preserve_srcdir and os.path.exists(srcdir): 416 shutil.rmtree(srcdir) 417 install(*args, **dargs) 418 if os.path.exists(srcdir): 419 pickle.dump(new_version, open(versionfile, 'w')) 420 421 422def get_stderr_level(stderr_is_expected): 423 if stderr_is_expected: 424 return DEFAULT_STDOUT_LEVEL 425 return DEFAULT_STDERR_LEVEL 426 427 428def run(command, timeout=None, ignore_status=False, 429 stdout_tee=None, stderr_tee=None, verbose=True, stdin=None, 430 stderr_is_expected=None, args=()): 431 """ 432 Run a command on the host. 433 434 @param command: the command line string. 435 @param timeout: time limit in seconds before attempting to kill the 436 running process. The run() function will take a few seconds 437 longer than 'timeout' to complete if it has to kill the process. 438 @param ignore_status: do not raise an exception, no matter what the exit 439 code of the command is. 440 @param stdout_tee: optional file-like object to which stdout data 441 will be written as it is generated (data will still be stored 442 in result.stdout). 443 @param stderr_tee: likewise for stderr. 444 @param verbose: if True, log the command being run. 445 @param stdin: stdin to pass to the executed process (can be a file 446 descriptor, a file object of a real file or a string). 447 @param args: sequence of strings of arguments to be given to the command 448 inside " quotes after they have been escaped for that; each 449 element in the sequence will be given as a separate command 450 argument 451 452 @return a CmdResult object 453 454 @raise CmdError: the exit code of the command execution was not 0 455 """ 456 if isinstance(args, basestring): 457 raise TypeError('Got a string for the "args" keyword argument, ' 458 'need a sequence.') 459 460 for arg in args: 461 command += ' "%s"' % sh_escape(arg) 462 if stderr_is_expected is None: 463 stderr_is_expected = ignore_status 464 465 bg_job = join_bg_jobs( 466 (BgJob(command, stdout_tee, stderr_tee, verbose, stdin=stdin, 467 stderr_level=get_stderr_level(stderr_is_expected)),), 468 timeout)[0] 469 if not ignore_status and bg_job.result.exit_status: 470 raise error.CmdError(command, bg_job.result, 471 "Command returned non-zero exit status") 472 473 return bg_job.result 474 475 476def run_parallel(commands, timeout=None, ignore_status=False, 477 stdout_tee=None, stderr_tee=None): 478 """ 479 Behaves the same as run() with the following exceptions: 480 481 - commands is a list of commands to run in parallel. 482 - ignore_status toggles whether or not an exception should be raised 483 on any error. 484 485 @return: a list of CmdResult objects 486 """ 487 bg_jobs = [] 488 for command in commands: 489 bg_jobs.append(BgJob(command, stdout_tee, stderr_tee, 490 stderr_level=get_stderr_level(ignore_status))) 491 492 # Updates objects in bg_jobs list with their process information 493 join_bg_jobs(bg_jobs, timeout) 494 495 for bg_job in bg_jobs: 496 if not ignore_status and bg_job.result.exit_status: 497 raise error.CmdError(command, bg_job.result, 498 "Command returned non-zero exit status") 499 500 return [bg_job.result for bg_job in bg_jobs] 501 502 503@deprecated 504def run_bg(command): 505 """Function deprecated. Please use BgJob class instead.""" 506 bg_job = BgJob(command) 507 return bg_job.sp, bg_job.result 508 509 510def join_bg_jobs(bg_jobs, timeout=None): 511 """Joins the bg_jobs with the current thread. 512 513 Returns the same list of bg_jobs objects that was passed in. 514 """ 515 ret, timeout_error = 0, False 516 for bg_job in bg_jobs: 517 bg_job.output_prepare(StringIO.StringIO(), StringIO.StringIO()) 518 519 try: 520 # We are holding ends to stdin, stdout pipes 521 # hence we need to be sure to close those fds no mater what 522 start_time = time.time() 523 timeout_error = _wait_for_commands(bg_jobs, start_time, timeout) 524 525 for bg_job in bg_jobs: 526 # Process stdout and stderr 527 bg_job.process_output(stdout=True,final_read=True) 528 bg_job.process_output(stdout=False,final_read=True) 529 finally: 530 # close our ends of the pipes to the sp no matter what 531 for bg_job in bg_jobs: 532 bg_job.cleanup() 533 534 if timeout_error: 535 # TODO: This needs to be fixed to better represent what happens when 536 # running in parallel. However this is backwards compatable, so it will 537 # do for the time being. 538 raise error.CmdError(bg_jobs[0].command, bg_jobs[0].result, 539 "Command(s) did not complete within %d seconds" 540 % timeout) 541 542 543 return bg_jobs 544 545 546def _wait_for_commands(bg_jobs, start_time, timeout): 547 # This returns True if it must return due to a timeout, otherwise False. 548 549 # To check for processes which terminate without producing any output 550 # a 1 second timeout is used in select. 551 SELECT_TIMEOUT = 1 552 553 read_list = [] 554 write_list = [] 555 reverse_dict = {} 556 557 for bg_job in bg_jobs: 558 read_list.append(bg_job.sp.stdout) 559 read_list.append(bg_job.sp.stderr) 560 reverse_dict[bg_job.sp.stdout] = (bg_job, True) 561 reverse_dict[bg_job.sp.stderr] = (bg_job, False) 562 if bg_job.string_stdin is not None: 563 write_list.append(bg_job.sp.stdin) 564 reverse_dict[bg_job.sp.stdin] = bg_job 565 566 if timeout: 567 stop_time = start_time + timeout 568 time_left = stop_time - time.time() 569 else: 570 time_left = None # so that select never times out 571 572 while not timeout or time_left > 0: 573 # select will return when we may write to stdin or when there is 574 # stdout/stderr output we can read (including when it is 575 # EOF, that is the process has terminated). 576 read_ready, write_ready, _ = select.select(read_list, write_list, [], 577 SELECT_TIMEOUT) 578 579 # os.read() has to be used instead of 580 # subproc.stdout.read() which will otherwise block 581 for file_obj in read_ready: 582 bg_job, is_stdout = reverse_dict[file_obj] 583 bg_job.process_output(is_stdout) 584 585 for file_obj in write_ready: 586 # we can write PIPE_BUF bytes without blocking 587 # POSIX requires PIPE_BUF is >= 512 588 bg_job = reverse_dict[file_obj] 589 file_obj.write(bg_job.string_stdin[:512]) 590 bg_job.string_stdin = bg_job.string_stdin[512:] 591 # no more input data, close stdin, remove it from the select set 592 if not bg_job.string_stdin: 593 file_obj.close() 594 write_list.remove(file_obj) 595 del reverse_dict[file_obj] 596 597 all_jobs_finished = True 598 for bg_job in bg_jobs: 599 if bg_job.result.exit_status is not None: 600 continue 601 602 bg_job.result.exit_status = bg_job.sp.poll() 603 if bg_job.result.exit_status is not None: 604 # process exited, remove its stdout/stdin from the select set 605 bg_job.result.duration = time.time() - start_time 606 read_list.remove(bg_job.sp.stdout) 607 read_list.remove(bg_job.sp.stderr) 608 del reverse_dict[bg_job.sp.stdout] 609 del reverse_dict[bg_job.sp.stderr] 610 else: 611 all_jobs_finished = False 612 613 if all_jobs_finished: 614 return False 615 616 if timeout: 617 time_left = stop_time - time.time() 618 619 # Kill all processes which did not complete prior to timeout 620 for bg_job in bg_jobs: 621 if bg_job.result.exit_status is not None: 622 continue 623 624 logging.warn('run process timeout (%s) fired on: %s', timeout, 625 bg_job.command) 626 nuke_subprocess(bg_job.sp) 627 bg_job.result.exit_status = bg_job.sp.poll() 628 bg_job.result.duration = time.time() - start_time 629 630 return True 631 632 633def pid_is_alive(pid): 634 """ 635 True if process pid exists and is not yet stuck in Zombie state. 636 Zombies are impossible to move between cgroups, etc. 637 pid can be integer, or text of integer. 638 """ 639 path = '/proc/%s/stat' % pid 640 641 try: 642 stat = read_one_line(path) 643 except IOError: 644 if not os.path.exists(path): 645 # file went away 646 return False 647 raise 648 649 return stat.split()[2] != 'Z' 650 651 652def signal_pid(pid, sig): 653 """ 654 Sends a signal to a process id. Returns True if the process terminated 655 successfully, False otherwise. 656 """ 657 try: 658 os.kill(pid, sig) 659 except OSError: 660 # The process may have died before we could kill it. 661 pass 662 663 for i in range(5): 664 if not pid_is_alive(pid): 665 return True 666 time.sleep(1) 667 668 # The process is still alive 669 return False 670 671 672def nuke_subprocess(subproc): 673 # check if the subprocess is still alive, first 674 if subproc.poll() is not None: 675 return subproc.poll() 676 677 # the process has not terminated within timeout, 678 # kill it via an escalating series of signals. 679 signal_queue = [signal.SIGTERM, signal.SIGKILL] 680 for sig in signal_queue: 681 signal_pid(subproc.pid, sig) 682 if subproc.poll() is not None: 683 return subproc.poll() 684 685 686def nuke_pid(pid, signal_queue=(signal.SIGTERM, signal.SIGKILL)): 687 # the process has not terminated within timeout, 688 # kill it via an escalating series of signals. 689 for sig in signal_queue: 690 if signal_pid(pid, sig): 691 return 692 693 # no signal successfully terminated the process 694 raise error.AutoservRunError('Could not kill %d' % pid, None) 695 696 697def system(command, timeout=None, ignore_status=False): 698 """ 699 Run a command 700 701 @param timeout: timeout in seconds 702 @param ignore_status: if ignore_status=False, throw an exception if the 703 command's exit code is non-zero 704 if ignore_stauts=True, return the exit code. 705 706 @return exit status of command 707 (note, this will always be zero unless ignore_status=True) 708 """ 709 return run(command, timeout=timeout, ignore_status=ignore_status, 710 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS).exit_status 711 712 713def system_parallel(commands, timeout=None, ignore_status=False): 714 """This function returns a list of exit statuses for the respective 715 list of commands.""" 716 return [bg_jobs.exit_status for bg_jobs in 717 run_parallel(commands, timeout=timeout, ignore_status=ignore_status, 718 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] 719 720 721def system_output(command, timeout=None, ignore_status=False, 722 retain_output=False, args=()): 723 """ 724 Run a command and return the stdout output. 725 726 @param command: command string to execute. 727 @param timeout: time limit in seconds before attempting to kill the 728 running process. The function will take a few seconds longer 729 than 'timeout' to complete if it has to kill the process. 730 @param ignore_status: do not raise an exception, no matter what the exit 731 code of the command is. 732 @param retain_output: set to True to make stdout/stderr of the command 733 output to be also sent to the logging system 734 @param args: sequence of strings of arguments to be given to the command 735 inside " quotes after they have been escaped for that; each 736 element in the sequence will be given as a separate command 737 argument 738 739 @return a string with the stdout output of the command. 740 """ 741 if retain_output: 742 out = run(command, timeout=timeout, ignore_status=ignore_status, 743 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS, 744 args=args).stdout 745 else: 746 out = run(command, timeout=timeout, ignore_status=ignore_status, 747 args=args).stdout 748 if out[-1:] == '\n': 749 out = out[:-1] 750 return out 751 752 753def system_output_parallel(commands, timeout=None, ignore_status=False, 754 retain_output=False): 755 if retain_output: 756 out = [bg_job.stdout for bg_job 757 in run_parallel(commands, timeout=timeout, 758 ignore_status=ignore_status, 759 stdout_tee=TEE_TO_LOGS, stderr_tee=TEE_TO_LOGS)] 760 else: 761 out = [bg_job.stdout for bg_job in run_parallel(commands, 762 timeout=timeout, ignore_status=ignore_status)] 763 for x in out: 764 if out[-1:] == '\n': out = out[:-1] 765 return out 766 767 768def strip_unicode(input): 769 if type(input) == list: 770 return [strip_unicode(i) for i in input] 771 elif type(input) == dict: 772 output = {} 773 for key in input.keys(): 774 output[str(key)] = strip_unicode(input[key]) 775 return output 776 elif type(input) == unicode: 777 return str(input) 778 else: 779 return input 780 781 782def get_cpu_percentage(function, *args, **dargs): 783 """Returns a tuple containing the CPU% and return value from function call. 784 785 This function calculates the usage time by taking the difference of 786 the user and system times both before and after the function call. 787 """ 788 child_pre = resource.getrusage(resource.RUSAGE_CHILDREN) 789 self_pre = resource.getrusage(resource.RUSAGE_SELF) 790 start = time.time() 791 to_return = function(*args, **dargs) 792 elapsed = time.time() - start 793 self_post = resource.getrusage(resource.RUSAGE_SELF) 794 child_post = resource.getrusage(resource.RUSAGE_CHILDREN) 795 796 # Calculate CPU Percentage 797 s_user, s_system = [a - b for a, b in zip(self_post, self_pre)[:2]] 798 c_user, c_system = [a - b for a, b in zip(child_post, child_pre)[:2]] 799 cpu_percent = (s_user + c_user + s_system + c_system) / elapsed 800 801 return cpu_percent, to_return 802 803 804def get_arch(run_function=run): 805 """ 806 Get the hardware architecture of the machine. 807 run_function is used to execute the commands. It defaults to 808 utils.run() but a custom method (if provided) should be of the 809 same schema as utils.run. It should return a CmdResult object and 810 throw a CmdError exception. 811 """ 812 arch = run_function('/bin/uname -m').stdout.rstrip() 813 if re.match(r'i\d86$', arch): 814 arch = 'i386' 815 return arch 816 817 818def get_num_logical_cpus_per_socket(run_function=run): 819 """ 820 Get the number of cores (including hyperthreading) per cpu. 821 run_function is used to execute the commands. It defaults to 822 utils.run() but a custom method (if provided) should be of the 823 same schema as utils.run. It should return a CmdResult object and 824 throw a CmdError exception. 825 """ 826 siblings = run_function('grep "^siblings" /proc/cpuinfo').stdout.rstrip() 827 num_siblings = map(int, 828 re.findall(r'^siblings\s*:\s*(\d+)\s*$', 829 siblings, re.M)) 830 if len(num_siblings) == 0: 831 raise error.TestError('Unable to find siblings info in /proc/cpuinfo') 832 if min(num_siblings) != max(num_siblings): 833 raise error.TestError('Number of siblings differ %r' % 834 num_siblings) 835 return num_siblings[0] 836 837 838def merge_trees(src, dest): 839 """ 840 Merges a source directory tree at 'src' into a destination tree at 841 'dest'. If a path is a file in both trees than the file in the source 842 tree is APPENDED to the one in the destination tree. If a path is 843 a directory in both trees then the directories are recursively merged 844 with this function. In any other case, the function will skip the 845 paths that cannot be merged (instead of failing). 846 """ 847 if not os.path.exists(src): 848 return # exists only in dest 849 elif not os.path.exists(dest): 850 if os.path.isfile(src): 851 shutil.copy2(src, dest) # file only in src 852 else: 853 shutil.copytree(src, dest, symlinks=True) # dir only in src 854 return 855 elif os.path.isfile(src) and os.path.isfile(dest): 856 # src & dest are files in both trees, append src to dest 857 destfile = open(dest, "a") 858 try: 859 srcfile = open(src) 860 try: 861 destfile.write(srcfile.read()) 862 finally: 863 srcfile.close() 864 finally: 865 destfile.close() 866 elif os.path.isdir(src) and os.path.isdir(dest): 867 # src & dest are directories in both trees, so recursively merge 868 for name in os.listdir(src): 869 merge_trees(os.path.join(src, name), os.path.join(dest, name)) 870 else: 871 # src & dest both exist, but are incompatible 872 return 873 874 875class CmdResult(object): 876 """ 877 Command execution result. 878 879 command: String containing the command line itself 880 exit_status: Integer exit code of the process 881 stdout: String containing stdout of the process 882 stderr: String containing stderr of the process 883 duration: Elapsed wall clock time running the process 884 """ 885 886 887 def __init__(self, command="", stdout="", stderr="", 888 exit_status=None, duration=0): 889 self.command = command 890 self.exit_status = exit_status 891 self.stdout = stdout 892 self.stderr = stderr 893 self.duration = duration 894 895 896 def __repr__(self): 897 wrapper = textwrap.TextWrapper(width = 78, 898 initial_indent="\n ", 899 subsequent_indent=" ") 900 901 stdout = self.stdout.rstrip() 902 if stdout: 903 stdout = "\nstdout:\n%s" % stdout 904 905 stderr = self.stderr.rstrip() 906 if stderr: 907 stderr = "\nstderr:\n%s" % stderr 908 909 return ("* Command: %s\n" 910 "Exit status: %s\n" 911 "Duration: %s\n" 912 "%s" 913 "%s" 914 % (wrapper.fill(self.command), self.exit_status, 915 self.duration, stdout, stderr)) 916 917 918class run_randomly: 919 def __init__(self, run_sequentially=False): 920 # Run sequentially is for debugging control files 921 self.test_list = [] 922 self.run_sequentially = run_sequentially 923 924 925 def add(self, *args, **dargs): 926 test = (args, dargs) 927 self.test_list.append(test) 928 929 930 def run(self, fn): 931 while self.test_list: 932 test_index = random.randint(0, len(self.test_list)-1) 933 if self.run_sequentially: 934 test_index = 0 935 (args, dargs) = self.test_list.pop(test_index) 936 fn(*args, **dargs) 937 938 939def import_site_module(path, module, dummy=None, modulefile=None): 940 """ 941 Try to import the site specific module if it exists. 942 943 @param path full filename of the source file calling this (ie __file__) 944 @param module full module name 945 @param dummy dummy value to return in case there is no symbol to import 946 @param modulefile module filename 947 948 @return site specific module or dummy 949 950 @raises ImportError if the site file exists but imports fails 951 """ 952 short_module = module[module.rfind(".") + 1:] 953 954 if not modulefile: 955 modulefile = short_module + ".py" 956 957 if os.path.exists(os.path.join(os.path.dirname(path), modulefile)): 958 return __import__(module, {}, {}, [short_module]) 959 return dummy 960 961 962def import_site_symbol(path, module, name, dummy=None, modulefile=None): 963 """ 964 Try to import site specific symbol from site specific file if it exists 965 966 @param path full filename of the source file calling this (ie __file__) 967 @param module full module name 968 @param name symbol name to be imported from the site file 969 @param dummy dummy value to return in case there is no symbol to import 970 @param modulefile module filename 971 972 @return site specific symbol or dummy 973 974 @raises ImportError if the site file exists but imports fails 975 """ 976 module = import_site_module(path, module, modulefile=modulefile) 977 if not module: 978 return dummy 979 980 # special unique value to tell us if the symbol can't be imported 981 cant_import = object() 982 983 obj = getattr(module, name, cant_import) 984 if obj is cant_import: 985 logging.debug("unable to import site symbol '%s', using non-site " 986 "implementation", name) 987 return dummy 988 989 return obj 990 991 992def import_site_class(path, module, classname, baseclass, modulefile=None): 993 """ 994 Try to import site specific class from site specific file if it exists 995 996 Args: 997 path: full filename of the source file calling this (ie __file__) 998 module: full module name 999 classname: class name to be loaded from site file 1000 baseclass: base class object to return when no site file present or 1001 to mixin when site class exists but is not inherited from baseclass 1002 modulefile: module filename 1003 1004 Returns: baseclass if site specific class does not exist, the site specific 1005 class if it exists and is inherited from baseclass or a mixin of the 1006 site specific class and baseclass when the site specific class exists 1007 and is not inherited from baseclass 1008 1009 Raises: ImportError if the site file exists but imports fails 1010 """ 1011 1012 res = import_site_symbol(path, module, classname, None, modulefile) 1013 if res: 1014 if not issubclass(res, baseclass): 1015 # if not a subclass of baseclass then mix in baseclass with the 1016 # site specific class object and return the result 1017 res = type(classname, (res, baseclass), {}) 1018 else: 1019 res = baseclass 1020 1021 return res 1022 1023 1024def import_site_function(path, module, funcname, dummy, modulefile=None): 1025 """ 1026 Try to import site specific function from site specific file if it exists 1027 1028 Args: 1029 path: full filename of the source file calling this (ie __file__) 1030 module: full module name 1031 funcname: function name to be imported from site file 1032 dummy: dummy function to return in case there is no function to import 1033 modulefile: module filename 1034 1035 Returns: site specific function object or dummy 1036 1037 Raises: ImportError if the site file exists but imports fails 1038 """ 1039 1040 return import_site_symbol(path, module, funcname, dummy, modulefile) 1041 1042 1043def _get_pid_path(program_name): 1044 my_path = os.path.dirname(__file__) 1045 return os.path.abspath(os.path.join(my_path, "..", "..", 1046 "%s.pid" % program_name)) 1047 1048 1049def write_pid(program_name): 1050 """ 1051 Try to drop <program_name>.pid in the main autotest directory. 1052 1053 Args: 1054 program_name: prefix for file name 1055 """ 1056 pidfile = open(_get_pid_path(program_name), "w") 1057 try: 1058 pidfile.write("%s\n" % os.getpid()) 1059 finally: 1060 pidfile.close() 1061 1062 1063def delete_pid_file_if_exists(program_name): 1064 """ 1065 Tries to remove <program_name>.pid from the main autotest directory. 1066 """ 1067 pidfile_path = _get_pid_path(program_name) 1068 1069 try: 1070 os.remove(pidfile_path) 1071 except OSError: 1072 if not os.path.exists(pidfile_path): 1073 return 1074 raise 1075 1076 1077def get_pid_from_file(program_name): 1078 """ 1079 Reads the pid from <program_name>.pid in the autotest directory. 1080 1081 @param program_name the name of the program 1082 @return the pid if the file exists, None otherwise. 1083 """ 1084 pidfile_path = _get_pid_path(program_name) 1085 if not os.path.exists(pidfile_path): 1086 return None 1087 1088 pidfile = open(_get_pid_path(program_name), 'r') 1089 1090 try: 1091 try: 1092 pid = int(pidfile.readline()) 1093 except IOError: 1094 if not os.path.exists(pidfile_path): 1095 return None 1096 raise 1097 finally: 1098 pidfile.close() 1099 1100 return pid 1101 1102 1103def program_is_alive(program_name): 1104 """ 1105 Checks if the process is alive and not in Zombie state. 1106 1107 @param program_name the name of the program 1108 @return True if still alive, False otherwise 1109 """ 1110 pid = get_pid_from_file(program_name) 1111 if pid is None: 1112 return False 1113 return pid_is_alive(pid) 1114 1115 1116def signal_program(program_name, sig=signal.SIGTERM): 1117 """ 1118 Sends a signal to the process listed in <program_name>.pid 1119 1120 @param program_name the name of the program 1121 @param sig signal to send 1122 """ 1123 pid = get_pid_from_file(program_name) 1124 if pid: 1125 signal_pid(pid, sig) 1126 1127 1128def get_relative_path(path, reference): 1129 """Given 2 absolute paths "path" and "reference", compute the path of 1130 "path" as relative to the directory "reference". 1131 1132 @param path the absolute path to convert to a relative path 1133 @param reference an absolute directory path to which the relative 1134 path will be computed 1135 """ 1136 # normalize the paths (remove double slashes, etc) 1137 assert(os.path.isabs(path)) 1138 assert(os.path.isabs(reference)) 1139 1140 path = os.path.normpath(path) 1141 reference = os.path.normpath(reference) 1142 1143 # we could use os.path.split() but it splits from the end 1144 path_list = path.split(os.path.sep)[1:] 1145 ref_list = reference.split(os.path.sep)[1:] 1146 1147 # find the longest leading common path 1148 for i in xrange(min(len(path_list), len(ref_list))): 1149 if path_list[i] != ref_list[i]: 1150 # decrement i so when exiting this loop either by no match or by 1151 # end of range we are one step behind 1152 i -= 1 1153 break 1154 i += 1 1155 # drop the common part of the paths, not interested in that anymore 1156 del path_list[:i] 1157 1158 # for each uncommon component in the reference prepend a ".." 1159 path_list[:0] = ['..'] * (len(ref_list) - i) 1160 1161 return os.path.join(*path_list) 1162 1163 1164def sh_escape(command): 1165 """ 1166 Escape special characters from a command so that it can be passed 1167 as a double quoted (" ") string in a (ba)sh command. 1168 1169 Args: 1170 command: the command string to escape. 1171 1172 Returns: 1173 The escaped command string. The required englobing double 1174 quotes are NOT added and so should be added at some point by 1175 the caller. 1176 1177 See also: http://www.tldp.org/LDP/abs/html/escapingsection.html 1178 """ 1179 command = command.replace("\\", "\\\\") 1180 command = command.replace("$", r'\$') 1181 command = command.replace('"', r'\"') 1182 command = command.replace('`', r'\`') 1183 return command 1184 1185 1186def configure(extra=None, configure='./configure'): 1187 """ 1188 Run configure passing in the correct host, build, and target options. 1189 1190 @param extra: extra command line arguments to pass to configure 1191 @param configure: which configure script to use 1192 """ 1193 args = [] 1194 if 'CHOST' in os.environ: 1195 args.append('--host=' + os.environ['CHOST']) 1196 if 'CBUILD' in os.environ: 1197 args.append('--build=' + os.environ['CBUILD']) 1198 if 'CTARGET' in os.environ: 1199 args.append('--target=' + os.environ['CTARGET']) 1200 if extra: 1201 args.append(extra) 1202 1203 system('%s %s' % (configure, ' '.join(args))) 1204 1205 1206def make(extra='', make='make', timeout=None, ignore_status=False): 1207 """ 1208 Run make, adding MAKEOPTS to the list of options. 1209 1210 @param extra: extra command line arguments to pass to make. 1211 """ 1212 cmd = '%s %s %s' % (make, os.environ.get('MAKEOPTS', ''), extra) 1213 return system(cmd, timeout=timeout, ignore_status=ignore_status) 1214 1215 1216def compare_versions(ver1, ver2): 1217 """Version number comparison between ver1 and ver2 strings. 1218 1219 >>> compare_tuple("1", "2") 1220 -1 1221 >>> compare_tuple("foo-1.1", "foo-1.2") 1222 -1 1223 >>> compare_tuple("1.2", "1.2a") 1224 -1 1225 >>> compare_tuple("1.2b", "1.2a") 1226 1 1227 >>> compare_tuple("1.3.5.3a", "1.3.5.3b") 1228 -1 1229 1230 Args: 1231 ver1: version string 1232 ver2: version string 1233 1234 Returns: 1235 int: 1 if ver1 > ver2 1236 0 if ver1 == ver2 1237 -1 if ver1 < ver2 1238 """ 1239 ax = re.split('[.-]', ver1) 1240 ay = re.split('[.-]', ver2) 1241 while len(ax) > 0 and len(ay) > 0: 1242 cx = ax.pop(0) 1243 cy = ay.pop(0) 1244 maxlen = max(len(cx), len(cy)) 1245 c = cmp(cx.zfill(maxlen), cy.zfill(maxlen)) 1246 if c != 0: 1247 return c 1248 return cmp(len(ax), len(ay)) 1249 1250 1251def args_to_dict(args): 1252 """Convert autoserv extra arguments in the form of key=val or key:val to a 1253 dictionary. Each argument key is converted to lowercase dictionary key. 1254 1255 Args: 1256 args - list of autoserv extra arguments. 1257 1258 Returns: 1259 dictionary 1260 """ 1261 arg_re = re.compile(r'(\w+)[:=](.*)$') 1262 dict = {} 1263 for arg in args: 1264 match = arg_re.match(arg) 1265 if match: 1266 dict[match.group(1).lower()] = match.group(2) 1267 else: 1268 logging.warning("args_to_dict: argument '%s' doesn't match " 1269 "'%s' pattern. Ignored." % (arg, arg_re.pattern)) 1270 return dict 1271 1272 1273def get_unused_port(): 1274 """ 1275 Finds a semi-random available port. A race condition is still 1276 possible after the port number is returned, if another process 1277 happens to bind it. 1278 1279 Returns: 1280 A port number that is unused on both TCP and UDP. 1281 """ 1282 1283 def try_bind(port, socket_type, socket_proto): 1284 s = socket.socket(socket.AF_INET, socket_type, socket_proto) 1285 try: 1286 try: 1287 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 1288 s.bind(('', port)) 1289 return s.getsockname()[1] 1290 except socket.error: 1291 return None 1292 finally: 1293 s.close() 1294 1295 # On the 2.6 kernel, calling try_bind() on UDP socket returns the 1296 # same port over and over. So always try TCP first. 1297 while True: 1298 # Ask the OS for an unused port. 1299 port = try_bind(0, socket.SOCK_STREAM, socket.IPPROTO_TCP) 1300 # Check if this port is unused on the other protocol. 1301 if port and try_bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP): 1302 return port 1303