base_utils.py revision 8ba770dc9bcf2e60bf2db1b7128d7502df6e8e60
1""" 2DO NOT import this file directly - import client/bin/utils.py, 3which will mix this in 4 5Convenience functions for use by tests or whomever. 6 7Note that this file is mixed in by utils.py - note very carefully the 8precedence order defined there 9""" 10import os, shutil, commands, pickle, glob 11import math, re, fnmatch, logging, multiprocessing 12from autotest_lib.client.common_lib import error, utils, magic 13 14 15def grep(pattern, file): 16 """ 17 This is mainly to fix the return code inversion from grep 18 Also handles compressed files. 19 20 returns 1 if the pattern is present in the file, 0 if not. 21 """ 22 command = 'grep "%s" > /dev/null' % pattern 23 ret = cat_file_to_cmd(file, command, ignore_status=True) 24 return not ret 25 26 27def difflist(list1, list2): 28 """returns items in list2 that are not in list1""" 29 diff = []; 30 for x in list2: 31 if x not in list1: 32 diff.append(x) 33 return diff 34 35 36def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 37 """ 38 equivalent to 'cat file | command' but knows to use 39 zcat or bzcat if appropriate 40 """ 41 if not os.path.isfile(file): 42 raise NameError('invalid file %s to cat to command %s' 43 % (file, command)) 44 45 if return_output: 46 run_cmd = utils.system_output 47 else: 48 run_cmd = utils.system 49 50 if magic.guess_type(file) == 'application/x-bzip2': 51 cat = 'bzcat' 52 elif magic.guess_type(file) == 'application/x-gzip': 53 cat = 'zcat' 54 else: 55 cat = 'cat' 56 return run_cmd('%s %s | %s' % (cat, file, command), 57 ignore_status=ignore_status) 58 59 60def extract_tarball_to_dir(tarball, dir): 61 """ 62 Extract a tarball to a specified directory name instead of whatever 63 the top level of a tarball is - useful for versioned directory names, etc 64 """ 65 if os.path.exists(dir): 66 if os.path.isdir(dir): 67 shutil.rmtree(dir) 68 else: 69 os.remove(dir) 70 pwd = os.getcwd() 71 os.chdir(os.path.dirname(os.path.abspath(dir))) 72 newdir = extract_tarball(tarball) 73 os.rename(newdir, dir) 74 os.chdir(pwd) 75 76 77def extract_tarball(tarball): 78 """Returns the directory extracted by the tarball.""" 79 extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 80 return_output=True).splitlines() 81 82 dir = None 83 84 for line in extracted: 85 line = re.sub(r'^./', '', line) 86 if not line or line == '.': 87 continue 88 topdir = line.split('/')[0] 89 if os.path.isdir(topdir): 90 if dir: 91 assert(dir == topdir) 92 else: 93 dir = topdir 94 if dir: 95 return dir 96 else: 97 raise NameError('extracting tarball produced no dir') 98 99 100def hash_file(filename, size=None, method="md5"): 101 """ 102 Calculate the hash of filename. 103 If size is not None, limit to first size bytes. 104 Throw exception if something is wrong with filename. 105 Can be also implemented with bash one-liner (assuming size%1024==0): 106 dd if=filename bs=1024 count=size/1024 | sha1sum - 107 108 @param filename: Path of the file that will have its hash calculated. 109 @param method: Method used to calculate the hash. Supported methods: 110 * md5 111 * sha1 112 @returns: Hash of the file, if something goes wrong, return None. 113 """ 114 chunksize = 4096 115 fsize = os.path.getsize(filename) 116 117 if not size or size > fsize: 118 size = fsize 119 f = open(filename, 'rb') 120 121 try: 122 hash = utils.hash(method) 123 except ValueError: 124 logging.error("Unknown hash type %s, returning None", method) 125 126 while size > 0: 127 if chunksize > size: 128 chunksize = size 129 data = f.read(chunksize) 130 if len(data) == 0: 131 logging.debug("Nothing left to read but size=%d", size) 132 break 133 hash.update(data) 134 size -= len(data) 135 f.close() 136 return hash.hexdigest() 137 138 139def unmap_url_cache(cachedir, url, expected_hash, method="md5"): 140 """ 141 Downloads a file from a URL to a cache directory. If the file is already 142 at the expected position and has the expected hash, let's not download it 143 again. 144 145 @param cachedir: Directory that might hold a copy of the file we want to 146 download. 147 @param url: URL for the file we want to download. 148 @param expected_hash: Hash string that we expect the file downloaded to 149 have. 150 @param method: Method used to calculate the hash string (md5, sha1). 151 """ 152 # Let's convert cachedir to a canonical path, if it's not already 153 cachedir = os.path.realpath(cachedir) 154 if not os.path.isdir(cachedir): 155 try: 156 os.makedirs(cachedir) 157 except: 158 raise ValueError('Could not create cache directory %s' % cachedir) 159 file_from_url = os.path.basename(url) 160 file_local_path = os.path.join(cachedir, file_from_url) 161 162 file_hash = None 163 failure_counter = 0 164 while not file_hash == expected_hash: 165 if os.path.isfile(file_local_path): 166 file_hash = hash_file(file_local_path, method) 167 if file_hash == expected_hash: 168 # File is already at the expected position and ready to go 169 src = file_from_url 170 else: 171 # Let's download the package again, it's corrupted... 172 logging.error("Seems that file %s is corrupted, trying to " 173 "download it again", file_from_url) 174 src = url 175 failure_counter += 1 176 else: 177 # File is not there, let's download it 178 src = url 179 if failure_counter > 1: 180 raise EnvironmentError("Consistently failed to download the " 181 "package %s. Aborting further download " 182 "attempts. This might mean either the " 183 "network connection has problems or the " 184 "expected hash string that was determined " 185 "for this file is wrong", file_from_url) 186 file_path = utils.unmap_url(cachedir, src, cachedir) 187 188 return file_path 189 190 191def force_copy(src, dest): 192 """Replace dest with a new copy of src, even if it exists""" 193 if os.path.isfile(dest): 194 os.remove(dest) 195 if os.path.isdir(dest): 196 dest = os.path.join(dest, os.path.basename(src)) 197 shutil.copyfile(src, dest) 198 return dest 199 200 201def force_link(src, dest): 202 """Link src to dest, overwriting it if it exists""" 203 return utils.system("ln -sf %s %s" % (src, dest)) 204 205 206def file_contains_pattern(file, pattern): 207 """Return true if file contains the specified egrep pattern""" 208 if not os.path.isfile(file): 209 raise NameError('file %s does not exist' % file) 210 return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 211 212 213def list_grep(list, pattern): 214 """True if any item in list matches the specified pattern.""" 215 compiled = re.compile(pattern) 216 for line in list: 217 match = compiled.search(line) 218 if (match): 219 return 1 220 return 0 221 222 223def get_os_vendor(): 224 """Try to guess what's the os vendor 225 """ 226 if os.path.isfile('/etc/SuSE-release'): 227 return 'SUSE' 228 229 issue = '/etc/issue' 230 231 if not os.path.isfile(issue): 232 return 'Unknown' 233 234 if file_contains_pattern(issue, 'Red Hat'): 235 return 'Red Hat' 236 elif file_contains_pattern(issue, 'Fedora'): 237 return 'Fedora Core' 238 elif file_contains_pattern(issue, 'SUSE'): 239 return 'SUSE' 240 elif file_contains_pattern(issue, 'Ubuntu'): 241 return 'Ubuntu' 242 elif file_contains_pattern(issue, 'Debian'): 243 return 'Debian' 244 else: 245 return 'Unknown' 246 247 248def get_cc(): 249 try: 250 return os.environ['CC'] 251 except KeyError: 252 return 'gcc' 253 254 255def get_vmlinux(): 256 """Return the full path to vmlinux 257 258 Ahem. This is crap. Pray harder. Bad Martin. 259 """ 260 vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') 261 if os.path.isfile(vmlinux): 262 return vmlinux 263 vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') 264 if os.path.isfile(vmlinux): 265 return vmlinux 266 return None 267 268 269def get_systemmap(): 270 """Return the full path to System.map 271 272 Ahem. This is crap. Pray harder. Bad Martin. 273 """ 274 map = '/boot/System.map-%s' % utils.system_output('uname -r') 275 if os.path.isfile(map): 276 return map 277 map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') 278 if os.path.isfile(map): 279 return map 280 return None 281 282 283def get_modules_dir(): 284 """Return the modules dir for the running kernel version""" 285 kernel_version = utils.system_output('uname -r') 286 return '/lib/modules/%s/kernel' % kernel_version 287 288 289def get_cpu_arch(): 290 """Work out which CPU architecture we're running on""" 291 f = open('/proc/cpuinfo', 'r') 292 cpuinfo = f.readlines() 293 f.close() 294 if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 295 return 'power' 296 elif list_grep(cpuinfo, '^cpu.*POWER4'): 297 return 'power4' 298 elif list_grep(cpuinfo, '^cpu.*POWER5'): 299 return 'power5' 300 elif list_grep(cpuinfo, '^cpu.*POWER6'): 301 return 'power6' 302 elif list_grep(cpuinfo, '^cpu.*POWER7'): 303 return 'power7' 304 elif list_grep(cpuinfo, '^cpu.*PPC970'): 305 return 'power970' 306 elif list_grep(cpuinfo, 'ARM'): 307 return 'arm' 308 elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 309 return 'x86_64' 310 else: 311 return 'i386' 312 313def get_arm_soc_family(): 314 """Work out which ARM SoC we're running on""" 315 f = open('/proc/cpuinfo', 'r') 316 cpuinfo = f.readlines() 317 f.close() 318 if list_grep(cpuinfo, 'EXYNOS5'): 319 return 'exynos5' 320 elif list_grep(cpuinfo, 'Tegra'): 321 return 'tegra' 322 return 'arm' 323 324def get_cpu_soc_family(): 325 """Like get_cpu_arch, but for ARM, returns the SoC family name""" 326 family = get_cpu_arch() 327 if family == 'arm': 328 family = get_arm_soc_family() 329 return family 330 331def get_current_kernel_arch(): 332 """Get the machine architecture, now just a wrap of 'uname -m'.""" 333 return os.popen('uname -m').read().rstrip() 334 335 336def get_file_arch(filename): 337 # -L means follow symlinks 338 file_data = utils.system_output('file -L ' + filename) 339 if file_data.count('80386'): 340 return 'i386' 341 return None 342 343 344def count_cpus(): 345 """number of CPUs in the local machine according to /proc/cpuinfo""" 346 try: 347 return multiprocessing.cpu_count() 348 except Exception as e: 349 logging.exception('can not get cpu count from' 350 ' multiprocessing.cpu_count()') 351 352 f = file('/proc/cpuinfo', 'r') 353 cpus = 0 354 for line in f.readlines(): 355 # Matches lines like "processor : 0" 356 if re.search(r'^processor\s*:\s*[0-9]+$', line): 357 cpus += 1 358 # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582. 359 return cpus if cpus > 0 else 1 360 361 362# Returns total memory in kb 363def read_from_meminfo(key): 364 meminfo = utils.system_output('grep %s /proc/meminfo' % key) 365 return int(re.search(r'\d+', meminfo).group(0)) 366 367 368def memtotal(): 369 return read_from_meminfo('MemTotal') 370 371 372def freememtotal(): 373 return read_from_meminfo('MemFree') 374 375def usable_memtotal(): 376 # Assume 30MB reserved by the OS 377 reserved = 30 * 1024 378 ret = read_from_meminfo('MemFree') 379 ret += read_from_meminfo('Buffers') 380 ret += read_from_meminfo('Cached') 381 ret = max(0, ret - reserved) 382 return ret 383 384 385def rounded_memtotal(): 386 # Get total of all physical mem, in kbytes 387 usable_kbytes = memtotal() 388 # usable_kbytes is system's usable DRAM in kbytes, 389 # as reported by memtotal() from device /proc/meminfo memtotal 390 # after Linux deducts 1.5% to 5.1% for system table overhead 391 # Undo the unknown actual deduction by rounding up 392 # to next small multiple of a big power-of-two 393 # eg 12GB - 5.1% gets rounded back up to 12GB 394 mindeduct = 0.015 # 1.5 percent 395 maxdeduct = 0.055 # 5.5 percent 396 # deduction range 1.5% .. 5.5% supports physical mem sizes 397 # 6GB .. 12GB in steps of .5GB 398 # 12GB .. 24GB in steps of 1 GB 399 # 24GB .. 48GB in steps of 2 GB ... 400 # Finer granularity in physical mem sizes would require 401 # tighter spread between min and max possible deductions 402 403 # increase mem size by at least min deduction, without rounding 404 min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 405 # increase mem size further by 2**n rounding, by 0..roundKb or more 406 round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 407 # find least binary roundup 2**n that covers worst-cast roundKb 408 mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 409 # have round_kbytes <= mod2n < round_kbytes*2 410 # round min_kbytes up to next multiple of mod2n 411 phys_kbytes = min_kbytes + mod2n - 1 412 phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 413 return phys_kbytes 414 415 416def sysctl(key, value=None): 417 """Generic implementation of sysctl, to read and write. 418 419 @param key: A location under /proc/sys 420 @param value: If not None, a value to write into the sysctl. 421 422 @return The single-line sysctl value as a string. 423 """ 424 path = '/proc/sys/%s' % key 425 if value is not None: 426 utils.write_one_line(path, str(value)) 427 return utils.read_one_line(path) 428 429 430def sysctl_kernel(key, value=None): 431 """(Very) partial implementation of sysctl, for kernel params""" 432 if value is not None: 433 # write 434 utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) 435 else: 436 # read 437 out = utils.read_one_line('/proc/sys/kernel/%s' % key) 438 return int(re.search(r'\d+', out).group(0)) 439 440 441def _convert_exit_status(sts): 442 if os.WIFSIGNALED(sts): 443 return -os.WTERMSIG(sts) 444 elif os.WIFEXITED(sts): 445 return os.WEXITSTATUS(sts) 446 else: 447 # impossible? 448 raise RuntimeError("Unknown exit status %d!" % sts) 449 450 451def where_art_thy_filehandles(): 452 """Dump the current list of filehandles""" 453 os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 454 455 456def print_to_tty(string): 457 """Output string straight to the tty""" 458 open('/dev/tty', 'w').write(string + '\n') 459 460 461def dump_object(object): 462 """Dump an object's attributes and methods 463 464 kind of like dir() 465 """ 466 for item in object.__dict__.iteritems(): 467 print item 468 try: 469 (key, value) = item 470 dump_object(value) 471 except: 472 continue 473 474 475def environ(env_key): 476 """return the requested environment variable, or '' if unset""" 477 if (os.environ.has_key(env_key)): 478 return os.environ[env_key] 479 else: 480 return '' 481 482 483def prepend_path(newpath, oldpath): 484 """prepend newpath to oldpath""" 485 if (oldpath): 486 return newpath + ':' + oldpath 487 else: 488 return newpath 489 490 491def append_path(oldpath, newpath): 492 """append newpath to oldpath""" 493 if (oldpath): 494 return oldpath + ':' + newpath 495 else: 496 return newpath 497 498 499def avgtime_print(dir): 500 """ Calculate some benchmarking statistics. 501 Input is a directory containing a file called 'time'. 502 File contains one-per-line results of /usr/bin/time. 503 Output is average Elapsed, User, and System time in seconds, 504 and average CPU percentage. 505 """ 506 f = open(dir + "/time") 507 user = system = elapsed = cpu = count = 0 508 r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU') 509 for line in f.readlines(): 510 try: 511 s = r.match(line); 512 user += float(s.group(1)) 513 system += float(s.group(2)) 514 elapsed += (float(s.group(3)) * 60) + float(s.group(4)) 515 cpu += float(s.group(5)) 516 count += 1 517 except: 518 raise ValueError("badly formatted times") 519 520 f.close() 521 return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 522 (elapsed / count, user / count, system / count, cpu / count) 523 524 525def running_config(): 526 """ 527 Return path of config file of the currently running kernel 528 """ 529 version = utils.system_output('uname -r') 530 for config in ('/proc/config.gz', \ 531 '/boot/config-%s' % version, 532 '/lib/modules/%s/build/.config' % version): 533 if os.path.isfile(config): 534 return config 535 return None 536 537 538def check_for_kernel_feature(feature): 539 config = running_config() 540 541 if not config: 542 raise TypeError("Can't find kernel config file") 543 544 if magic.guess_type(config) == 'application/x-gzip': 545 grep = 'zgrep' 546 else: 547 grep = 'grep' 548 grep += ' ^CONFIG_%s= %s' % (feature, config) 549 550 if not utils.system_output(grep, ignore_status=True): 551 raise ValueError("Kernel doesn't have a %s feature" % (feature)) 552 553 554def cpu_online_map(): 555 """ 556 Check out the available cpu online map 557 """ 558 cpus = [] 559 for line in open('/proc/cpuinfo', 'r').readlines(): 560 if line.startswith('processor'): 561 cpus.append(line.split()[2]) # grab cpu number 562 return cpus 563 564 565def check_glibc_ver(ver): 566 glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 567 glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 568 if utils.compare_versions(glibc_ver, ver) == -1: 569 raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % 570 (glibc_ver, ver)) 571 572def check_kernel_ver(ver): 573 kernel_ver = utils.system_output('uname -r') 574 kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 575 # In compare_versions, if v1 < v2, return value == -1 576 if utils.compare_versions(kv_tmp[0], ver) == -1: 577 raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % 578 (kernel_ver, ver)) 579 580 581def human_format(number): 582 # Convert number to kilo / mega / giga format. 583 if number < 1024: 584 return "%d" % number 585 kilo = float(number) / 1024.0 586 if kilo < 1024: 587 return "%.2fk" % kilo 588 meg = kilo / 1024.0 589 if meg < 1024: 590 return "%.2fM" % meg 591 gig = meg / 1024.0 592 return "%.2fG" % gig 593 594 595def numa_nodes(): 596 node_paths = glob.glob('/sys/devices/system/node/node*') 597 nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 598 return (sorted(nodes)) 599 600 601def node_size(): 602 nodes = max(len(numa_nodes()), 1) 603 return ((memtotal() * 1024) / nodes) 604 605 606def to_seconds(time_string): 607 """Converts a string in M+:SS.SS format to S+.SS""" 608 elts = time_string.split(':') 609 if len(elts) == 1: 610 return time_string 611 return str(int(elts[0]) * 60 + float(elts[1])) 612 613 614def extract_all_time_results(results_string): 615 """Extract user, system, and elapsed times into a list of tuples""" 616 pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed") 617 results = [] 618 for result in pattern.findall(results_string): 619 results.append(tuple([to_seconds(elt) for elt in result])) 620 return results 621 622 623def pickle_load(filename): 624 return pickle.load(open(filename, 'r')) 625 626 627# Return the kernel version and build timestamp. 628def running_os_release(): 629 return os.uname()[2:4] 630 631 632def running_os_ident(): 633 (version, timestamp) = running_os_release() 634 return version + '::' + timestamp 635 636 637def running_os_full_version(): 638 (version, timestamp) = running_os_release() 639 return version 640 641 642# much like find . -name 'pattern' 643def locate(pattern, root=os.getcwd()): 644 for path, dirs, files in os.walk(root): 645 for f in files: 646 if fnmatch.fnmatch(f, pattern): 647 yield os.path.abspath(os.path.join(path, f)) 648 649 650def freespace(path): 651 """Return the disk free space, in bytes""" 652 s = os.statvfs(path) 653 return s.f_bavail * s.f_bsize 654 655 656def disk_block_size(path): 657 """Return the disk block size, in bytes""" 658 return os.statvfs(path).f_bsize 659 660 661def get_cpu_family(): 662 procinfo = utils.system_output('cat /proc/cpuinfo') 663 CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M) 664 matches = CPU_FAMILY_RE.findall(procinfo) 665 if matches: 666 return int(matches[0]) 667 else: 668 raise error.TestError('Could not get valid cpu family data') 669 670 671def get_disks(): 672 df_output = utils.system_output('df') 673 disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 674 return disk_re.findall(df_output) 675 676 677def get_disk_size(disk_name): 678 """ 679 Return size of disk in byte. Return 0 in Error Case 680 681 @param disk_name: disk name to find size 682 """ 683 device = os.path.basename(disk_name) 684 for line in file('/proc/partitions'): 685 try: 686 _, _, blocks, name = re.split(r' +', line.strip()) 687 except ValueError: 688 continue 689 if name == device: 690 return 1024 * int(blocks) 691 return 0 692 693 694def get_disk_size_gb(disk_name): 695 """ 696 Return size of disk in GB (10^9). Return 0 in Error Case 697 698 @param disk_name: disk name to find size 699 """ 700 return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5) 701 702 703def get_disk_model(disk_name): 704 """ 705 Return model name for internal storage device 706 707 @param disk_name: disk name to find model 708 """ 709 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 710 cmd2 = 'grep -E "ID_(NAME|MODEL)="' 711 cmd3 = 'cut -f 2 -d"="' 712 cmd = ' | '.join([cmd1, cmd2, cmd3]) 713 return utils.system_output(cmd) 714 715 716def get_disk_from_filename(filename): 717 """ 718 Return the disk device the filename is on. 719 If the file is on tmpfs or other special file systems, 720 return None. 721 722 @param filename: name of file, full path. 723 """ 724 re_disk = re.compile('/dev/sd[a-z]|/dev/mmcblk[0-9]*') 725 726 if not os.path.isfile(filename): 727 raise error.TestError('file %s missing' % filename) 728 729 if filename[0] != '/': 730 raise error.TestError('This code works only with full path') 731 732 m = re_disk.match(filename) 733 while not m: 734 if filename[0] != '/': 735 return None 736 if filename == '/dev/root': 737 cmd = 'rootdev -d -s' 738 elif filename.startswith('/dev/mapper'): 739 cmd = 'dmsetup table "%s"' % os.path.basename(filename) 740 dmsetup_output = utils.system_output(cmd).split(' ') 741 if dmsetup_output[2] == 'verity': 742 maj_min = dmsetup_output[4] 743 elif dmsetup_output[2] == 'crypt': 744 maj_min = dmsetup_output[6] 745 cmd = 'realpath "/dev/block/%s"' % maj_min 746 elif filename.startswith('/dev/loop'): 747 cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename 748 else: 749 cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename 750 filename = utils.system_output(cmd) 751 m = re_disk.match(filename) 752 return m.group(0) 753 754 755def get_disk_firmware_version(disk_name): 756 """ 757 Return firmware version for internal storage device. (empty string for eMMC) 758 759 @param disk_name: disk name to find model 760 """ 761 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 762 cmd2 = 'grep -E "ID_REVISION="' 763 cmd3 = 'cut -f 2 -d"="' 764 cmd = ' | '.join([cmd1, cmd2, cmd3]) 765 return utils.system_output(cmd) 766 767 768def is_disk_scsi(disk_name): 769 """ 770 Return true if disk is a scsi device, return false otherwise 771 772 @param disk_name: disk name check 773 """ 774 return re.match('/dev/sd[a-z]+', disk_name) 775 776 777def is_disk_harddisk(disk_name): 778 """ 779 Return true if disk is a harddisk, return false otherwise 780 781 @param disk_name: disk name check 782 """ 783 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 784 cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="' 785 cmd3 = 'cut -f 2 -d"="' 786 cmd = ' | '.join([cmd1, cmd2, cmd3]) 787 788 rtt = utils.system_output(cmd) 789 790 # eMMC will not have this field; rtt == '' 791 # SSD will have zero rotation rate; rtt == '0' 792 # For harddisk rtt > 0 793 return rtt and int(rtt) > 0 794 795 796def verify_hdparm_feature(disk_name, feature): 797 """ 798 Check for feature support for SCSI disk using hdparm 799 800 @param disk_name: target disk 801 @param feature: hdparm output string of the feature 802 """ 803 cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature) 804 ret = utils.system(cmd, ignore_status=True) 805 if ret == 0: 806 return True 807 elif ret == 1: 808 return False 809 else: 810 raise error.TestFail('Error running command %s' % cmd) 811 812 813def get_storage_error_msg(disk_name, reason): 814 """ 815 Get Error message for storage test which include disk model. 816 and also include the firmware version for the SCSI disk 817 818 @param disk_name: target disk 819 @param reason: Reason of the error. 820 """ 821 822 msg = reason 823 824 model = get_disk_model(disk_name) 825 msg += ' Disk model: %s' % model 826 827 if is_disk_scsi(disk_name): 828 fw = get_disk_firmware_version(disk_name) 829 msg += ' firmware: %s' % fw 830 831 return msg 832 833 834def load_module(module_name): 835 # Checks if a module has already been loaded 836 if module_is_loaded(module_name): 837 return False 838 839 utils.system('/sbin/modprobe ' + module_name) 840 return True 841 842 843def unload_module(module_name): 844 """ 845 Removes a module. Handles dependencies. If even then it's not possible 846 to remove one of the modules, it will trhow an error.CmdError exception. 847 848 @param module_name: Name of the module we want to remove. 849 """ 850 l_raw = utils.system_output("/bin/lsmod").splitlines() 851 lsmod = [x for x in l_raw if x.split()[0] == module_name] 852 if len(lsmod) > 0: 853 line_parts = lsmod[0].split() 854 if len(line_parts) == 4: 855 submodules = line_parts[3].split(",") 856 for submodule in submodules: 857 unload_module(submodule) 858 utils.system("/sbin/modprobe -r %s" % module_name) 859 logging.info("Module %s unloaded", module_name) 860 else: 861 logging.info("Module %s is already unloaded", module_name) 862 863 864def module_is_loaded(module_name): 865 module_name = module_name.replace('-', '_') 866 modules = utils.system_output('/bin/lsmod').splitlines() 867 for module in modules: 868 if module.startswith(module_name) and module[len(module_name)] == ' ': 869 return True 870 return False 871 872 873def get_loaded_modules(): 874 lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:] 875 return [line.split(None, 1)[0] for line in lsmod_output] 876 877 878def get_huge_page_size(): 879 output = utils.system_output('grep Hugepagesize /proc/meminfo') 880 return int(output.split()[1]) # Assumes units always in kB. :( 881 882 883def get_num_huge_pages(): 884 raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') 885 return int(raw_hugepages.split()[2]) 886 887 888def set_num_huge_pages(num): 889 utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) 890 891 892def get_cpu_vendor(): 893 cpuinfo = open('/proc/cpuinfo').read() 894 vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo) 895 for i in xrange(1, len(vendors)): 896 if vendors[i] != vendors[0]: 897 raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 898 return vendors[0] 899 900 901def probe_cpus(): 902 """ 903 This routine returns a list of cpu devices found under 904 /sys/devices/system/cpu. 905 """ 906 cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 907 return utils.system_output(cmd).splitlines() 908 909 910def ping_default_gateway(): 911 """Ping the default gateway.""" 912 913 network = open('/etc/sysconfig/network') 914 m = re.search('GATEWAY=(\S+)', network.read()) 915 916 if m: 917 gw = m.group(1) 918 cmd = 'ping %s -c 5 > /dev/null' % gw 919 return utils.system(cmd, ignore_status=True) 920 921 raise error.TestError('Unable to find default gateway') 922 923 924def drop_caches(): 925 """Writes back all dirty pages to disk and clears all the caches.""" 926 utils.system("sync") 927 # We ignore failures here as this will fail on 2.6.11 kernels. 928 utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 929 930 931def process_is_alive(name_pattern): 932 """ 933 'pgrep name' misses all python processes and also long process names. 934 'pgrep -f name' gets all shell commands with name in args. 935 So look only for command whose initial pathname ends with name. 936 Name itself is an egrep pattern, so it can use | etc for variations. 937 """ 938 return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern, 939 ignore_status=True) == 0 940 941 942def get_hwclock_seconds(utc=True): 943 """ 944 Return the hardware clock in seconds as a floating point value. 945 Use Coordinated Universal Time if utc is True, local time otherwise. 946 Raise a ValueError if unable to read the hardware clock. 947 """ 948 cmd = '/sbin/hwclock --debug' 949 if utc: 950 cmd += ' --utc' 951 hwclock_output = utils.system_output(cmd, ignore_status=True) 952 match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$', 953 hwclock_output, re.DOTALL) 954 if match: 955 seconds = int(match.group(1)) + float(match.group(2)) 956 logging.debug('hwclock seconds = %f', seconds) 957 return seconds 958 959 raise ValueError('Unable to read the hardware clock -- ' + 960 hwclock_output) 961 962 963def set_wake_alarm(alarm_time): 964 """ 965 Set the hardware RTC-based wake alarm to 'alarm_time'. 966 """ 967 utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time)) 968 969 970def set_power_state(state): 971 """ 972 Set the system power state to 'state'. 973 """ 974 utils.write_one_line('/sys/power/state', state) 975 976 977def standby(): 978 """ 979 Power-on suspend (S1) 980 """ 981 set_power_state('standby') 982 983 984def suspend_to_ram(): 985 """ 986 Suspend the system to RAM (S3) 987 """ 988 set_power_state('mem') 989 990 991def suspend_to_disk(): 992 """ 993 Suspend the system to disk (S4) 994 """ 995 set_power_state('disk') 996