base_utils.py revision eb5e7a3ea391cbf48e8990937a1cb96e20e52715
1""" 2DO NOT import this file directly - import client/bin/utils.py, 3which will mix this in 4 5Convenience functions for use by tests or whomever. 6 7Note that this file is mixed in by utils.py - note very carefully the 8precedence order defined there 9""" 10import os, shutil, commands, pickle, glob 11import math, re, fnmatch, logging, multiprocessing 12from autotest_lib.client.common_lib import error, utils, magic 13 14 15def grep(pattern, file): 16 """ 17 This is mainly to fix the return code inversion from grep 18 Also handles compressed files. 19 20 returns 1 if the pattern is present in the file, 0 if not. 21 """ 22 command = 'grep "%s" > /dev/null' % pattern 23 ret = cat_file_to_cmd(file, command, ignore_status=True) 24 return not ret 25 26 27def difflist(list1, list2): 28 """returns items in list2 that are not in list1""" 29 diff = []; 30 for x in list2: 31 if x not in list1: 32 diff.append(x) 33 return diff 34 35 36def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 37 """ 38 equivalent to 'cat file | command' but knows to use 39 zcat or bzcat if appropriate 40 """ 41 if not os.path.isfile(file): 42 raise NameError('invalid file %s to cat to command %s' 43 % (file, command)) 44 45 if return_output: 46 run_cmd = utils.system_output 47 else: 48 run_cmd = utils.system 49 50 if magic.guess_type(file) == 'application/x-bzip2': 51 cat = 'bzcat' 52 elif magic.guess_type(file) == 'application/x-gzip': 53 cat = 'zcat' 54 else: 55 cat = 'cat' 56 return run_cmd('%s %s | %s' % (cat, file, command), 57 ignore_status=ignore_status) 58 59 60def extract_tarball_to_dir(tarball, dir): 61 """ 62 Extract a tarball to a specified directory name instead of whatever 63 the top level of a tarball is - useful for versioned directory names, etc 64 """ 65 if os.path.exists(dir): 66 if os.path.isdir(dir): 67 shutil.rmtree(dir) 68 else: 69 os.remove(dir) 70 pwd = os.getcwd() 71 os.chdir(os.path.dirname(os.path.abspath(dir))) 72 newdir = extract_tarball(tarball) 73 os.rename(newdir, dir) 74 os.chdir(pwd) 75 76 77def extract_tarball(tarball): 78 """Returns the directory extracted by the tarball.""" 79 extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 80 return_output=True).splitlines() 81 82 dir = None 83 84 for line in extracted: 85 line = re.sub(r'^./', '', line) 86 if not line or line == '.': 87 continue 88 topdir = line.split('/')[0] 89 if os.path.isdir(topdir): 90 if dir: 91 assert(dir == topdir) 92 else: 93 dir = topdir 94 if dir: 95 return dir 96 else: 97 raise NameError('extracting tarball produced no dir') 98 99 100def hash_file(filename, size=None, method="md5"): 101 """ 102 Calculate the hash of filename. 103 If size is not None, limit to first size bytes. 104 Throw exception if something is wrong with filename. 105 Can be also implemented with bash one-liner (assuming size%1024==0): 106 dd if=filename bs=1024 count=size/1024 | sha1sum - 107 108 @param filename: Path of the file that will have its hash calculated. 109 @param method: Method used to calculate the hash. Supported methods: 110 * md5 111 * sha1 112 @returns: Hash of the file, if something goes wrong, return None. 113 """ 114 chunksize = 4096 115 fsize = os.path.getsize(filename) 116 117 if not size or size > fsize: 118 size = fsize 119 f = open(filename, 'rb') 120 121 try: 122 hash = utils.hash(method) 123 except ValueError: 124 logging.error("Unknown hash type %s, returning None", method) 125 126 while size > 0: 127 if chunksize > size: 128 chunksize = size 129 data = f.read(chunksize) 130 if len(data) == 0: 131 logging.debug("Nothing left to read but size=%d", size) 132 break 133 hash.update(data) 134 size -= len(data) 135 f.close() 136 return hash.hexdigest() 137 138 139def unmap_url_cache(cachedir, url, expected_hash, method="md5"): 140 """ 141 Downloads a file from a URL to a cache directory. If the file is already 142 at the expected position and has the expected hash, let's not download it 143 again. 144 145 @param cachedir: Directory that might hold a copy of the file we want to 146 download. 147 @param url: URL for the file we want to download. 148 @param expected_hash: Hash string that we expect the file downloaded to 149 have. 150 @param method: Method used to calculate the hash string (md5, sha1). 151 """ 152 # Let's convert cachedir to a canonical path, if it's not already 153 cachedir = os.path.realpath(cachedir) 154 if not os.path.isdir(cachedir): 155 try: 156 os.makedirs(cachedir) 157 except: 158 raise ValueError('Could not create cache directory %s' % cachedir) 159 file_from_url = os.path.basename(url) 160 file_local_path = os.path.join(cachedir, file_from_url) 161 162 file_hash = None 163 failure_counter = 0 164 while not file_hash == expected_hash: 165 if os.path.isfile(file_local_path): 166 file_hash = hash_file(file_local_path, method) 167 if file_hash == expected_hash: 168 # File is already at the expected position and ready to go 169 src = file_from_url 170 else: 171 # Let's download the package again, it's corrupted... 172 logging.error("Seems that file %s is corrupted, trying to " 173 "download it again", file_from_url) 174 src = url 175 failure_counter += 1 176 else: 177 # File is not there, let's download it 178 src = url 179 if failure_counter > 1: 180 raise EnvironmentError("Consistently failed to download the " 181 "package %s. Aborting further download " 182 "attempts. This might mean either the " 183 "network connection has problems or the " 184 "expected hash string that was determined " 185 "for this file is wrong", file_from_url) 186 file_path = utils.unmap_url(cachedir, src, cachedir) 187 188 return file_path 189 190 191def force_copy(src, dest): 192 """Replace dest with a new copy of src, even if it exists""" 193 if os.path.isfile(dest): 194 os.remove(dest) 195 if os.path.isdir(dest): 196 dest = os.path.join(dest, os.path.basename(src)) 197 shutil.copyfile(src, dest) 198 return dest 199 200 201def force_link(src, dest): 202 """Link src to dest, overwriting it if it exists""" 203 return utils.system("ln -sf %s %s" % (src, dest)) 204 205 206def file_contains_pattern(file, pattern): 207 """Return true if file contains the specified egrep pattern""" 208 if not os.path.isfile(file): 209 raise NameError('file %s does not exist' % file) 210 return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 211 212 213def list_grep(list, pattern): 214 """True if any item in list matches the specified pattern.""" 215 compiled = re.compile(pattern) 216 for line in list: 217 match = compiled.search(line) 218 if (match): 219 return 1 220 return 0 221 222 223def get_os_vendor(): 224 """Try to guess what's the os vendor 225 """ 226 if os.path.isfile('/etc/SuSE-release'): 227 return 'SUSE' 228 229 issue = '/etc/issue' 230 231 if not os.path.isfile(issue): 232 return 'Unknown' 233 234 if file_contains_pattern(issue, 'Red Hat'): 235 return 'Red Hat' 236 elif file_contains_pattern(issue, 'Fedora'): 237 return 'Fedora Core' 238 elif file_contains_pattern(issue, 'SUSE'): 239 return 'SUSE' 240 elif file_contains_pattern(issue, 'Ubuntu'): 241 return 'Ubuntu' 242 elif file_contains_pattern(issue, 'Debian'): 243 return 'Debian' 244 else: 245 return 'Unknown' 246 247 248def get_cc(): 249 try: 250 return os.environ['CC'] 251 except KeyError: 252 return 'gcc' 253 254 255def get_vmlinux(): 256 """Return the full path to vmlinux 257 258 Ahem. This is crap. Pray harder. Bad Martin. 259 """ 260 vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') 261 if os.path.isfile(vmlinux): 262 return vmlinux 263 vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') 264 if os.path.isfile(vmlinux): 265 return vmlinux 266 return None 267 268 269def get_systemmap(): 270 """Return the full path to System.map 271 272 Ahem. This is crap. Pray harder. Bad Martin. 273 """ 274 map = '/boot/System.map-%s' % utils.system_output('uname -r') 275 if os.path.isfile(map): 276 return map 277 map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') 278 if os.path.isfile(map): 279 return map 280 return None 281 282 283def get_modules_dir(): 284 """Return the modules dir for the running kernel version""" 285 kernel_version = utils.system_output('uname -r') 286 return '/lib/modules/%s/kernel' % kernel_version 287 288 289_CPUINFO_RE = re.compile(r'^(?P<key>[^\t]*)\t*: ?(?P<value>.*)$') 290 291def get_cpuinfo(): 292 """Read /proc/cpuinfo and convert to a list of dicts.""" 293 cpuinfo = [] 294 with open('/proc/cpuinfo', 'r') as f: 295 cpu = {} 296 for line in f: 297 line = line.strip() 298 if not line: 299 cpuinfo.append(cpu) 300 cpu = {} 301 continue 302 match = _CPUINFO_RE.match(line) 303 cpu[match.group('key')] = match.group('value') 304 if cpu: 305 # cpuinfo usually ends in a blank line, so this shouldn't happen. 306 cpuinfo.append(cpu) 307 return cpuinfo 308 309def get_cpu_arch(): 310 """Work out which CPU architecture we're running on""" 311 f = open('/proc/cpuinfo', 'r') 312 cpuinfo = f.readlines() 313 f.close() 314 if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 315 return 'power' 316 elif list_grep(cpuinfo, '^cpu.*POWER4'): 317 return 'power4' 318 elif list_grep(cpuinfo, '^cpu.*POWER5'): 319 return 'power5' 320 elif list_grep(cpuinfo, '^cpu.*POWER6'): 321 return 'power6' 322 elif list_grep(cpuinfo, '^cpu.*POWER7'): 323 return 'power7' 324 elif list_grep(cpuinfo, '^cpu.*PPC970'): 325 return 'power970' 326 elif list_grep(cpuinfo, 'ARM'): 327 return 'arm' 328 elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 329 return 'x86_64' 330 else: 331 return 'i386' 332 333def get_arm_soc_family(): 334 """Work out which ARM SoC we're running on""" 335 f = open('/proc/cpuinfo', 'r') 336 cpuinfo = f.readlines() 337 f.close() 338 if list_grep(cpuinfo, 'EXYNOS5'): 339 return 'exynos5' 340 elif list_grep(cpuinfo, 'Tegra'): 341 return 'tegra' 342 elif list_grep(cpuinfo, 'Rockchip'): 343 return 'rockchip' 344 return 'arm' 345 346def get_cpu_soc_family(): 347 """Like get_cpu_arch, but for ARM, returns the SoC family name""" 348 family = get_cpu_arch() 349 if family == 'arm': 350 family = get_arm_soc_family() 351 return family 352 353INTEL_UARCH_TABLE = { 354 '06_36': 'Atom', 355 '06_26': 'Atom', 356 '06_1C': 'Atom', 357 '06_3D': 'Broadwell', 358 '06_3F': 'Haswell', 359 '06_3C': 'Haswell', 360 '06_46': 'Haswell', 361 '06_45': 'Haswell', 362 '06_3E': 'IvyBridge', 363 '06_3A': 'IvyBridge', 364 '06_2D': 'SandyBridge', 365 '06_2A': 'SandyBridge', 366 '06_2F': 'Westmere', 367 '06_2C': 'Westmere', 368 '06_25': 'Westmere', 369 '06_2E': 'Nehalem', 370 '06_1F': 'Nehalem', 371 '06_1E': 'Nehalem', 372 '06_1D': 'Nehalem', 373 '06_1A': 'Nehalem', 374 '06_17': 'Nehalem', 375 '06_16': 'Merom', 376 '06_0F': 'Merom', 377 '0F_06': 'Presler', 378 '0F_04': 'Prescott', 379 '0F_03': 'Prescott', 380 '06_0D': 'Dothan', 381} 382 383def get_intel_cpu_uarch(numeric=False): 384 """Return the Intel microarchitecture we're running on, or None. 385 386 Returns None if this is not an Intel CPU. Returns the family and model as 387 underscore-separated hex (per Intel manual convention) if the uarch is not 388 known, or if numeric is True. 389 """ 390 if not get_current_kernel_arch().startswith('x86'): 391 return None 392 cpuinfo = get_cpuinfo()[0] 393 if cpuinfo['vendor_id'] != 'GenuineIntel': 394 return None 395 family_model = '%02X_%02X' % (int(cpuinfo['cpu family']), 396 int(cpuinfo['model'])) 397 if numeric: 398 return family_model 399 return INTEL_UARCH_TABLE.get(family_model, family_model) 400 401 402def get_current_kernel_arch(): 403 """Get the machine architecture, now just a wrap of 'uname -m'.""" 404 return os.popen('uname -m').read().rstrip() 405 406 407def get_file_arch(filename): 408 # -L means follow symlinks 409 file_data = utils.system_output('file -L ' + filename) 410 if file_data.count('80386'): 411 return 'i386' 412 return None 413 414 415def count_cpus(): 416 """number of CPUs in the local machine according to /proc/cpuinfo""" 417 try: 418 return multiprocessing.cpu_count() 419 except Exception as e: 420 logging.exception('can not get cpu count from' 421 ' multiprocessing.cpu_count()') 422 423 f = file('/proc/cpuinfo', 'r') 424 cpus = 0 425 for line in f.readlines(): 426 # Matches lines like "processor : 0" 427 if re.search(r'^processor\s*:\s*[0-9]+$', line): 428 cpus += 1 429 # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582. 430 return cpus if cpus > 0 else 1 431 432 433# Returns total memory in kb 434def read_from_meminfo(key): 435 meminfo = utils.system_output('grep %s /proc/meminfo' % key) 436 return int(re.search(r'\d+', meminfo).group(0)) 437 438 439def memtotal(): 440 return read_from_meminfo('MemTotal') 441 442 443def freememtotal(): 444 return read_from_meminfo('MemFree') 445 446def usable_memtotal(): 447 # Reserved 5% for OS use 448 return int(read_from_meminfo('MemFree') * 0.95) 449 450 451def rounded_memtotal(): 452 # Get total of all physical mem, in kbytes 453 usable_kbytes = memtotal() 454 # usable_kbytes is system's usable DRAM in kbytes, 455 # as reported by memtotal() from device /proc/meminfo memtotal 456 # after Linux deducts 1.5% to 5.1% for system table overhead 457 # Undo the unknown actual deduction by rounding up 458 # to next small multiple of a big power-of-two 459 # eg 12GB - 5.1% gets rounded back up to 12GB 460 mindeduct = 0.015 # 1.5 percent 461 maxdeduct = 0.055 # 5.5 percent 462 # deduction range 1.5% .. 5.5% supports physical mem sizes 463 # 6GB .. 12GB in steps of .5GB 464 # 12GB .. 24GB in steps of 1 GB 465 # 24GB .. 48GB in steps of 2 GB ... 466 # Finer granularity in physical mem sizes would require 467 # tighter spread between min and max possible deductions 468 469 # increase mem size by at least min deduction, without rounding 470 min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 471 # increase mem size further by 2**n rounding, by 0..roundKb or more 472 round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 473 # find least binary roundup 2**n that covers worst-cast roundKb 474 mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 475 # have round_kbytes <= mod2n < round_kbytes*2 476 # round min_kbytes up to next multiple of mod2n 477 phys_kbytes = min_kbytes + mod2n - 1 478 phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 479 return phys_kbytes 480 481 482def sysctl(key, value=None): 483 """Generic implementation of sysctl, to read and write. 484 485 @param key: A location under /proc/sys 486 @param value: If not None, a value to write into the sysctl. 487 488 @return The single-line sysctl value as a string. 489 """ 490 path = '/proc/sys/%s' % key 491 if value is not None: 492 utils.write_one_line(path, str(value)) 493 return utils.read_one_line(path) 494 495 496def sysctl_kernel(key, value=None): 497 """(Very) partial implementation of sysctl, for kernel params""" 498 if value is not None: 499 # write 500 utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) 501 else: 502 # read 503 out = utils.read_one_line('/proc/sys/kernel/%s' % key) 504 return int(re.search(r'\d+', out).group(0)) 505 506 507def _convert_exit_status(sts): 508 if os.WIFSIGNALED(sts): 509 return -os.WTERMSIG(sts) 510 elif os.WIFEXITED(sts): 511 return os.WEXITSTATUS(sts) 512 else: 513 # impossible? 514 raise RuntimeError("Unknown exit status %d!" % sts) 515 516 517def where_art_thy_filehandles(): 518 """Dump the current list of filehandles""" 519 os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 520 521 522def print_to_tty(string): 523 """Output string straight to the tty""" 524 open('/dev/tty', 'w').write(string + '\n') 525 526 527def dump_object(object): 528 """Dump an object's attributes and methods 529 530 kind of like dir() 531 """ 532 for item in object.__dict__.iteritems(): 533 print item 534 try: 535 (key, value) = item 536 dump_object(value) 537 except: 538 continue 539 540 541def environ(env_key): 542 """return the requested environment variable, or '' if unset""" 543 if (os.environ.has_key(env_key)): 544 return os.environ[env_key] 545 else: 546 return '' 547 548 549def prepend_path(newpath, oldpath): 550 """prepend newpath to oldpath""" 551 if (oldpath): 552 return newpath + ':' + oldpath 553 else: 554 return newpath 555 556 557def append_path(oldpath, newpath): 558 """append newpath to oldpath""" 559 if (oldpath): 560 return oldpath + ':' + newpath 561 else: 562 return newpath 563 564 565def avgtime_print(dir): 566 """ Calculate some benchmarking statistics. 567 Input is a directory containing a file called 'time'. 568 File contains one-per-line results of /usr/bin/time. 569 Output is average Elapsed, User, and System time in seconds, 570 and average CPU percentage. 571 """ 572 f = open(dir + "/time") 573 user = system = elapsed = cpu = count = 0 574 r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU') 575 for line in f.readlines(): 576 try: 577 s = r.match(line); 578 user += float(s.group(1)) 579 system += float(s.group(2)) 580 elapsed += (float(s.group(3)) * 60) + float(s.group(4)) 581 cpu += float(s.group(5)) 582 count += 1 583 except: 584 raise ValueError("badly formatted times") 585 586 f.close() 587 return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 588 (elapsed / count, user / count, system / count, cpu / count) 589 590 591def running_config(): 592 """ 593 Return path of config file of the currently running kernel 594 """ 595 version = utils.system_output('uname -r') 596 for config in ('/proc/config.gz', \ 597 '/boot/config-%s' % version, 598 '/lib/modules/%s/build/.config' % version): 599 if os.path.isfile(config): 600 return config 601 return None 602 603 604def check_for_kernel_feature(feature): 605 config = running_config() 606 607 if not config: 608 raise TypeError("Can't find kernel config file") 609 610 if magic.guess_type(config) == 'application/x-gzip': 611 grep = 'zgrep' 612 else: 613 grep = 'grep' 614 grep += ' ^CONFIG_%s= %s' % (feature, config) 615 616 if not utils.system_output(grep, ignore_status=True): 617 raise ValueError("Kernel doesn't have a %s feature" % (feature)) 618 619 620def cpu_online_map(): 621 """ 622 Check out the available cpu online map 623 """ 624 cpus = [] 625 for line in open('/proc/cpuinfo', 'r').readlines(): 626 if line.startswith('processor'): 627 cpus.append(line.split()[2]) # grab cpu number 628 return cpus 629 630 631def check_glibc_ver(ver): 632 glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 633 glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 634 if utils.compare_versions(glibc_ver, ver) == -1: 635 raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % 636 (glibc_ver, ver)) 637 638def check_kernel_ver(ver): 639 kernel_ver = utils.system_output('uname -r') 640 kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 641 # In compare_versions, if v1 < v2, return value == -1 642 if utils.compare_versions(kv_tmp[0], ver) == -1: 643 raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % 644 (kernel_ver, ver)) 645 646 647def human_format(number): 648 # Convert number to kilo / mega / giga format. 649 if number < 1024: 650 return "%d" % number 651 kilo = float(number) / 1024.0 652 if kilo < 1024: 653 return "%.2fk" % kilo 654 meg = kilo / 1024.0 655 if meg < 1024: 656 return "%.2fM" % meg 657 gig = meg / 1024.0 658 return "%.2fG" % gig 659 660 661def numa_nodes(): 662 node_paths = glob.glob('/sys/devices/system/node/node*') 663 nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 664 return (sorted(nodes)) 665 666 667def node_size(): 668 nodes = max(len(numa_nodes()), 1) 669 return ((memtotal() * 1024) / nodes) 670 671 672def to_seconds(time_string): 673 """Converts a string in M+:SS.SS format to S+.SS""" 674 elts = time_string.split(':') 675 if len(elts) == 1: 676 return time_string 677 return str(int(elts[0]) * 60 + float(elts[1])) 678 679 680def extract_all_time_results(results_string): 681 """Extract user, system, and elapsed times into a list of tuples""" 682 pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed") 683 results = [] 684 for result in pattern.findall(results_string): 685 results.append(tuple([to_seconds(elt) for elt in result])) 686 return results 687 688 689def pickle_load(filename): 690 return pickle.load(open(filename, 'r')) 691 692 693# Return the kernel version and build timestamp. 694def running_os_release(): 695 return os.uname()[2:4] 696 697 698def running_os_ident(): 699 (version, timestamp) = running_os_release() 700 return version + '::' + timestamp 701 702 703def running_os_full_version(): 704 (version, timestamp) = running_os_release() 705 return version 706 707 708# much like find . -name 'pattern' 709def locate(pattern, root=os.getcwd()): 710 for path, dirs, files in os.walk(root): 711 for f in files: 712 if fnmatch.fnmatch(f, pattern): 713 yield os.path.abspath(os.path.join(path, f)) 714 715 716def freespace(path): 717 """Return the disk free space, in bytes""" 718 s = os.statvfs(path) 719 return s.f_bavail * s.f_bsize 720 721 722def disk_block_size(path): 723 """Return the disk block size, in bytes""" 724 return os.statvfs(path).f_bsize 725 726 727def get_cpu_family(): 728 procinfo = utils.system_output('cat /proc/cpuinfo') 729 CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M) 730 matches = CPU_FAMILY_RE.findall(procinfo) 731 if matches: 732 return int(matches[0]) 733 else: 734 raise error.TestError('Could not get valid cpu family data') 735 736 737def get_disks(): 738 df_output = utils.system_output('df') 739 disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 740 return disk_re.findall(df_output) 741 742 743def get_disk_size(disk_name): 744 """ 745 Return size of disk in byte. Return 0 in Error Case 746 747 @param disk_name: disk name to find size 748 """ 749 device = os.path.basename(disk_name) 750 for line in file('/proc/partitions'): 751 try: 752 _, _, blocks, name = re.split(r' +', line.strip()) 753 except ValueError: 754 continue 755 if name == device: 756 return 1024 * int(blocks) 757 return 0 758 759 760def get_disk_size_gb(disk_name): 761 """ 762 Return size of disk in GB (10^9). Return 0 in Error Case 763 764 @param disk_name: disk name to find size 765 """ 766 return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5) 767 768 769def get_disk_model(disk_name): 770 """ 771 Return model name for internal storage device 772 773 @param disk_name: disk name to find model 774 """ 775 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 776 cmd2 = 'grep -E "ID_(NAME|MODEL)="' 777 cmd3 = 'cut -f 2 -d"="' 778 cmd = ' | '.join([cmd1, cmd2, cmd3]) 779 return utils.system_output(cmd) 780 781 782def get_disk_from_filename(filename): 783 """ 784 Return the disk device the filename is on. 785 If the file is on tmpfs or other special file systems, 786 return None. 787 788 @param filename: name of file, full path. 789 """ 790 re_disk = re.compile('/dev/sd[a-z]|/dev/mmcblk[0-9]*') 791 792 if not os.path.exists(filename): 793 raise error.TestError('file %s missing' % filename) 794 795 if filename[0] != '/': 796 raise error.TestError('This code works only with full path') 797 798 m = re_disk.match(filename) 799 while not m: 800 if filename[0] != '/': 801 return None 802 if filename == '/dev/root': 803 cmd = 'rootdev -d -s' 804 elif filename.startswith('/dev/mapper'): 805 cmd = 'dmsetup table "%s"' % os.path.basename(filename) 806 dmsetup_output = utils.system_output(cmd).split(' ') 807 if dmsetup_output[2] == 'verity': 808 maj_min = dmsetup_output[4] 809 elif dmsetup_output[2] == 'crypt': 810 maj_min = dmsetup_output[6] 811 cmd = 'realpath "/dev/block/%s"' % maj_min 812 elif filename.startswith('/dev/loop'): 813 cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename 814 else: 815 cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename 816 filename = utils.system_output(cmd) 817 m = re_disk.match(filename) 818 return m.group(0) 819 820 821def get_disk_firmware_version(disk_name): 822 """ 823 Return firmware version for internal storage device. (empty string for eMMC) 824 825 @param disk_name: disk name to find model 826 """ 827 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 828 cmd2 = 'grep -E "ID_REVISION="' 829 cmd3 = 'cut -f 2 -d"="' 830 cmd = ' | '.join([cmd1, cmd2, cmd3]) 831 return utils.system_output(cmd) 832 833 834def is_disk_scsi(disk_name): 835 """ 836 Return true if disk is a scsi device, return false otherwise 837 838 @param disk_name: disk name check 839 """ 840 return re.match('/dev/sd[a-z]+', disk_name) 841 842 843def is_disk_harddisk(disk_name): 844 """ 845 Return true if disk is a harddisk, return false otherwise 846 847 @param disk_name: disk name check 848 """ 849 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 850 cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="' 851 cmd3 = 'cut -f 2 -d"="' 852 cmd = ' | '.join([cmd1, cmd2, cmd3]) 853 854 rtt = utils.system_output(cmd) 855 856 # eMMC will not have this field; rtt == '' 857 # SSD will have zero rotation rate; rtt == '0' 858 # For harddisk rtt > 0 859 return rtt and int(rtt) > 0 860 861 862def verify_hdparm_feature(disk_name, feature): 863 """ 864 Check for feature support for SCSI disk using hdparm 865 866 @param disk_name: target disk 867 @param feature: hdparm output string of the feature 868 """ 869 cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature) 870 ret = utils.system(cmd, ignore_status=True) 871 if ret == 0: 872 return True 873 elif ret == 1: 874 return False 875 else: 876 raise error.TestFail('Error running command %s' % cmd) 877 878 879def get_storage_error_msg(disk_name, reason): 880 """ 881 Get Error message for storage test which include disk model. 882 and also include the firmware version for the SCSI disk 883 884 @param disk_name: target disk 885 @param reason: Reason of the error. 886 """ 887 888 msg = reason 889 890 model = get_disk_model(disk_name) 891 msg += ' Disk model: %s' % model 892 893 if is_disk_scsi(disk_name): 894 fw = get_disk_firmware_version(disk_name) 895 msg += ' firmware: %s' % fw 896 897 return msg 898 899 900def load_module(module_name): 901 # Checks if a module has already been loaded 902 if module_is_loaded(module_name): 903 return False 904 905 utils.system('/sbin/modprobe ' + module_name) 906 return True 907 908 909def unload_module(module_name): 910 """ 911 Removes a module. Handles dependencies. If even then it's not possible 912 to remove one of the modules, it will trhow an error.CmdError exception. 913 914 @param module_name: Name of the module we want to remove. 915 """ 916 l_raw = utils.system_output("/bin/lsmod").splitlines() 917 lsmod = [x for x in l_raw if x.split()[0] == module_name] 918 if len(lsmod) > 0: 919 line_parts = lsmod[0].split() 920 if len(line_parts) == 4: 921 submodules = line_parts[3].split(",") 922 for submodule in submodules: 923 unload_module(submodule) 924 utils.system("/sbin/modprobe -r %s" % module_name) 925 logging.info("Module %s unloaded", module_name) 926 else: 927 logging.info("Module %s is already unloaded", module_name) 928 929 930def module_is_loaded(module_name): 931 module_name = module_name.replace('-', '_') 932 modules = utils.system_output('/bin/lsmod').splitlines() 933 for module in modules: 934 if module.startswith(module_name) and module[len(module_name)] == ' ': 935 return True 936 return False 937 938 939def get_loaded_modules(): 940 lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:] 941 return [line.split(None, 1)[0] for line in lsmod_output] 942 943 944def get_huge_page_size(): 945 output = utils.system_output('grep Hugepagesize /proc/meminfo') 946 return int(output.split()[1]) # Assumes units always in kB. :( 947 948 949def get_num_huge_pages(): 950 raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') 951 return int(raw_hugepages.split()[2]) 952 953 954def set_num_huge_pages(num): 955 utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) 956 957 958def get_cpu_vendor(): 959 cpuinfo = open('/proc/cpuinfo').read() 960 vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo) 961 for i in xrange(1, len(vendors)): 962 if vendors[i] != vendors[0]: 963 raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 964 return vendors[0] 965 966 967def probe_cpus(): 968 """ 969 This routine returns a list of cpu devices found under 970 /sys/devices/system/cpu. 971 """ 972 cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 973 return utils.system_output(cmd).splitlines() 974 975 976def ping_default_gateway(): 977 """Ping the default gateway.""" 978 979 network = open('/etc/sysconfig/network') 980 m = re.search('GATEWAY=(\S+)', network.read()) 981 982 if m: 983 gw = m.group(1) 984 cmd = 'ping %s -c 5 > /dev/null' % gw 985 return utils.system(cmd, ignore_status=True) 986 987 raise error.TestError('Unable to find default gateway') 988 989 990def drop_caches(): 991 """Writes back all dirty pages to disk and clears all the caches.""" 992 utils.system("sync") 993 # We ignore failures here as this will fail on 2.6.11 kernels. 994 utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 995 996 997def process_is_alive(name_pattern): 998 """ 999 'pgrep name' misses all python processes and also long process names. 1000 'pgrep -f name' gets all shell commands with name in args. 1001 So look only for command whose initial pathname ends with name. 1002 Name itself is an egrep pattern, so it can use | etc for variations. 1003 """ 1004 return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern, 1005 ignore_status=True) == 0 1006 1007 1008def get_hwclock_seconds(utc=True): 1009 """ 1010 Return the hardware clock in seconds as a floating point value. 1011 Use Coordinated Universal Time if utc is True, local time otherwise. 1012 Raise a ValueError if unable to read the hardware clock. 1013 """ 1014 cmd = '/sbin/hwclock --debug' 1015 if utc: 1016 cmd += ' --utc' 1017 hwclock_output = utils.system_output(cmd, ignore_status=True) 1018 match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$', 1019 hwclock_output, re.DOTALL) 1020 if match: 1021 seconds = int(match.group(1)) + float(match.group(2)) 1022 logging.debug('hwclock seconds = %f', seconds) 1023 return seconds 1024 1025 raise ValueError('Unable to read the hardware clock -- ' + 1026 hwclock_output) 1027 1028 1029def set_wake_alarm(alarm_time): 1030 """ 1031 Set the hardware RTC-based wake alarm to 'alarm_time'. 1032 """ 1033 utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time)) 1034 1035 1036def set_power_state(state): 1037 """ 1038 Set the system power state to 'state'. 1039 """ 1040 utils.write_one_line('/sys/power/state', state) 1041 1042 1043def standby(): 1044 """ 1045 Power-on suspend (S1) 1046 """ 1047 set_power_state('standby') 1048 1049 1050def suspend_to_ram(): 1051 """ 1052 Suspend the system to RAM (S3) 1053 """ 1054 set_power_state('mem') 1055 1056 1057def suspend_to_disk(): 1058 """ 1059 Suspend the system to disk (S4) 1060 """ 1061 set_power_state('disk') 1062