base_utils.py revision e6bc5b9ab0fdd65f131948143ee2eb97e1352d74
1""" 2DO NOT import this file directly - import client/bin/utils.py, 3which will mix this in 4 5Convenience functions for use by tests or whomever. 6 7Note that this file is mixed in by utils.py - note very carefully the 8precedence order defined there 9""" 10import os, shutil, sys, signal, commands, pickle, glob, statvfs 11import math, re, string, fnmatch, logging 12from autotest_lib.client.common_lib import error, utils 13 14 15def grep(pattern, file): 16 """ 17 This is mainly to fix the return code inversion from grep 18 Also handles compressed files. 19 20 returns 1 if the pattern is present in the file, 0 if not. 21 """ 22 command = 'grep "%s" > /dev/null' % pattern 23 ret = cat_file_to_cmd(file, command, ignore_status=True) 24 return not ret 25 26 27def difflist(list1, list2): 28 """returns items in list2 that are not in list1""" 29 diff = []; 30 for x in list2: 31 if x not in list1: 32 diff.append(x) 33 return diff 34 35 36def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 37 """ 38 equivalent to 'cat file | command' but knows to use 39 zcat or bzcat if appropriate 40 """ 41 if not os.path.isfile(file): 42 raise NameError('invalid file %s to cat to command %s' 43 % (file, command)) 44 45 if return_output: 46 run_cmd = utils.system_output 47 else: 48 run_cmd = utils.system 49 50 if file.endswith('.bz2'): 51 cat = 'bzcat' 52 elif (file.endswith('.gz') or file.endswith('.tgz')): 53 cat = 'zcat' 54 else: 55 cat = 'cat' 56 return run_cmd('%s %s | %s' % (cat, file, command), 57 ignore_status=ignore_status) 58 59 60def extract_tarball_to_dir(tarball, dir): 61 """ 62 Extract a tarball to a specified directory name instead of whatever 63 the top level of a tarball is - useful for versioned directory names, etc 64 """ 65 if os.path.exists(dir): 66 raise NameError, 'target %s already exists' % dir 67 pwd = os.getcwd() 68 os.chdir(os.path.dirname(os.path.abspath(dir))) 69 newdir = extract_tarball(tarball) 70 os.rename(newdir, dir) 71 os.chdir(pwd) 72 73 74def extract_tarball(tarball): 75 """Returns the directory extracted by the tarball.""" 76 extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 77 return_output=True).splitlines() 78 79 dir = None 80 81 for line in extracted: 82 line = re.sub(r'^./', '', line) 83 if not line or line == '.': 84 continue 85 topdir = line.split('/')[0] 86 if os.path.isdir(topdir): 87 if dir: 88 assert(dir == topdir) 89 else: 90 dir = topdir 91 if dir: 92 return dir 93 else: 94 raise NameError('extracting tarball produced no dir') 95 96 97def get_md5sum(file_path): 98 """Gets the md5sum of a file. You must provide a valid path to the file""" 99 if not os.path.isfile(file_path): 100 raise ValueError, 'invalid file %s to verify' % file_path 101 md5sum = utils.system_output("md5sum " + file_path) 102 return md5sum.split()[0] 103 104 105def unmap_url_cache(cachedir, url, expected_md5): 106 """ 107 Downloads a file from a URL to a cache directory. If the file is already 108 at the expected position and has the expected md5 number, let's not 109 download it again. 110 """ 111 # Let's convert cachedir to a canonical path, if it's not already 112 cachedir = os.path.realpath(cachedir) 113 if not os.path.isdir(cachedir): 114 try: 115 utils.system('mkdir -p ' + cachedir) 116 except: 117 raise ValueError('Could not create cache directory %s' % cachedir) 118 file_from_url = os.path.basename(url) 119 file_local_path = os.path.join(cachedir, file_from_url) 120 if os.path.isfile(file_local_path): 121 file_md5 = get_md5sum(file_local_path) 122 if file_md5 == expected_md5: 123 # File is already at the expected position and ready to go 124 src = file_from_url 125 else: 126 # Let's download the package again, it's corrupted... 127 src = url 128 else: 129 # File is not there, let's download it 130 src = url 131 return utils.unmap_url(cachedir, src, cachedir) 132 133 134def force_copy(src, dest): 135 """Replace dest with a new copy of src, even if it exists""" 136 if os.path.isfile(dest): 137 os.remove(dest) 138 if os.path.isdir(dest): 139 dest = os.path.join(dest, os.path.basename(src)) 140 shutil.copyfile(src, dest) 141 return dest 142 143 144def force_link(src, dest): 145 """Link src to dest, overwriting it if it exists""" 146 return utils.system("ln -sf %s %s" % (src, dest)) 147 148 149def file_contains_pattern(file, pattern): 150 """Return true if file contains the specified egrep pattern""" 151 if not os.path.isfile(file): 152 raise NameError('file %s does not exist' % file) 153 return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 154 155 156def list_grep(list, pattern): 157 """True if any item in list matches the specified pattern.""" 158 compiled = re.compile(pattern) 159 for line in list: 160 match = compiled.search(line) 161 if (match): 162 return 1 163 return 0 164 165 166def get_os_vendor(): 167 """Try to guess what's the os vendor 168 """ 169 if os.path.isfile('/etc/SuSE-release'): 170 return 'SUSE' 171 172 issue = '/etc/issue' 173 174 if not os.path.isfile(issue): 175 return 'Unknown' 176 177 if file_contains_pattern(issue, 'Red Hat'): 178 return 'Red Hat' 179 elif file_contains_pattern(issue, 'Fedora'): 180 return 'Fedora Core' 181 elif file_contains_pattern(issue, 'SUSE'): 182 return 'SUSE' 183 elif file_contains_pattern(issue, 'Ubuntu'): 184 return 'Ubuntu' 185 elif file_contains_pattern(issue, 'Debian'): 186 return 'Debian' 187 else: 188 return 'Unknown' 189 190 191def get_vmlinux(): 192 """Return the full path to vmlinux 193 194 Ahem. This is crap. Pray harder. Bad Martin. 195 """ 196 vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') 197 if os.path.isfile(vmlinux): 198 return vmlinux 199 vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') 200 if os.path.isfile(vmlinux): 201 return vmlinux 202 return None 203 204 205def get_systemmap(): 206 """Return the full path to System.map 207 208 Ahem. This is crap. Pray harder. Bad Martin. 209 """ 210 map = '/boot/System.map-%s' % utils.system_output('uname -r') 211 if os.path.isfile(map): 212 return map 213 map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') 214 if os.path.isfile(map): 215 return map 216 return None 217 218 219def get_modules_dir(): 220 """Return the modules dir for the running kernel version""" 221 kernel_version = utils.system_output('uname -r') 222 return '/lib/modules/%s/kernel' % kernel_version 223 224 225def get_cpu_arch(): 226 """Work out which CPU architecture we're running on""" 227 f = open('/proc/cpuinfo', 'r') 228 cpuinfo = f.readlines() 229 f.close() 230 if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 231 return 'power' 232 elif list_grep(cpuinfo, '^cpu.*POWER4'): 233 return 'power4' 234 elif list_grep(cpuinfo, '^cpu.*POWER5'): 235 return 'power5' 236 elif list_grep(cpuinfo, '^cpu.*POWER6'): 237 return 'power6' 238 elif list_grep(cpuinfo, '^cpu.*POWER7'): 239 return 'power7' 240 elif list_grep(cpuinfo, '^cpu.*PPC970'): 241 return 'power970' 242 elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 243 return 'x86_64' 244 else: 245 return 'i386' 246 247 248def get_current_kernel_arch(): 249 """Get the machine architecture, now just a wrap of 'uname -m'.""" 250 return os.popen('uname -m').read().rstrip() 251 252 253def get_file_arch(filename): 254 # -L means follow symlinks 255 file_data = utils.system_output('file -L ' + filename) 256 if file_data.count('80386'): 257 return 'i386' 258 return None 259 260 261def count_cpus(): 262 """number of CPUs in the local machine according to /proc/cpuinfo""" 263 f = file('/proc/cpuinfo', 'r') 264 cpus = 0 265 for line in f.readlines(): 266 if line.startswith('processor'): 267 cpus += 1 268 return cpus 269 270 271# Returns total memory in kb 272def read_from_meminfo(key): 273 meminfo = utils.system_output('grep %s /proc/meminfo' % key) 274 return int(re.search(r'\d+', meminfo).group(0)) 275 276 277def memtotal(): 278 return read_from_meminfo('MemTotal') 279 280 281def freememtotal(): 282 return read_from_meminfo('MemFree') 283 284 285def rounded_memtotal(): 286 # Get total of all physical mem, in kbytes 287 usable_kbytes = memtotal() 288 # usable_kbytes is system's usable DRAM in kbytes, 289 # as reported by memtotal() from device /proc/meminfo memtotal 290 # after Linux deducts 1.5% to 5.1% for system table overhead 291 # Undo the unknown actual deduction by rounding up 292 # to next small multiple of a big power-of-two 293 # eg 12GB - 5.1% gets rounded back up to 12GB 294 mindeduct = 0.015 # 1.5 percent 295 maxdeduct = 0.055 # 5.5 percent 296 # deduction range 1.5% .. 5.5% supports physical mem sizes 297 # 6GB .. 12GB in steps of .5GB 298 # 12GB .. 24GB in steps of 1 GB 299 # 24GB .. 48GB in steps of 2 GB ... 300 # Finer granularity in physical mem sizes would require 301 # tighter spread between min and max possible deductions 302 303 # increase mem size by at least min deduction, without rounding 304 min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 305 # increase mem size further by 2**n rounding, by 0..roundKb or more 306 round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 307 # find least binary roundup 2**n that covers worst-cast roundKb 308 mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 309 # have round_kbytes <= mod2n < round_kbytes*2 310 # round min_kbytes up to next multiple of mod2n 311 phys_kbytes = min_kbytes + mod2n - 1 312 phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 313 return phys_kbytes 314 315 316def sysctl_kernel(key, value=None): 317 """(Very) partial implementation of sysctl, for kernel params""" 318 if value: 319 # write 320 utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) 321 else: 322 # read 323 out = utils.read_one_line('/proc/sys/kernel/%s' % key) 324 return int(re.search(r'\d+', out).group(0)) 325 326 327def _convert_exit_status(sts): 328 if os.WIFSIGNALED(sts): 329 return -os.WTERMSIG(sts) 330 elif os.WIFEXITED(sts): 331 return os.WEXITSTATUS(sts) 332 else: 333 # impossible? 334 raise RuntimeError("Unknown exit status %d!" % sts) 335 336 337def where_art_thy_filehandles(): 338 """Dump the current list of filehandles""" 339 os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 340 341 342def print_to_tty(string): 343 """Output string straight to the tty""" 344 open('/dev/tty', 'w').write(string + '\n') 345 346 347def dump_object(object): 348 """Dump an object's attributes and methods 349 350 kind of like dir() 351 """ 352 for item in object.__dict__.iteritems(): 353 print item 354 try: 355 (key,value) = item 356 dump_object(value) 357 except: 358 continue 359 360 361def environ(env_key): 362 """return the requested environment variable, or '' if unset""" 363 if (os.environ.has_key(env_key)): 364 return os.environ[env_key] 365 else: 366 return '' 367 368 369def prepend_path(newpath, oldpath): 370 """prepend newpath to oldpath""" 371 if (oldpath): 372 return newpath + ':' + oldpath 373 else: 374 return newpath 375 376 377def append_path(oldpath, newpath): 378 """append newpath to oldpath""" 379 if (oldpath): 380 return oldpath + ':' + newpath 381 else: 382 return newpath 383 384 385def avgtime_print(dir): 386 """ Calculate some benchmarking statistics. 387 Input is a directory containing a file called 'time'. 388 File contains one-per-line results of /usr/bin/time. 389 Output is average Elapsed, User, and System time in seconds, 390 and average CPU percentage. 391 """ 392 f = open(dir + "/time") 393 user = system = elapsed = cpu = count = 0 394 r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU') 395 for line in f.readlines(): 396 try: 397 s = r.match(line); 398 user += float(s.group(1)) 399 system += float(s.group(2)) 400 elapsed += (float(s.group(3)) * 60) + float(s.group(4)) 401 cpu += float(s.group(5)) 402 count += 1 403 except: 404 raise ValueError("badly formatted times") 405 406 f.close() 407 return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 408 (elapsed/count, user/count, system/count, cpu/count) 409 410 411def running_config(): 412 """ 413 Return path of config file of the currently running kernel 414 """ 415 version = utils.system_output('uname -r') 416 for config in ('/proc/config.gz', \ 417 '/boot/config-%s' % version, 418 '/lib/modules/%s/build/.config' % version): 419 if os.path.isfile(config): 420 return config 421 return None 422 423 424def check_for_kernel_feature(feature): 425 config = running_config() 426 427 if not config: 428 raise TypeError("Can't find kernel config file") 429 430 if config.endswith('.gz'): 431 grep = 'zgrep' 432 else: 433 grep = 'grep' 434 grep += ' ^CONFIG_%s= %s' % (feature, config) 435 436 if not utils.system_output(grep, ignore_status=True): 437 raise ValueError("Kernel doesn't have a %s feature" % (feature)) 438 439 440def cpu_online_map(): 441 """ 442 Check out the available cpu online map 443 """ 444 cpus = [] 445 for line in open('/proc/cpuinfo', 'r').readlines(): 446 if line.startswith('processor'): 447 cpus.append(line.split()[2]) # grab cpu number 448 return cpus 449 450 451def check_glibc_ver(ver): 452 glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 453 glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 454 if glibc_ver.split('.') < ver.split('.'): 455 raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % \ 456 (glibc_ver, ver)) 457 458def check_kernel_ver(ver): 459 kernel_ver = utils.system_output('uname -r') 460 kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 461 if kv_tmp[0].split('.') < ver.split('.'): 462 raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % \ 463 (kernel_ver, ver)) 464 465 466def human_format(number): 467 # Convert number to kilo / mega / giga format. 468 if number < 1024: 469 return "%d" % number 470 kilo = float(number) / 1024.0 471 if kilo < 1024: 472 return "%.2fk" % kilo 473 meg = kilo / 1024.0 474 if meg < 1024: 475 return "%.2fM" % meg 476 gig = meg / 1024.0 477 return "%.2fG" % gig 478 479 480def numa_nodes(): 481 node_paths = glob.glob('/sys/devices/system/node/node*') 482 nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 483 return (sorted(nodes)) 484 485 486def node_size(): 487 nodes = max(len(numa_nodes()), 1) 488 return ((memtotal() * 1024) / nodes) 489 490 491def to_seconds(time_string): 492 """Converts a string in M+:SS.SS format to S+.SS""" 493 elts = time_string.split(':') 494 if len(elts) == 1: 495 return time_string 496 return str(int(elts[0]) * 60 + float(elts[1])) 497 498 499def extract_all_time_results(results_string): 500 """Extract user, system, and elapsed times into a list of tuples""" 501 pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed") 502 results = [] 503 for result in pattern.findall(results_string): 504 results.append(tuple([to_seconds(elt) for elt in result])) 505 return results 506 507 508def pickle_load(filename): 509 return pickle.load(open(filename, 'r')) 510 511 512# Return the kernel version and build timestamp. 513def running_os_release(): 514 return os.uname()[2:4] 515 516 517def running_os_ident(): 518 (version, timestamp) = running_os_release() 519 return version + '::' + timestamp 520 521 522def running_os_full_version(): 523 (version, timestamp) = running_os_release() 524 return version 525 526 527# much like find . -name 'pattern' 528def locate(pattern, root=os.getcwd()): 529 for path, dirs, files in os.walk(root): 530 for f in files: 531 if fnmatch.fnmatch(f, pattern): 532 yield os.path.abspath(os.path.join(path, f)) 533 534 535def freespace(path): 536 """Return the disk free space, in bytes""" 537 s = os.statvfs(path) 538 return s.f_bavail * s.f_bsize 539 540 541def disk_block_size(path): 542 """Return the disk block size, in bytes""" 543 return os.statvfs(path).f_bsize 544 545 546def get_cpu_family(): 547 procinfo = utils.system_output('cat /proc/cpuinfo') 548 CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M) 549 matches = CPU_FAMILY_RE.findall(procinfo) 550 if matches: 551 return int(matches[0]) 552 else: 553 raise error.TestError('Could not get valid cpu family data') 554 555 556def get_disks(): 557 df_output = utils.system_output('df') 558 disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 559 return disk_re.findall(df_output) 560 561 562def load_module(module_name): 563 # Checks if a module has already been loaded 564 if module_is_loaded(module_name): 565 return False 566 567 utils.system('/sbin/modprobe ' + module_name) 568 return True 569 570 571def unload_module(module_name): 572 """ 573 Removes a module. Handles dependencies. If even then it's not possible 574 to remove one of the modules, it will trhow an error.CmdError exception. 575 576 @param module_name: Name of the module we want to remove. 577 """ 578 l_raw = utils.system_output("/sbin/lsmod").splitlines() 579 lsmod = [x for x in l_raw if x.split()[0] == module_name] 580 if len(lsmod) > 0: 581 line_parts = lsmod[0].split() 582 if len(line_parts) == 4: 583 submodules = line_parts[3].split(",") 584 for submodule in submodules: 585 unload_module(submodule) 586 utils.system("/sbin/modprobe -r %s" % module_name) 587 logging.info("Module %s unloaded" % module_name) 588 else: 589 logging.info("Module %s is already unloaded" % module_name) 590 591 592def module_is_loaded(module_name): 593 module_name = module_name.replace('-', '_') 594 modules = utils.system_output('/sbin/lsmod').splitlines() 595 for module in modules: 596 if module.startswith(module_name) and module[len(module_name)] == ' ': 597 return True 598 return False 599 600 601def get_loaded_modules(): 602 lsmod_output = utils.system_output('/sbin/lsmod').splitlines()[1:] 603 return [line.split(None, 1)[0] for line in lsmod_output] 604 605 606def get_huge_page_size(): 607 output = utils.system_output('grep Hugepagesize /proc/meminfo') 608 return int(output.split()[1]) # Assumes units always in kB. :( 609 610 611def get_num_huge_pages(): 612 raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') 613 return int(raw_hugepages.split()[2]) 614 615 616def set_num_huge_pages(num): 617 utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) 618 619 620def get_cpu_vendor(): 621 cpuinfo = open('/proc/cpuinfo').read() 622 vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo) 623 for i in xrange(1, len(vendors)): 624 if vendors[i] != vendors[0]: 625 raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 626 return vendors[0] 627 628 629def probe_cpus(): 630 """ 631 This routine returns a list of cpu devices found under 632 /sys/devices/system/cpu. 633 """ 634 cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 635 return utils.system_output(cmd).splitlines() 636 637 638def ping_default_gateway(): 639 """Ping the default gateway.""" 640 641 network = open('/etc/sysconfig/network') 642 m = re.search('GATEWAY=(\S+)', network.read()) 643 644 if m: 645 gw = m.group(1) 646 cmd = 'ping %s -c 5 > /dev/null' % gw 647 return utils.system(cmd, ignore_status=True) 648 649 raise error.TestError('Unable to find default gateway') 650 651 652def drop_caches(): 653 """Writes back all dirty pages to disk and clears all the caches.""" 654 utils.system("sync") 655 # We ignore failures here as this will fail on 2.6.11 kernels. 656 utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 657 658 659def process_is_alive(name_pattern): 660 """ 661 'pgrep name' misses all python processes and also long process names. 662 'pgrep -f name' gets all shell commands with name in args. 663 So look only for command whose initial pathname ends with name. 664 Name itself is an egrep pattern, so it can use | etc for variations. 665 """ 666 return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern, 667 ignore_status=True) == 0 668