base_utils.py revision 0758c82ce9ef641b746b5b9a2d0ce409ba6169c7
1""" 2DO NOT import this file directly - import client/bin/utils.py, 3which will mix this in 4 5Convenience functions for use by tests or whomever. 6 7Note that this file is mixed in by utils.py - note very carefully the 8precedence order defined there 9""" 10import os, shutil, sys, signal, commands, pickle, glob, statvfs 11import math, re, string, fnmatch, logging 12from autotest_lib.client.common_lib import error, utils 13 14 15def grep(pattern, file): 16 """ 17 This is mainly to fix the return code inversion from grep 18 Also handles compressed files. 19 20 returns 1 if the pattern is present in the file, 0 if not. 21 """ 22 command = 'grep "%s" > /dev/null' % pattern 23 ret = cat_file_to_cmd(file, command, ignore_status=True) 24 return not ret 25 26 27def difflist(list1, list2): 28 """returns items in list2 that are not in list1""" 29 diff = []; 30 for x in list2: 31 if x not in list1: 32 diff.append(x) 33 return diff 34 35 36def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 37 """ 38 equivalent to 'cat file | command' but knows to use 39 zcat or bzcat if appropriate 40 """ 41 if not os.path.isfile(file): 42 raise NameError('invalid file %s to cat to command %s' 43 % (file, command)) 44 45 if return_output: 46 run_cmd = utils.system_output 47 else: 48 run_cmd = utils.system 49 50 if file.endswith('.bz2'): 51 cat = 'bzcat' 52 elif (file.endswith('.gz') or file.endswith('.tgz')): 53 cat = 'zcat' 54 else: 55 cat = 'cat' 56 return run_cmd('%s %s | %s' % (cat, file, command), 57 ignore_status=ignore_status) 58 59 60def extract_tarball_to_dir(tarball, dir): 61 """ 62 Extract a tarball to a specified directory name instead of whatever 63 the top level of a tarball is - useful for versioned directory names, etc 64 """ 65 if os.path.exists(dir): 66 raise NameError, 'target %s already exists' % dir 67 pwd = os.getcwd() 68 os.chdir(os.path.dirname(os.path.abspath(dir))) 69 newdir = extract_tarball(tarball) 70 os.rename(newdir, dir) 71 os.chdir(pwd) 72 73 74def extract_tarball(tarball): 75 """Returns the directory extracted by the tarball.""" 76 extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 77 return_output=True).splitlines() 78 79 dir = None 80 81 for line in extracted: 82 line = re.sub(r'^./', '', line) 83 if not line or line == '.': 84 continue 85 topdir = line.split('/')[0] 86 if os.path.isdir(topdir): 87 if dir: 88 assert(dir == topdir) 89 else: 90 dir = topdir 91 if dir: 92 return dir 93 else: 94 raise NameError('extracting tarball produced no dir') 95 96 97def get_md5sum(file_path): 98 """Gets the md5sum of a file. You must provide a valid path to the file""" 99 if not os.path.isfile(file_path): 100 raise ValueError, 'invalid file %s to verify' % file_path 101 md5sum = utils.system_output("md5sum " + file_path) 102 return md5sum.split()[0] 103 104 105def unmap_url_cache(cachedir, url, expected_md5): 106 """ 107 Downloads a file from a URL to a cache directory. If the file is already 108 at the expected position and has the expected md5 number, let's not 109 download it again. 110 """ 111 # Let's convert cachedir to a canonical path, if it's not already 112 cachedir = os.path.realpath(cachedir) 113 if not os.path.isdir(cachedir): 114 try: 115 utils.system('mkdir -p ' + cachedir) 116 except: 117 raise ValueError('Could not create cache directory %s' % cachedir) 118 file_from_url = os.path.basename(url) 119 file_local_path = os.path.join(cachedir, file_from_url) 120 if os.path.isfile(file_local_path): 121 file_md5 = get_md5sum(file_local_path) 122 if file_md5 == expected_md5: 123 # File is already at the expected position and ready to go 124 src = file_from_url 125 else: 126 # Let's download the package again, it's corrupted... 127 src = url 128 else: 129 # File is not there, let's download it 130 src = url 131 return utils.unmap_url(cachedir, src, cachedir) 132 133 134def force_copy(src, dest): 135 """Replace dest with a new copy of src, even if it exists""" 136 if os.path.isfile(dest): 137 os.remove(dest) 138 if os.path.isdir(dest): 139 dest = os.path.join(dest, os.path.basename(src)) 140 shutil.copyfile(src, dest) 141 return dest 142 143 144def force_link(src, dest): 145 """Link src to dest, overwriting it if it exists""" 146 return utils.system("ln -sf %s %s" % (src, dest)) 147 148 149def file_contains_pattern(file, pattern): 150 """Return true if file contains the specified egrep pattern""" 151 if not os.path.isfile(file): 152 raise NameError('file %s does not exist' % file) 153 return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 154 155 156def list_grep(list, pattern): 157 """True if any item in list matches the specified pattern.""" 158 compiled = re.compile(pattern) 159 for line in list: 160 match = compiled.search(line) 161 if (match): 162 return 1 163 return 0 164 165 166def get_os_vendor(): 167 """Try to guess what's the os vendor 168 """ 169 if os.path.isfile('/etc/SuSE-release'): 170 return 'SUSE' 171 172 issue = '/etc/issue' 173 174 if not os.path.isfile(issue): 175 return 'Unknown' 176 177 if file_contains_pattern(issue, 'Red Hat'): 178 return 'Red Hat' 179 elif file_contains_pattern(issue, 'Fedora'): 180 return 'Fedora Core' 181 elif file_contains_pattern(issue, 'SUSE'): 182 return 'SUSE' 183 elif file_contains_pattern(issue, 'Ubuntu'): 184 return 'Ubuntu' 185 elif file_contains_pattern(issue, 'Debian'): 186 return 'Debian' 187 else: 188 return 'Unknown' 189 190 191def get_vmlinux(): 192 """Return the full path to vmlinux 193 194 Ahem. This is crap. Pray harder. Bad Martin. 195 """ 196 vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') 197 if os.path.isfile(vmlinux): 198 return vmlinux 199 vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') 200 if os.path.isfile(vmlinux): 201 return vmlinux 202 return None 203 204 205def get_systemmap(): 206 """Return the full path to System.map 207 208 Ahem. This is crap. Pray harder. Bad Martin. 209 """ 210 map = '/boot/System.map-%s' % utils.system_output('uname -r') 211 if os.path.isfile(map): 212 return map 213 map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') 214 if os.path.isfile(map): 215 return map 216 return None 217 218 219def get_modules_dir(): 220 """Return the modules dir for the running kernel version""" 221 kernel_version = utils.system_output('uname -r') 222 return '/lib/modules/%s/kernel' % kernel_version 223 224 225def get_cpu_arch(): 226 """Work out which CPU architecture we're running on""" 227 f = open('/proc/cpuinfo', 'r') 228 cpuinfo = f.readlines() 229 f.close() 230 if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 231 return 'power' 232 elif list_grep(cpuinfo, '^cpu.*POWER4'): 233 return 'power4' 234 elif list_grep(cpuinfo, '^cpu.*POWER5'): 235 return 'power5' 236 elif list_grep(cpuinfo, '^cpu.*POWER6'): 237 return 'power6' 238 elif list_grep(cpuinfo, '^cpu.*POWER7'): 239 return 'power7' 240 elif list_grep(cpuinfo, '^cpu.*PPC970'): 241 return 'power970' 242 elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 243 return 'x86_64' 244 else: 245 return 'i386' 246 247 248def get_current_kernel_arch(): 249 """Get the machine architecture, now just a wrap of 'uname -m'.""" 250 return os.popen('uname -m').read().rstrip() 251 252 253def get_file_arch(filename): 254 # -L means follow symlinks 255 file_data = utils.system_output('file -L ' + filename) 256 if file_data.count('80386'): 257 return 'i386' 258 return None 259 260 261def count_cpus(): 262 """number of CPUs in the local machine according to /proc/cpuinfo""" 263 f = file('/proc/cpuinfo', 'r') 264 cpus = 0 265 for line in f.readlines(): 266 if line.startswith('processor'): 267 cpus += 1 268 return cpus 269 270 271# Returns total memory in kb 272def read_from_meminfo(key): 273 meminfo = utils.system_output('grep %s /proc/meminfo' % key) 274 return int(re.search(r'\d+', meminfo).group(0)) 275 276 277def memtotal(): 278 return read_from_meminfo('MemTotal') 279 280 281def freememtotal(): 282 return read_from_meminfo('MemFree') 283 284 285def rounded_memtotal(): 286 # Get total of all physical mem, in kbytes 287 usable_kbytes = memtotal() 288 # usable_kbytes is system's usable DRAM in kbytes, 289 # as reported by memtotal() from device /proc/meminfo memtotal 290 # after Linux deducts 1.5% to 5.1% for system table overhead 291 # Undo the unknown actual deduction by rounding up 292 # to next small multiple of a big power-of-two 293 # eg 12GB - 5.1% gets rounded back up to 12GB 294 mindeduct = 0.015 # 1.5 percent 295 maxdeduct = 0.055 # 5.5 percent 296 # deduction range 1.5% .. 5.5% supports physical mem sizes 297 # 6GB .. 12GB in steps of .5GB 298 # 12GB .. 24GB in steps of 1 GB 299 # 24GB .. 48GB in steps of 2 GB ... 300 # Finer granularity in physical mem sizes would require 301 # tighter spread between min and max possible deductions 302 303 # increase mem size by at least min deduction, without rounding 304 min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 305 # increase mem size further by 2**n rounding, by 0..roundKb or more 306 round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 307 # find least binary roundup 2**n that covers worst-cast roundKb 308 mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 309 # have round_kbytes <= mod2n < round_kbytes*2 310 # round min_kbytes up to next multiple of mod2n 311 phys_kbytes = min_kbytes + mod2n - 1 312 phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 313 return phys_kbytes 314 315 316def sysctl_kernel(key, value=None): 317 """(Very) partial implementation of sysctl, for kernel params""" 318 if value: 319 # write 320 utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) 321 else: 322 # read 323 out = utils.read_one_line('/proc/sys/kernel/%s' % key) 324 return int(re.search(r'\d+', out).group(0)) 325 326 327def _convert_exit_status(sts): 328 if os.WIFSIGNALED(sts): 329 return -os.WTERMSIG(sts) 330 elif os.WIFEXITED(sts): 331 return os.WEXITSTATUS(sts) 332 else: 333 # impossible? 334 raise RuntimeError("Unknown exit status %d!" % sts) 335 336 337def where_art_thy_filehandles(): 338 """Dump the current list of filehandles""" 339 os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 340 341 342def print_to_tty(string): 343 """Output string straight to the tty""" 344 open('/dev/tty', 'w').write(string + '\n') 345 346 347def dump_object(object): 348 """Dump an object's attributes and methods 349 350 kind of like dir() 351 """ 352 for item in object.__dict__.iteritems(): 353 print item 354 try: 355 (key,value) = item 356 dump_object(value) 357 except: 358 continue 359 360 361def environ(env_key): 362 """return the requested environment variable, or '' if unset""" 363 if (os.environ.has_key(env_key)): 364 return os.environ[env_key] 365 else: 366 return '' 367 368 369def prepend_path(newpath, oldpath): 370 """prepend newpath to oldpath""" 371 if (oldpath): 372 return newpath + ':' + oldpath 373 else: 374 return newpath 375 376 377def append_path(oldpath, newpath): 378 """append newpath to oldpath""" 379 if (oldpath): 380 return oldpath + ':' + newpath 381 else: 382 return newpath 383 384 385def avgtime_print(dir): 386 """ Calculate some benchmarking statistics. 387 Input is a directory containing a file called 'time'. 388 File contains one-per-line results of /usr/bin/time. 389 Output is average Elapsed, User, and System time in seconds, 390 and average CPU percentage. 391 """ 392 f = open(dir + "/time") 393 user = system = elapsed = cpu = count = 0 394 r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU') 395 for line in f.readlines(): 396 try: 397 s = r.match(line); 398 user += float(s.group(1)) 399 system += float(s.group(2)) 400 elapsed += (float(s.group(3)) * 60) + float(s.group(4)) 401 cpu += float(s.group(5)) 402 count += 1 403 except: 404 raise ValueError("badly formatted times") 405 406 f.close() 407 return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 408 (elapsed/count, user/count, system/count, cpu/count) 409 410 411def running_config(): 412 """ 413 Return path of config file of the currently running kernel 414 """ 415 version = utils.system_output('uname -r') 416 for config in ('/proc/config.gz', \ 417 '/boot/config-%s' % version, 418 '/lib/modules/%s/build/.config' % version): 419 if os.path.isfile(config): 420 return config 421 return None 422 423 424def check_for_kernel_feature(feature): 425 config = running_config() 426 427 if not config: 428 raise TypeError("Can't find kernel config file") 429 430 if config.endswith('.gz'): 431 grep = 'zgrep' 432 else: 433 grep = 'grep' 434 grep += ' ^CONFIG_%s= %s' % (feature, config) 435 436 if not utils.system_output(grep, ignore_status=True): 437 raise ValueError("Kernel doesn't have a %s feature" % (feature)) 438 439 440def cpu_online_map(): 441 """ 442 Check out the available cpu online map 443 """ 444 cpus = [] 445 for line in open('/proc/cpuinfo', 'r').readlines(): 446 if line.startswith('processor'): 447 cpus.append(line.split()[2]) # grab cpu number 448 return cpus 449 450 451def check_glibc_ver(ver): 452 glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 453 glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 454 if glibc_ver.split('.') < ver.split('.'): 455 raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % \ 456 (glibc_ver, ver)) 457 458def check_kernel_ver(ver): 459 kernel_ver = utils.system_output('uname -r') 460 kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 461 if kv_tmp[0].split('.') < ver.split('.'): 462 raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % \ 463 (kernel_ver, ver)) 464 465 466def human_format(number): 467 # Convert number to kilo / mega / giga format. 468 if number < 1024: 469 return "%d" % number 470 kilo = float(number) / 1024.0 471 if kilo < 1024: 472 return "%.2fk" % kilo 473 meg = kilo / 1024.0 474 if meg < 1024: 475 return "%.2fM" % meg 476 gig = meg / 1024.0 477 return "%.2fG" % gig 478 479 480def numa_nodes(): 481 node_paths = glob.glob('/sys/devices/system/node/node*') 482 nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 483 return (sorted(nodes)) 484 485 486def node_size(): 487 nodes = max(len(numa_nodes()), 1) 488 return ((memtotal() * 1024) / nodes) 489 490 491def to_seconds(time_string): 492 """Converts a string in M+:SS.SS format to S+.SS""" 493 elts = time_string.split(':') 494 if len(elts) == 1: 495 return time_string 496 return str(int(elts[0]) * 60 + float(elts[1])) 497 498 499def extract_all_time_results(results_string): 500 """Extract user, system, and elapsed times into a list of tuples""" 501 pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed") 502 results = [] 503 for result in pattern.findall(results_string): 504 results.append(tuple([to_seconds(elt) for elt in result])) 505 return results 506 507 508def pickle_load(filename): 509 return pickle.load(open(filename, 'r')) 510 511 512# Return the kernel version and build timestamp. 513def running_os_release(): 514 return os.uname()[2:4] 515 516 517def running_os_ident(): 518 (version, timestamp) = running_os_release() 519 return version + '::' + timestamp 520 521 522def running_os_full_version(): 523 (version, timestamp) = running_os_release() 524 return version 525 526 527# much like find . -name 'pattern' 528def locate(pattern, root=os.getcwd()): 529 for path, dirs, files in os.walk(root): 530 for f in files: 531 if fnmatch.fnmatch(f, pattern): 532 yield os.path.abspath(os.path.join(path, f)) 533 534 535def freespace(path): 536 """Return the disk free space, in bytes""" 537 s = os.statvfs(path) 538 return s.f_bavail * s.f_bsize 539 540 541def disk_block_size(path): 542 """Return the disk block size, in bytes""" 543 return os.statvfs(path).f_bsize 544 545 546def get_cpu_family(): 547 procinfo = utils.system_output('cat /proc/cpuinfo') 548 CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M) 549 matches = CPU_FAMILY_RE.findall(procinfo) 550 if matches: 551 return int(matches[0]) 552 else: 553 raise error.TestError('Could not get valid cpu family data') 554 555 556def get_disks(): 557 df_output = utils.system_output('df') 558 disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 559 return disk_re.findall(df_output) 560 561 562def load_module(module_name): 563 # Checks if a module has already been loaded 564 if module_is_loaded(module_name): 565 return False 566 567 utils.system('/sbin/modprobe ' + module_name) 568 return True 569 570 571def unload_module(module_name): 572 """ 573 Removes a module. Handles dependencies. If even then it's not possible 574 to remove one of the modules, it will trhow an error.CmdError exception. 575 576 @param module_name: Name of the module we want to remove. 577 """ 578 l_raw = utils.system_output("/sbin/lsmod").splitlines() 579 lsmod = [x for x in l_raw if x.split()[0] == module_name] 580 if len(lsmod) > 0: 581 line_parts = lsmod[0].split() 582 if len(line_parts) == 4: 583 submodules = line_parts[3].split(",") 584 for submodule in submodules: 585 unload_module(submodule) 586 utils.system("/sbin/modprobe -r %s" % module_name) 587 logging.info("Module %s unloaded" % module_name) 588 else: 589 logging.info("Module %s is already unloaded" % module_name) 590 591 592def module_is_loaded(module_name): 593 module_name = module_name.replace('-', '_') 594 modules = utils.system_output('/sbin/lsmod').splitlines() 595 for module in modules: 596 if module.startswith(module_name) and module[len(module_name)] == ' ': 597 return True 598 return False 599 600 601def get_loaded_modules(): 602 lsmod_output = utils.system_output('/sbin/lsmod').splitlines()[1:] 603 return [line.split(None, 1)[0] for line in lsmod_output] 604 605 606def get_huge_page_size(): 607 output = utils.system_output('grep Hugepagesize /proc/meminfo') 608 return int(output.split()[1]) # Assumes units always in kB. :( 609 610 611def get_num_huge_pages(): 612 raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') 613 return int(raw_hugepages.split()[2]) 614 615 616def set_num_huge_pages(num): 617 utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) 618 619 620def get_cpu_vendor(): 621 cpuinfo = open('/proc/cpuinfo').read() 622 vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo) 623 for i in xrange(1, len(vendors)): 624 if vendors[i] != vendors[0]: 625 raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 626 return vendors[0] 627 628 629def probe_cpus(): 630 """ 631 This routine returns a list of cpu devices found under 632 /sys/devices/system/cpu. 633 """ 634 cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 635 return utils.system_output(cmd).splitlines() 636 637 638def ping_default_gateway(): 639 """Ping the default gateway.""" 640 641 network = open('/etc/sysconfig/network') 642 m = re.search('GATEWAY=(\S+)', network.read()) 643 644 if m: 645 gw = m.group(1) 646 cmd = 'ping %s -c 5 > /dev/null' % gw 647 return utils.system(cmd, ignore_status=True) 648 649 raise error.TestError('Unable to find default gateway') 650 651 652def drop_caches(): 653 """Writes back all dirty pages to disk and clears all the caches.""" 654 utils.system("sync") 655 utils.system("sync") 656 # We ignore failures here as this will fail on 2.6.11 kernels. 657 utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 658 659 660def process_is_alive(name): 661 """ 662 'pgrep name' misses all python processes and also long process names. 663 'pgrep -f name' gets all shell commands with name in args. 664 So look only for command whose first nonblank word ends with name. 665 """ 666 return utils.system("pgrep -f '^[^ ]*%s\W'" % name, 667 ignore_status=True) == 0 668