utils.py revision 2f29c19882287b5533312b1c4367819bf3a53f91
1""" 2Convenience functions for use by tests or whomever. 3 4NOTE: this is a mixin library that pulls in functions from both here and 5client/common_lib/utils.py (which are functions shared with the server code) 6""" 7import os, shutil, sys, signal, commands, pickle, glob, statvfs 8import math, re, string, fnmatch 9from autotest_lib.client.common_lib import error 10from autotest_lib.client.common_lib.utils import * 11 12 13def grep(pattern, file): 14 """ 15 This is mainly to fix the return code inversion from grep 16 Also handles compressed files. 17 18 returns 1 if the pattern is present in the file, 0 if not. 19 """ 20 command = 'grep "%s" > /dev/null' % pattern 21 ret = cat_file_to_cmd(file, command, ignore_status=True) 22 return not ret 23 24 25def difflist(list1, list2): 26 """returns items in list2 that are not in list1""" 27 diff = []; 28 for x in list2: 29 if x not in list1: 30 diff.append(x) 31 return diff 32 33 34def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 35 """ 36 equivalent to 'cat file | command' but knows to use 37 zcat or bzcat if appropriate 38 """ 39 if not os.path.isfile(file): 40 raise NameError('invalid file %s to cat to command %s' 41 % (file, command)) 42 43 if return_output: 44 run_cmd = system_output 45 else: 46 run_cmd = system 47 48 if file.endswith('.bz2'): 49 cat = 'bzcat' 50 elif (file.endswith('.gz') or file.endswith('.tgz')): 51 cat = 'zcat' 52 else: 53 cat = 'cat' 54 return run_cmd('%s %s | %s' % (cat, file, command), 55 ignore_status=ignore_status) 56 57 58def extract_tarball_to_dir(tarball, dir): 59 """ 60 Extract a tarball to a specified directory name instead of whatever 61 the top level of a tarball is - useful for versioned directory names, etc 62 """ 63 if os.path.exists(dir): 64 raise NameError, 'target %s already exists' % dir 65 pwd = os.getcwd() 66 os.chdir(os.path.dirname(os.path.abspath(dir))) 67 newdir = extract_tarball(tarball) 68 os.rename(newdir, dir) 69 os.chdir(pwd) 70 71 72def extract_tarball(tarball): 73 """Returns the directory extracted by the tarball.""" 74 extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 75 return_output=True).splitlines() 76 77 dir = None 78 79 for line in extracted: 80 line = re.sub(r'^./', '', line) 81 if not line or line == '.': 82 continue 83 topdir = line.split('/')[0] 84 if os.path.isdir(topdir): 85 if dir: 86 assert(dir == topdir) 87 else: 88 dir = topdir 89 if dir: 90 return dir 91 else: 92 raise NameError('extracting tarball produced no dir') 93 94 95def get_md5sum(file_path): 96 """Gets the md5sum of a file. You must provide a valid path to the file""" 97 if not os.path.isfile(file_path): 98 raise ValueError, 'invalid file %s to verify' % file_path 99 md5sum = system_output("md5sum " + file_path) 100 return md5sum.split()[0] 101 102 103def unmap_url_cache(cachedir, url, expected_md5): 104 """ 105 Downloads a file from a URL to a cache directory. If the file is already 106 at the expected position and has the expected md5 number, let's not 107 download it again. 108 """ 109 # Let's convert cachedir to a canonical path, if it's not already 110 cachedir = os.path.realpath(cachedir) 111 if not os.path.isdir(cachedir): 112 try: 113 system('mkdir -p ' + cachedir) 114 except: 115 raise ValueError('Could not create cache directory %s' % cachedir) 116 file_from_url = os.path.basename(url) 117 file_local_path = os.path.join(cachedir, file_from_url) 118 if os.path.isfile(file_local_path): 119 file_md5 = get_md5sum(file_local_path) 120 if file_md5 == expected_md5: 121 # File is already at the expected position and ready to go 122 src = file_from_url 123 else: 124 # Let's download the package again, it's corrupted... 125 src = url 126 else: 127 # File is not there, let's download it 128 src = url 129 return unmap_url(cachedir, src, cachedir) 130 131 132def force_copy(src, dest): 133 """Replace dest with a new copy of src, even if it exists""" 134 if os.path.isfile(dest): 135 os.remove(dest) 136 if os.path.isdir(dest): 137 dest = os.path.join(dest, os.path.basename(src)) 138 shutil.copyfile(src, dest) 139 return dest 140 141 142def force_link(src, dest): 143 """Link src to dest, overwriting it if it exists""" 144 return system("ln -sf %s %s" % (src, dest)) 145 146 147def file_contains_pattern(file, pattern): 148 """Return true if file contains the specified egrep pattern""" 149 if not os.path.isfile(file): 150 raise NameError('file %s does not exist' % file) 151 return not system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 152 153 154def list_grep(list, pattern): 155 """True if any item in list matches the specified pattern.""" 156 compiled = re.compile(pattern) 157 for line in list: 158 match = compiled.search(line) 159 if (match): 160 return 1 161 return 0 162 163 164def get_os_vendor(): 165 """Try to guess what's the os vendor 166 """ 167 issue = '/etc/issue' 168 169 if not os.path.isfile(issue): 170 return 'Unknown' 171 172 if file_contains_pattern(issue, 'Red Hat'): 173 return 'Red Hat' 174 elif file_contains_pattern(issue, 'Fedora'): 175 return 'Fedora Core' 176 elif file_contains_pattern(issue, 'SUSE'): 177 return 'SUSE' 178 elif file_contains_pattern(issue, 'Ubuntu'): 179 return 'Ubuntu' 180 elif file_contains_pattern(issue, 'Debian'): 181 return 'Debian' 182 else: 183 return 'Unknown' 184 185 186def get_vmlinux(): 187 """Return the full path to vmlinux 188 189 Ahem. This is crap. Pray harder. Bad Martin. 190 """ 191 vmlinux = '/boot/vmlinux-%s' % system_output('uname -r') 192 if os.path.isfile(vmlinux): 193 return vmlinux 194 vmlinux = '/lib/modules/%s/build/vmlinux' % system_output('uname -r') 195 if os.path.isfile(vmlinux): 196 return vmlinux 197 return None 198 199 200def get_systemmap(): 201 """Return the full path to System.map 202 203 Ahem. This is crap. Pray harder. Bad Martin. 204 """ 205 map = '/boot/System.map-%s' % system_output('uname -r') 206 if os.path.isfile(map): 207 return map 208 map = '/lib/modules/%s/build/System.map' % system_output('uname -r') 209 if os.path.isfile(map): 210 return map 211 return None 212 213 214def get_modules_dir(): 215 """Return the modules dir for the running kernel version""" 216 kernel_version = system_output('uname -r') 217 return '/lib/modules/%s/kernel' % kernel_version 218 219 220def get_cpu_arch(): 221 """Work out which CPU architecture we're running on""" 222 f = open('/proc/cpuinfo', 'r') 223 cpuinfo = f.readlines() 224 f.close() 225 if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 226 return 'power' 227 elif list_grep(cpuinfo, '^cpu.*POWER4'): 228 return 'power4' 229 elif list_grep(cpuinfo, '^cpu.*POWER5'): 230 return 'power5' 231 elif list_grep(cpuinfo, '^cpu.*POWER6'): 232 return 'power6' 233 elif list_grep(cpuinfo, '^cpu.*PPC970'): 234 return 'power970' 235 elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 236 return 'x86_64' 237 else: 238 return 'i386' 239 240 241def get_current_kernel_arch(): 242 """Get the machine architecture, now just a wrap of 'uname -m'.""" 243 return os.popen('uname -m').read().rstrip() 244 245 246def get_file_arch(filename): 247 # -L means follow symlinks 248 file_data = system_output('file -L ' + filename) 249 if file_data.count('80386'): 250 return 'i386' 251 return None 252 253 254def count_cpus(): 255 """number of CPUs in the local machine according to /proc/cpuinfo""" 256 f = file('/proc/cpuinfo', 'r') 257 cpus = 0 258 for line in f.readlines(): 259 if line.startswith('processor'): 260 cpus += 1 261 return cpus 262 263 264# Returns total memory in kb 265def read_from_meminfo(key): 266 meminfo = system_output('grep %s /proc/meminfo' % key) 267 return int(re.search(r'\d+', meminfo).group(0)) 268 269 270def memtotal(): 271 return read_from_meminfo('MemTotal') 272 273 274def freememtotal(): 275 return read_from_meminfo('MemFree') 276 277 278def rounded_memtotal(): 279 # Get total of all physical mem, in kbytes 280 usable_kbytes = memtotal() 281 # usable_kbytes is system's usable DRAM in kbytes, 282 # as reported by memtotal() from device /proc/meminfo memtotal 283 # after Linux deducts 1.5% to 5.1% for system table overhead 284 # Undo the unknown actual deduction by rounding up 285 # to next small multiple of a big power-of-two 286 # eg 12GB - 5.1% gets rounded back up to 12GB 287 mindeduct = 0.015 # 1.5 percent 288 maxdeduct = 0.055 # 5.5 percent 289 # deduction range 1.5% .. 5.5% supports physical mem sizes 290 # 6GB .. 12GB in steps of .5GB 291 # 12GB .. 24GB in steps of 1 GB 292 # 24GB .. 48GB in steps of 2 GB ... 293 # Finer granularity in physical mem sizes would require 294 # tighter spread between min and max possible deductions 295 296 # increase mem size by at least min deduction, without rounding 297 min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 298 # increase mem size further by 2**n rounding, by 0..roundKb or more 299 round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 300 # find least binary roundup 2**n that covers worst-cast roundKb 301 mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 302 # have round_kbytes <= mod2n < round_kbytes*2 303 # round min_kbytes up to next multiple of mod2n 304 phys_kbytes = min_kbytes + mod2n - 1 305 phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 306 return phys_kbytes 307 308 309def sysctl_kernel(key, value=None): 310 """(Very) partial implementation of sysctl, for kernel params""" 311 if value: 312 # write 313 write_one_line('/proc/sys/kernel/%s' % key, str(value)) 314 else: 315 # read 316 out = read_one_line('/proc/sys/kernel/%s' % key) 317 return int(re.search(r'\d+', out).group(0)) 318 319 320def _convert_exit_status(sts): 321 if os.WIFSIGNALED(sts): 322 return -os.WTERMSIG(sts) 323 elif os.WIFEXITED(sts): 324 return os.WEXITSTATUS(sts) 325 else: 326 # impossible? 327 raise RuntimeError("Unknown exit status %d!" % sts) 328 329 330def where_art_thy_filehandles(): 331 """Dump the current list of filehandles""" 332 os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 333 334 335def print_to_tty(string): 336 """Output string straight to the tty""" 337 open('/dev/tty', 'w').write(string + '\n') 338 339 340def dump_object(object): 341 """Dump an object's attributes and methods 342 343 kind of like dir() 344 """ 345 for item in object.__dict__.iteritems(): 346 print item 347 try: 348 (key,value) = item 349 dump_object(value) 350 except: 351 continue 352 353 354def environ(env_key): 355 """return the requested environment variable, or '' if unset""" 356 if (os.environ.has_key(env_key)): 357 return os.environ[env_key] 358 else: 359 return '' 360 361 362def prepend_path(newpath, oldpath): 363 """prepend newpath to oldpath""" 364 if (oldpath): 365 return newpath + ':' + oldpath 366 else: 367 return newpath 368 369 370def append_path(oldpath, newpath): 371 """append newpath to oldpath""" 372 if (oldpath): 373 return oldpath + ':' + newpath 374 else: 375 return newpath 376 377 378def avgtime_print(dir): 379 """ Calculate some benchmarking statistics. 380 Input is a directory containing a file called 'time'. 381 File contains one-per-line results of /usr/bin/time. 382 Output is average Elapsed, User, and System time in seconds, 383 and average CPU percentage. 384 """ 385 f = open(dir + "/time") 386 user = system = elapsed = cpu = count = 0 387 r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU') 388 for line in f.readlines(): 389 try: 390 s = r.match(line); 391 user += float(s.group(1)) 392 system += float(s.group(2)) 393 elapsed += (float(s.group(3)) * 60) + float(s.group(4)) 394 cpu += float(s.group(5)) 395 count += 1 396 except: 397 raise ValueError("badly formatted times") 398 399 f.close() 400 return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 401 (elapsed/count, user/count, system/count, cpu/count) 402 403 404def running_config(): 405 """ 406 Return path of config file of the currently running kernel 407 """ 408 version = system_output('uname -r') 409 for config in ('/proc/config.gz', \ 410 '/boot/config-%s' % version, 411 '/lib/modules/%s/build/.config' % version): 412 if os.path.isfile(config): 413 return config 414 return None 415 416 417def check_for_kernel_feature(feature): 418 config = running_config() 419 420 if not config: 421 raise TypeError("Can't find kernel config file") 422 423 if config.endswith('.gz'): 424 grep = 'zgrep' 425 else: 426 grep = 'grep' 427 grep += ' ^CONFIG_%s= %s' % (feature, config) 428 429 if not system_output(grep, ignore_status=True): 430 raise ValueError("Kernel doesn't have a %s feature" % (feature)) 431 432 433def cpu_online_map(): 434 """ 435 Check out the available cpu online map 436 """ 437 cpus = [] 438 for line in open('/proc/cpuinfo', 'r').readlines(): 439 if line.startswith('processor'): 440 cpus.append(line.split()[2]) # grab cpu number 441 return cpus 442 443 444def check_glibc_ver(ver): 445 glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 446 glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 447 if glibc_ver.split('.') < ver.split('.'): 448 raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % \ 449 (glibc_ver, ver)) 450 451def check_kernel_ver(ver): 452 kernel_ver = system_output('uname -r') 453 kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 454 if kv_tmp[0].split('.') < ver.split('.'): 455 raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % \ 456 (kernel_ver, ver)) 457 458 459def human_format(number): 460 # Convert number to kilo / mega / giga format. 461 if number < 1024: 462 return "%d" % number 463 kilo = float(number) / 1024.0 464 if kilo < 1024: 465 return "%.2fk" % kilo 466 meg = kilo / 1024.0 467 if meg < 1024: 468 return "%.2fM" % meg 469 gig = meg / 1024.0 470 return "%.2fG" % gig 471 472 473def numa_nodes(): 474 node_paths = glob.glob('/sys/devices/system/node/node*') 475 nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 476 return (sorted(nodes)) 477 478 479def node_size(): 480 nodes = max(len(numa_nodes()), 1) 481 return ((memtotal() * 1024) / nodes) 482 483 484def to_seconds(time_string): 485 """Converts a string in M+:SS.SS format to S+.SS""" 486 elts = time_string.split(':') 487 if len(elts) == 1: 488 return time_string 489 return str(int(elts[0]) * 60 + float(elts[1])) 490 491 492def extract_all_time_results(results_string): 493 """Extract user, system, and elapsed times into a list of tuples""" 494 pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed") 495 results = [] 496 for result in pattern.findall(results_string): 497 results.append(tuple([to_seconds(elt) for elt in result])) 498 return results 499 500 501def pickle_load(filename): 502 return pickle.load(open(filename, 'r')) 503 504 505# Return the kernel version and build timestamp. 506def running_os_release(): 507 return os.uname()[2:4] 508 509 510def running_os_ident(): 511 (version, timestamp) = running_os_release() 512 return version + '::' + timestamp 513 514 515# much like find . -name 'pattern' 516def locate(pattern, root=os.getcwd()): 517 for path, dirs, files in os.walk(root): 518 for f in files: 519 if fnmatch.fnmatch(f, pattern): 520 yield os.path.abspath(os.path.join(path, f)) 521 522 523def freespace(path): 524 """Return the disk free space, in bytes""" 525 s = os.statvfs(path) 526 return s.f_bavail * s.f_bsize 527 528 529def disk_block_size(path): 530 """Return the disk block size, in bytes""" 531 return os.statvfs(path).f_bsize 532 533 534def get_cpu_family(): 535 procinfo = system_output('cat /proc/cpuinfo') 536 CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M) 537 matches = CPU_FAMILY_RE.findall(procinfo) 538 if matches: 539 return int(matches[0]) 540 else: 541 raise error.TestError('Could not get valid cpu family data') 542 543 544def get_disks(): 545 df_output = system_output('df') 546 disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 547 return disk_re.findall(df_output) 548 549 550def load_module(module_name): 551 # Checks if a module has already been loaded 552 if module_is_loaded(module_name): 553 return False 554 555 system('/sbin/modprobe ' + module_name) 556 return True 557 558 559def unload_module(module_name): 560 system('/sbin/rmmod ' + module_name) 561 562 563def module_is_loaded(module_name): 564 module_name = module_name.replace('-', '_') 565 modules = system_output('/sbin/lsmod').splitlines() 566 for module in modules: 567 if module.startswith(module_name) and module[len(module_name)] == ' ': 568 return True 569 return False 570 571 572def get_loaded_modules(): 573 lsmod_output = system_output('/sbin/lsmod').splitlines()[1:] 574 return [line.split(None, 1)[0] for line in lsmod_output] 575 576 577def get_huge_page_size(): 578 output = system_output('grep Hugepagesize /proc/meminfo') 579 return int(output.split()[1]) # Assumes units always in kB. :( 580 581 582def get_num_huge_pages(): 583 raw_hugepages = system_output('/sbin/sysctl vm.nr_hugepages') 584 return int(raw_hugepages.split()[2]) 585 586 587def set_num_huge_pages(num): 588 system('/sbin/sysctl vm.nr_hugepages=%d' % num) 589 590 591def get_system_nodes(): 592 nodes = os.listdir('/sys/devices/system/node') 593 nodes.sort() 594 return nodes 595 596 597def get_cpu_vendor(): 598 cpuinfo = open('/proc/cpuinfo').read() 599 vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo) 600 for i in xrange(1, len(vendors)): 601 if vendors[i] != vendors[0]: 602 raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 603 return vendors[0] 604 605 606def probe_cpus(): 607 """ 608 This routine returns a list of cpu devices found under 609 /sys/devices/system/cpu. 610 """ 611 cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 612 return system_output(cmd).splitlines() 613 614 615def ping_default_gateway(): 616 """Ping the default gateway.""" 617 618 network = open('/etc/sysconfig/network') 619 m = re.search('GATEWAY=(\S+)', network.read()) 620 621 if m: 622 gw = m.group(1) 623 cmd = 'ping %s -c 5 > /dev/null' % gw 624 return system(cmd, ignore_status=True) 625 626 raise error.TestError('Unable to find default gateway') 627 628 629def drop_caches(): 630 """Writes back all dirty pages to disk and clears all the caches.""" 631 system("sync") 632 system("sync") 633 # We ignore failures here as this will fail on 2.6.11 kernels. 634 system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 635 636 637