base_utils.py revision b923fc2a01db6e4548ffd541d86f9be49bc5805a
1c86b0b45cd0198d99d271a79f8b6de29feb98cd5mbligh""" 22da4d885be334f7db795c3f017daa06ff8ea703dmblighDO NOT import this file directly - import client/bin/utils.py, 32da4d885be334f7db795c3f017daa06ff8ea703dmblighwhich will mix this in 43bf79ca4d5490f9af09e1ddc39b8df5343b34d06mbligh 52da4d885be334f7db795c3f017daa06ff8ea703dmblighConvenience functions for use by tests or whomever. 63bf79ca4d5490f9af09e1ddc39b8df5343b34d06mbligh 72da4d885be334f7db795c3f017daa06ff8ea703dmblighNote that this file is mixed in by utils.py - note very carefully the 82da4d885be334f7db795c3f017daa06ff8ea703dmblighprecedence order defined there 953da18eddf69243ca175d9a4603cba5b55300726mbligh""" 109fb37cba7a49579cf484350b9b1e1a1135d6af0emblighimport os, shutil, sys, signal, commands, pickle, glob, statvfs 11b923fc2a01db6e4548ffd541d86f9be49bc5805almrimport math, re, string, fnmatch, logging 122da4d885be334f7db795c3f017daa06ff8ea703dmblighfrom autotest_lib.client.common_lib import error, utils 13ea397bbc85f1a3eda39c9f2ef4fc209b4ff336f4mbligh 14ea397bbc85f1a3eda39c9f2ef4fc209b4ff336f4mbligh 15f4c35322b200d65f41a8332b4f3503beb497840dmblighdef grep(pattern, file): 160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski This is mainly to fix the return code inversion from grep 180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Also handles compressed files. 197bdbfbdf088f9c2a72c199d5a530194bb2ac257fmbligh 200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski returns 1 if the pattern is present in the file, 0 if not. 210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski command = 'grep "%s" > /dev/null' % pattern 230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski ret = cat_file_to_cmd(file, command, ignore_status=True) 240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return not ret 25af4efc23471265a3bfa7fef0146fbc0cbb54515bmbligh 26af4efc23471265a3bfa7fef0146fbc0cbb54515bmbligh 27c86b0b45cd0198d99d271a79f8b6de29feb98cd5mblighdef difflist(list1, list2): 280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """returns items in list2 that are not in list1""" 290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski diff = []; 300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for x in list2: 310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if x not in list1: 320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski diff.append(x) 330afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return diff 34f4c35322b200d65f41a8332b4f3503beb497840dmbligh 35c86b0b45cd0198d99d271a79f8b6de29feb98cd5mbligh 368ea61e2f9f55bd9f646735069e1f029e86d08320mblighdef cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski equivalent to 'cat file | command' but knows to use 390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski zcat or bzcat if appropriate 400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 41f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh if not os.path.isfile(file): 42f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh raise NameError('invalid file %s to cat to command %s' 43f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh % (file, command)) 44f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh 450afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if return_output: 462da4d885be334f7db795c3f017daa06ff8ea703dmbligh run_cmd = utils.system_output 470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 482da4d885be334f7db795c3f017daa06ff8ea703dmbligh run_cmd = utils.system 490afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if file.endswith('.bz2'): 51f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh cat = 'bzcat' 520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif (file.endswith('.gz') or file.endswith('.tgz')): 53f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh cat = 'zcat' 540afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 55f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh cat = 'cat' 56f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh return run_cmd('%s %s | %s' % (cat, file, command), 57f2fa471f5306887847024a50ebfca57dd0c6bd71mbligh ignore_status=ignore_status) 58712cd145672c0033a55173717787c54cd68e6b83mbligh 59c86b0b45cd0198d99d271a79f8b6de29feb98cd5mbligh 60712cd145672c0033a55173717787c54cd68e6b83mblighdef extract_tarball_to_dir(tarball, dir): 610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Extract a tarball to a specified directory name instead of whatever 630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski the top level of a tarball is - useful for versioned directory names, etc 640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 650afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.exists(dir): 660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise NameError, 'target %s already exists' % dir 670afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski pwd = os.getcwd() 680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.chdir(os.path.dirname(os.path.abspath(dir))) 690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski newdir = extract_tarball(tarball) 700afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.rename(newdir, dir) 710afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.chdir(pwd) 72712cd145672c0033a55173717787c54cd68e6b83mbligh 73f4c35322b200d65f41a8332b4f3503beb497840dmbligh 74712cd145672c0033a55173717787c54cd68e6b83mblighdef extract_tarball(tarball): 750afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Returns the directory extracted by the tarball.""" 760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 770afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return_output=True).splitlines() 780afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 790afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski dir = None 800afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for line in extracted: 820afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski line = re.sub(r'^./', '', line) 830afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not line or line == '.': 840afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski continue 850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski topdir = line.split('/')[0] 860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isdir(topdir): 870afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if dir: 880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski assert(dir == topdir) 890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 900afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski dir = topdir 910afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if dir: 920afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return dir 930afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise NameError('extracting tarball produced no dir') 95712cd145672c0033a55173717787c54cd68e6b83mbligh 96cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 9760418bb82d1734a7f73a5e4c8870d06799795397mblighdef get_md5sum(file_path): 980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Gets the md5sum of a file. You must provide a valid path to the file""" 990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not os.path.isfile(file_path): 1000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise ValueError, 'invalid file %s to verify' % file_path 1012da4d885be334f7db795c3f017daa06ff8ea703dmbligh md5sum = utils.system_output("md5sum " + file_path) 10253da18eddf69243ca175d9a4603cba5b55300726mbligh return md5sum.split()[0] 10360418bb82d1734a7f73a5e4c8870d06799795397mbligh 10460418bb82d1734a7f73a5e4c8870d06799795397mbligh 10560418bb82d1734a7f73a5e4c8870d06799795397mblighdef unmap_url_cache(cachedir, url, expected_md5): 10653da18eddf69243ca175d9a4603cba5b55300726mbligh """ 1070afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Downloads a file from a URL to a cache directory. If the file is already 1080afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski at the expected position and has the expected md5 number, let's not 1090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski download it again. 1100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 1110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # Let's convert cachedir to a canonical path, if it's not already 1120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cachedir = os.path.realpath(cachedir) 1130afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not os.path.isdir(cachedir): 1140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 1152da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.system('mkdir -p ' + cachedir) 1160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except: 1170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise ValueError('Could not create cache directory %s' % cachedir) 1180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski file_from_url = os.path.basename(url) 1190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski file_local_path = os.path.join(cachedir, file_from_url) 1200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(file_local_path): 1210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski file_md5 = get_md5sum(file_local_path) 1220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if file_md5 == expected_md5: 1230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # File is already at the expected position and ready to go 1240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski src = file_from_url 1250afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 1260afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # Let's download the package again, it's corrupted... 1270afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski src = url 1280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 1290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # File is not there, let's download it 1300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski src = url 1312da4d885be334f7db795c3f017daa06ff8ea703dmbligh return utils.unmap_url(cachedir, src, cachedir) 132ea30c8a19ee292ee57b5223ca079f7a311195b41mbligh 133ea30c8a19ee292ee57b5223ca079f7a311195b41mbligh 134f4c35322b200d65f41a8332b4f3503beb497840dmblighdef force_copy(src, dest): 1350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Replace dest with a new copy of src, even if it exists""" 1360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(dest): 1370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.remove(dest) 1380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isdir(dest): 1390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski dest = os.path.join(dest, os.path.basename(src)) 1400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski shutil.copyfile(src, dest) 1410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return dest 142f4c35322b200d65f41a8332b4f3503beb497840dmbligh 143f4c35322b200d65f41a8332b4f3503beb497840dmbligh 144fdbcaec15092d8b4af80970c495038bdf9b0e63fmblighdef force_link(src, dest): 1450afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Link src to dest, overwriting it if it exists""" 1462da4d885be334f7db795c3f017daa06ff8ea703dmbligh return utils.system("ln -sf %s %s" % (src, dest)) 147fdbcaec15092d8b4af80970c495038bdf9b0e63fmbligh 148fdbcaec15092d8b4af80970c495038bdf9b0e63fmbligh 149cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmblighdef file_contains_pattern(file, pattern): 1500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Return true if file contains the specified egrep pattern""" 1510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not os.path.isfile(file): 1520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise NameError('file %s does not exist' % file) 1532da4d885be334f7db795c3f017daa06ff8ea703dmbligh return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 154cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 155cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 156cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmblighdef list_grep(list, pattern): 1570afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """True if any item in list matches the specified pattern.""" 1580afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski compiled = re.compile(pattern) 1590afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for line in list: 1600afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski match = compiled.search(line) 1610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if (match): 1620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 1 1630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 0 164cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 165987071e1fba830910658ce987bb803d142480c03mbligh 16642b81ca63a0ab336b844e9c5cce6fe30dae85357mblighdef get_os_vendor(): 1670afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Try to guess what's the os vendor 1680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 16990b97ff1834d2c65df82d9b830be7ebb1dbf2167mbligh if os.path.isfile('/etc/SuSE-release'): 17090b97ff1834d2c65df82d9b830be7ebb1dbf2167mbligh return 'SUSE' 17190b97ff1834d2c65df82d9b830be7ebb1dbf2167mbligh 1720afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski issue = '/etc/issue' 1730afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1740afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not os.path.isfile(issue): 1750afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'Unknown' 1760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 1770afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if file_contains_pattern(issue, 'Red Hat'): 1780afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'Red Hat' 1792f29c19882287b5533312b1c4367819bf3a53f91mbligh elif file_contains_pattern(issue, 'Fedora'): 1800afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'Fedora Core' 1810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif file_contains_pattern(issue, 'SUSE'): 1820afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'SUSE' 1830afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif file_contains_pattern(issue, 'Ubuntu'): 1840afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'Ubuntu' 1850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif file_contains_pattern(issue, 'Debian'): 1860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'Debian' 1870afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 1880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'Unknown' 189af4efc23471265a3bfa7fef0146fbc0cbb54515bmbligh 190cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 191f49d5cfa9fc49a34b3b5e5e1cb97ce2c0a77dd0cmblighdef get_vmlinux(): 1920afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Return the full path to vmlinux 193c86b0b45cd0198d99d271a79f8b6de29feb98cd5mbligh 1940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Ahem. This is crap. Pray harder. Bad Martin. 1950afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 1962da4d885be334f7db795c3f017daa06ff8ea703dmbligh vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') 1970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(vmlinux): 1980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return vmlinux 1992da4d885be334f7db795c3f017daa06ff8ea703dmbligh vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') 2000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(vmlinux): 2010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return vmlinux 2020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 203f49d5cfa9fc49a34b3b5e5e1cb97ce2c0a77dd0cmbligh 204f49d5cfa9fc49a34b3b5e5e1cb97ce2c0a77dd0cmbligh 205f49d5cfa9fc49a34b3b5e5e1cb97ce2c0a77dd0cmblighdef get_systemmap(): 2060afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Return the full path to System.map 207c86b0b45cd0198d99d271a79f8b6de29feb98cd5mbligh 2080afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Ahem. This is crap. Pray harder. Bad Martin. 2090afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 2102da4d885be334f7db795c3f017daa06ff8ea703dmbligh map = '/boot/System.map-%s' % utils.system_output('uname -r') 2110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(map): 2120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return map 2132da4d885be334f7db795c3f017daa06ff8ea703dmbligh map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') 2140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(map): 2150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return map 2160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 21767b5ece7667fc71429ee05b07ac7654ed5a05df7mbligh 21867b5ece7667fc71429ee05b07ac7654ed5a05df7mbligh 21967b5ece7667fc71429ee05b07ac7654ed5a05df7mblighdef get_modules_dir(): 2200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Return the modules dir for the running kernel version""" 2212da4d885be334f7db795c3f017daa06ff8ea703dmbligh kernel_version = utils.system_output('uname -r') 2220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return '/lib/modules/%s/kernel' % kernel_version 223f49d5cfa9fc49a34b3b5e5e1cb97ce2c0a77dd0cmbligh 224f49d5cfa9fc49a34b3b5e5e1cb97ce2c0a77dd0cmbligh 2255970cf057674accdb00a7742eae50e40c57e52d0mblighdef get_cpu_arch(): 2260afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Work out which CPU architecture we're running on""" 2270afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski f = open('/proc/cpuinfo', 'r') 2280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpuinfo = f.readlines() 2290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski f.close() 2300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 2310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'power' 2320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif list_grep(cpuinfo, '^cpu.*POWER4'): 2330afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'power4' 2340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif list_grep(cpuinfo, '^cpu.*POWER5'): 2350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'power5' 2360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif list_grep(cpuinfo, '^cpu.*POWER6'): 2370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'power6' 2380afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif list_grep(cpuinfo, '^cpu.*PPC970'): 2390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'power970' 24012e8b89ac5a26bf29bc89c4d19f2771fa715f52dmbligh elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 2410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'x86_64' 2420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 2430afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'i386' 244f4c35322b200d65f41a8332b4f3503beb497840dmbligh 245f4c35322b200d65f41a8332b4f3503beb497840dmbligh 246548f29af9e38f87c3609838f5aabeac94bb69f13mblighdef get_current_kernel_arch(): 2470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Get the machine architecture, now just a wrap of 'uname -m'.""" 2480afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return os.popen('uname -m').read().rstrip() 249cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 250cdf02a40b97d0570b0e8f8a74a2ede7a868bb1edmbligh 251fdbcaec15092d8b4af80970c495038bdf9b0e63fmblighdef get_file_arch(filename): 2520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # -L means follow symlinks 2532da4d885be334f7db795c3f017daa06ff8ea703dmbligh file_data = utils.system_output('file -L ' + filename) 2540afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if file_data.count('80386'): 2550afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return 'i386' 2560afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 257fdbcaec15092d8b4af80970c495038bdf9b0e63fmbligh 258fdbcaec15092d8b4af80970c495038bdf9b0e63fmbligh 259f4c35322b200d65f41a8332b4f3503beb497840dmblighdef count_cpus(): 2600afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """number of CPUs in the local machine according to /proc/cpuinfo""" 2610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski f = file('/proc/cpuinfo', 'r') 2620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpus = 0 2630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for line in f.readlines(): 2640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if line.startswith('processor'): 2650afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpus += 1 2660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return cpus 267f4c35322b200d65f41a8332b4f3503beb497840dmbligh 268e7a170fb097da48af81ca598e29643f3e7cca584mbligh 269e7a170fb097da48af81ca598e29643f3e7cca584mbligh# Returns total memory in kb 270558885e290e3fa847b6a77b6dd66cac33ec6f65cmblighdef read_from_meminfo(key): 2712da4d885be334f7db795c3f017daa06ff8ea703dmbligh meminfo = utils.system_output('grep %s /proc/meminfo' % key) 2720afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return int(re.search(r'\d+', meminfo).group(0)) 273558885e290e3fa847b6a77b6dd66cac33ec6f65cmbligh 274558885e290e3fa847b6a77b6dd66cac33ec6f65cmbligh 275e7a170fb097da48af81ca598e29643f3e7cca584mblighdef memtotal(): 2760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return read_from_meminfo('MemTotal') 277558885e290e3fa847b6a77b6dd66cac33ec6f65cmbligh 278558885e290e3fa847b6a77b6dd66cac33ec6f65cmbligh 279558885e290e3fa847b6a77b6dd66cac33ec6f65cmblighdef freememtotal(): 2800afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return read_from_meminfo('MemFree') 281558885e290e3fa847b6a77b6dd66cac33ec6f65cmbligh 282558885e290e3fa847b6a77b6dd66cac33ec6f65cmbligh 2838493be44f65668c6ebad5d1bee3b97b76abdebbdmblighdef rounded_memtotal(): 28453da18eddf69243ca175d9a4603cba5b55300726mbligh # Get total of all physical mem, in kbytes 28553da18eddf69243ca175d9a4603cba5b55300726mbligh usable_kbytes = memtotal() 28653da18eddf69243ca175d9a4603cba5b55300726mbligh # usable_kbytes is system's usable DRAM in kbytes, 2878493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # as reported by memtotal() from device /proc/meminfo memtotal 2888493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # after Linux deducts 1.5% to 5.1% for system table overhead 2898493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # Undo the unknown actual deduction by rounding up 2908493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # to next small multiple of a big power-of-two 2918493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # eg 12GB - 5.1% gets rounded back up to 12GB 2928493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh mindeduct = 0.015 # 1.5 percent 2938493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh maxdeduct = 0.055 # 5.5 percent 2948493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # deduction range 1.5% .. 5.5% supports physical mem sizes 2958493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # 6GB .. 12GB in steps of .5GB 2968493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # 12GB .. 24GB in steps of 1 GB 2978493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # 24GB .. 48GB in steps of 2 GB ... 2988493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # Finer granularity in physical mem sizes would require 2998493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # tighter spread between min and max possible deductions 3008493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh 3018493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # increase mem size by at least min deduction, without rounding 30253da18eddf69243ca175d9a4603cba5b55300726mbligh min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 3038493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # increase mem size further by 2**n rounding, by 0..roundKb or more 30453da18eddf69243ca175d9a4603cba5b55300726mbligh round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 3058493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh # find least binary roundup 2**n that covers worst-cast roundKb 30653da18eddf69243ca175d9a4603cba5b55300726mbligh mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 30753da18eddf69243ca175d9a4603cba5b55300726mbligh # have round_kbytes <= mod2n < round_kbytes*2 30853da18eddf69243ca175d9a4603cba5b55300726mbligh # round min_kbytes up to next multiple of mod2n 30953da18eddf69243ca175d9a4603cba5b55300726mbligh phys_kbytes = min_kbytes + mod2n - 1 31053da18eddf69243ca175d9a4603cba5b55300726mbligh phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 31153da18eddf69243ca175d9a4603cba5b55300726mbligh return phys_kbytes 3128493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh 3138493be44f65668c6ebad5d1bee3b97b76abdebbdmbligh 314558885e290e3fa847b6a77b6dd66cac33ec6f65cmblighdef sysctl_kernel(key, value=None): 3150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """(Very) partial implementation of sysctl, for kernel params""" 3160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if value: 3170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # write 3182da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) 3190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 3200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # read 3212da4d885be334f7db795c3f017daa06ff8ea703dmbligh out = utils.read_one_line('/proc/sys/kernel/%s' % key) 3220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return int(re.search(r'\d+', out).group(0)) 323e7a170fb097da48af81ca598e29643f3e7cca584mbligh 324e7a170fb097da48af81ca598e29643f3e7cca584mbligh 3255285a2dc3c5edbc010825e6d64ecdb219b3ad52cmblighdef _convert_exit_status(sts): 3260afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.WIFSIGNALED(sts): 3270afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return -os.WTERMSIG(sts) 3280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elif os.WIFEXITED(sts): 3290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return os.WEXITSTATUS(sts) 3300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 3310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # impossible? 3320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise RuntimeError("Unknown exit status %d!" % sts) 3335285a2dc3c5edbc010825e6d64ecdb219b3ad52cmbligh 3345285a2dc3c5edbc010825e6d64ecdb219b3ad52cmbligh 335f4c35322b200d65f41a8332b4f3503beb497840dmblighdef where_art_thy_filehandles(): 3360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Dump the current list of filehandles""" 3370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 338f4c35322b200d65f41a8332b4f3503beb497840dmbligh 339f4c35322b200d65f41a8332b4f3503beb497840dmbligh 340f4c35322b200d65f41a8332b4f3503beb497840dmblighdef print_to_tty(string): 3410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Output string straight to the tty""" 3420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski open('/dev/tty', 'w').write(string + '\n') 343f4c35322b200d65f41a8332b4f3503beb497840dmbligh 344f4c35322b200d65f41a8332b4f3503beb497840dmbligh 345b8a14e358b611f2d97c06e863be9b53ab2bedeefmblighdef dump_object(object): 3460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Dump an object's attributes and methods 347c86b0b45cd0198d99d271a79f8b6de29feb98cd5mbligh 3480afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski kind of like dir() 3490afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 3500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for item in object.__dict__.iteritems(): 3510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski print item 3520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 3530afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski (key,value) = item 3540afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski dump_object(value) 3550afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except: 3560afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski continue 357b8a14e358b611f2d97c06e863be9b53ab2bedeefmbligh 358b8a14e358b611f2d97c06e863be9b53ab2bedeefmbligh 3594b089663460063b638124ff665f38d2fe7427648mblighdef environ(env_key): 3600afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """return the requested environment variable, or '' if unset""" 3610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if (os.environ.has_key(env_key)): 3620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return os.environ[env_key] 3630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 3640afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return '' 3654b089663460063b638124ff665f38d2fe7427648mbligh 3664b089663460063b638124ff665f38d2fe7427648mbligh 3674b089663460063b638124ff665f38d2fe7427648mblighdef prepend_path(newpath, oldpath): 3680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """prepend newpath to oldpath""" 3690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if (oldpath): 3700afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return newpath + ':' + oldpath 3710afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 3720afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return newpath 3734b089663460063b638124ff665f38d2fe7427648mbligh 3744b089663460063b638124ff665f38d2fe7427648mbligh 3754b089663460063b638124ff665f38d2fe7427648mblighdef append_path(oldpath, newpath): 3760afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """append newpath to oldpath""" 3770afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if (oldpath): 3780afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return oldpath + ':' + newpath 3790afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 3800afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return newpath 3814b089663460063b638124ff665f38d2fe7427648mbligh 3824b089663460063b638124ff665f38d2fe7427648mbligh 3834e75b0d3b020f901456217fb8ff0d7d4391fa869mblighdef avgtime_print(dir): 3840afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ Calculate some benchmarking statistics. 3850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Input is a directory containing a file called 'time'. 3860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski File contains one-per-line results of /usr/bin/time. 3870afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Output is average Elapsed, User, and System time in seconds, 3880afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski and average CPU percentage. 3890afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 3900afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski f = open(dir + "/time") 3910afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski user = system = elapsed = cpu = count = 0 3920afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski r = re.compile('([\d\.]*)user ([\d\.]*)system (\d*):([\d\.]*)elapsed (\d*)%CPU') 3930afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for line in f.readlines(): 3940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski try: 3950afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski s = r.match(line); 3960afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski user += float(s.group(1)) 3970afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski system += float(s.group(2)) 3980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elapsed += (float(s.group(3)) * 60) + float(s.group(4)) 3990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpu += float(s.group(5)) 4000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski count += 1 4010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski except: 4020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise ValueError("badly formatted times") 4030afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 4040afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski f.close() 4050afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 4060afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski (elapsed/count, user/count, system/count, cpu/count) 4074e75b0d3b020f901456217fb8ff0d7d4391fa869mbligh 4084e75b0d3b020f901456217fb8ff0d7d4391fa869mbligh 409f06db0f9b381737b9c85bef1c23ea1a1ca73c559mblighdef running_config(): 4100afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Return path of config file of the currently running kernel 4120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4132da4d885be334f7db795c3f017daa06ff8ea703dmbligh version = utils.system_output('uname -r') 4140afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for config in ('/proc/config.gz', \ 4150afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski '/boot/config-%s' % version, 4160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski '/lib/modules/%s/build/.config' % version): 4170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if os.path.isfile(config): 4180afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return config 4190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return None 4209ec8acc50c0fe06a40353db0df8e135cebb2ec58mbligh 4219ec8acc50c0fe06a40353db0df8e135cebb2ec58mbligh 422a1bef1f2774b539df11c134d9a9a177304c34f3cmblighdef check_for_kernel_feature(feature): 4230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski config = running_config() 424a1bef1f2774b539df11c134d9a9a177304c34f3cmbligh 4250afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if not config: 4260afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise TypeError("Can't find kernel config file") 427a1bef1f2774b539df11c134d9a9a177304c34f3cmbligh 4280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if config.endswith('.gz'): 4290afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski grep = 'zgrep' 4300afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 4310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski grep = 'grep' 4320afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski grep += ' ^CONFIG_%s= %s' % (feature, config) 433a1bef1f2774b539df11c134d9a9a177304c34f3cmbligh 4342da4d885be334f7db795c3f017daa06ff8ea703dmbligh if not utils.system_output(grep, ignore_status=True): 4350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise ValueError("Kernel doesn't have a %s feature" % (feature)) 436a1bef1f2774b539df11c134d9a9a177304c34f3cmbligh 437a1bef1f2774b539df11c134d9a9a177304c34f3cmbligh 4389ec8acc50c0fe06a40353db0df8e135cebb2ec58mblighdef cpu_online_map(): 4390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski Check out the available cpu online map 4410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 4420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpus = [] 4430afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for line in open('/proc/cpuinfo', 'r').readlines(): 4440afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if line.startswith('processor'): 4450afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpus.append(line.split()[2]) # grab cpu number 4460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return cpus 447663e4f664465dbaa9e86c5c20d035ad9bd1e65edmbligh 448663e4f664465dbaa9e86c5c20d035ad9bd1e65edmbligh 449663e4f664465dbaa9e86c5c20d035ad9bd1e65edmblighdef check_glibc_ver(ver): 4500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 4510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 4520afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if glibc_ver.split('.') < ver.split('.'): 453bd8f99817d7b694eb8ee682e0081cf25aa07df0ambligh raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % \ 4540afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski (glibc_ver, ver)) 4550763522bf0dcd6297385a7db965086ed8ba18adambligh 4560763522bf0dcd6297385a7db965086ed8ba18adamblighdef check_kernel_ver(ver): 4572da4d885be334f7db795c3f017daa06ff8ea703dmbligh kernel_ver = utils.system_output('uname -r') 4580afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 4590afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if kv_tmp[0].split('.') < ver.split('.'): 460bd8f99817d7b694eb8ee682e0081cf25aa07df0ambligh raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % \ 4610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski (kernel_ver, ver)) 46260418bb82d1734a7f73a5e4c8870d06799795397mbligh 4639061a273dfd2e522c15f6dd77cfed1e2815e4e9embligh 464264cd8f4889cea73fa5fb7873e3243c1c770a1bcmblighdef human_format(number): 4650afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # Convert number to kilo / mega / giga format. 4660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if number < 1024: 4670afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return "%d" % number 4680afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski kilo = float(number) / 1024.0 4690afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if kilo < 1024: 4700afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return "%.2fk" % kilo 4710afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski meg = kilo / 1024.0 4720afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if meg < 1024: 4730afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return "%.2fM" % meg 4740afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski gig = meg / 1024.0 4750afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return "%.2fG" % gig 476264cd8f4889cea73fa5fb7873e3243c1c770a1bcmbligh 4778eca3a98728afabc7fc761c08dbb8e91260c7de4mbligh 4788eca3a98728afabc7fc761c08dbb8e91260c7de4mblighdef numa_nodes(): 4790afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski node_paths = glob.glob('/sys/devices/system/node/node*') 4800afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 4810afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return (sorted(nodes)) 4828eca3a98728afabc7fc761c08dbb8e91260c7de4mbligh 4838eca3a98728afabc7fc761c08dbb8e91260c7de4mbligh 4848eca3a98728afabc7fc761c08dbb8e91260c7de4mblighdef node_size(): 4850afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski nodes = max(len(numa_nodes()), 1) 4860afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return ((memtotal() * 1024) / nodes) 4878eca3a98728afabc7fc761c08dbb8e91260c7de4mbligh 48832bcff3382844181eaf3d74e0fb252d88edfc8fbmbligh 48932bcff3382844181eaf3d74e0fb252d88edfc8fbmblighdef to_seconds(time_string): 4900afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Converts a string in M+:SS.SS format to S+.SS""" 4910afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski elts = time_string.split(':') 4920afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if len(elts) == 1: 4930afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return time_string 4940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return str(int(elts[0]) * 60 + float(elts[1])) 49532bcff3382844181eaf3d74e0fb252d88edfc8fbmbligh 49632bcff3382844181eaf3d74e0fb252d88edfc8fbmbligh 49732bcff3382844181eaf3d74e0fb252d88edfc8fbmblighdef extract_all_time_results(results_string): 4980afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Extract user, system, and elapsed times into a list of tuples""" 4990afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski pattern = re.compile(r"(.*?)user (.*?)system (.*?)elapsed") 5000afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski results = [] 5010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for result in pattern.findall(results_string): 5020afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski results.append(tuple([to_seconds(elt) for elt in result])) 5030afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return results 504c421164e16db7233ef8fcc4ecfd83f2979aec16ambligh 505c421164e16db7233ef8fcc4ecfd83f2979aec16ambligh 506c421164e16db7233ef8fcc4ecfd83f2979aec16amblighdef pickle_load(filename): 5070afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return pickle.load(open(filename, 'r')) 508c421164e16db7233ef8fcc4ecfd83f2979aec16ambligh 509237bed32e0110ccd0db10823df742534dd7dc50dmbligh 510237bed32e0110ccd0db10823df742534dd7dc50dmbligh# Return the kernel version and build timestamp. 511237bed32e0110ccd0db10823df742534dd7dc50dmblighdef running_os_release(): 5120afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return os.uname()[2:4] 513237bed32e0110ccd0db10823df742534dd7dc50dmbligh 514237bed32e0110ccd0db10823df742534dd7dc50dmbligh 515237bed32e0110ccd0db10823df742534dd7dc50dmblighdef running_os_ident(): 5160afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski (version, timestamp) = running_os_release() 5170afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return version + '::' + timestamp 518b830e28297dca51aed2c4b57469a3e04845e54e7mbligh 519b830e28297dca51aed2c4b57469a3e04845e54e7mbligh 5203bf79ca4d5490f9af09e1ddc39b8df5343b34d06mblighdef running_os_full_version(): 5213bf79ca4d5490f9af09e1ddc39b8df5343b34d06mbligh (version, timestamp) = running_os_release() 5223bf79ca4d5490f9af09e1ddc39b8df5343b34d06mbligh return version 5233bf79ca4d5490f9af09e1ddc39b8df5343b34d06mbligh 5243bf79ca4d5490f9af09e1ddc39b8df5343b34d06mbligh 525523a19b532a778dd9e752e507ac75d3ffc5c701ambligh# much like find . -name 'pattern' 526523a19b532a778dd9e752e507ac75d3ffc5c701amblighdef locate(pattern, root=os.getcwd()): 5270afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for path, dirs, files in os.walk(root): 528987071e1fba830910658ce987bb803d142480c03mbligh for f in files: 529987071e1fba830910658ce987bb803d142480c03mbligh if fnmatch.fnmatch(f, pattern): 5307076b192936caad193898cff194e82f83dd13d63mbligh yield os.path.abspath(os.path.join(path, f)) 531523a19b532a778dd9e752e507ac75d3ffc5c701ambligh 532523a19b532a778dd9e752e507ac75d3ffc5c701ambligh 53325bb1e1fbb67950ac480f6500138c19647472889mblighdef freespace(path): 5340afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Return the disk free space, in bytes""" 5350afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski s = os.statvfs(path) 5360afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return s.f_bavail * s.f_bsize 5378415f966acaaa3aa3ca6a651d9a190081edbb0eejadmanski 5388415f966acaaa3aa3ca6a651d9a190081edbb0eejadmanski 5398415f966acaaa3aa3ca6a651d9a190081edbb0eejadmanskidef disk_block_size(path): 5400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Return the disk block size, in bytes""" 5410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return os.statvfs(path).f_bsize 5426de9cdfe8ce95c1d87b01aa768526a64755090cambligh 5436de9cdfe8ce95c1d87b01aa768526a64755090cambligh 5446de9cdfe8ce95c1d87b01aa768526a64755090camblighdef get_cpu_family(): 5452da4d885be334f7db795c3f017daa06ff8ea703dmbligh procinfo = utils.system_output('cat /proc/cpuinfo') 5460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski CPU_FAMILY_RE = re.compile(r'^cpu family\s+:\s+(\S+)', re.M) 5470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski matches = CPU_FAMILY_RE.findall(procinfo) 5480afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if matches: 5490afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return int(matches[0]) 5500afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski else: 5510afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise error.TestError('Could not get valid cpu family data') 5526de9cdfe8ce95c1d87b01aa768526a64755090cambligh 5533e9062e854a920eead8e250c4e79838c30a0657cmbligh 5542316e52c84ac8799330c52359bc4dd5e7f7324d5mblighdef get_disks(): 5552da4d885be334f7db795c3f017daa06ff8ea703dmbligh df_output = utils.system_output('df') 5560afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski disk_re = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 5570afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return disk_re.findall(df_output) 5586de9cdfe8ce95c1d87b01aa768526a64755090cambligh 5593e9062e854a920eead8e250c4e79838c30a0657cmbligh 5603e9062e854a920eead8e250c4e79838c30a0657cmblighdef load_module(module_name): 5610afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski # Checks if a module has already been loaded 5620afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if module_is_loaded(module_name): 5630afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return False 564af4efc23471265a3bfa7fef0146fbc0cbb54515bmbligh 5652da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.system('/sbin/modprobe ' + module_name) 5660afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return True 5673e9062e854a920eead8e250c4e79838c30a0657cmbligh 5683e9062e854a920eead8e250c4e79838c30a0657cmbligh 5693e9062e854a920eead8e250c4e79838c30a0657cmblighdef unload_module(module_name): 570b923fc2a01db6e4548ffd541d86f9be49bc5805almr """ 571b923fc2a01db6e4548ffd541d86f9be49bc5805almr Removes a module. Handles dependencies. If even then it's not possible 572b923fc2a01db6e4548ffd541d86f9be49bc5805almr to remove one of the modules, it will trhow an error.CmdError exception. 573b923fc2a01db6e4548ffd541d86f9be49bc5805almr 574b923fc2a01db6e4548ffd541d86f9be49bc5805almr @param module_name: Name of the module we want to remove. 575b923fc2a01db6e4548ffd541d86f9be49bc5805almr """ 576b923fc2a01db6e4548ffd541d86f9be49bc5805almr l_raw = utils.system_output("/sbin/lsmod").splitlines() 577b923fc2a01db6e4548ffd541d86f9be49bc5805almr lsmod = [x for x in l_raw if x.split()[0] == module_name] 578b923fc2a01db6e4548ffd541d86f9be49bc5805almr if len(lsmod) > 0: 579b923fc2a01db6e4548ffd541d86f9be49bc5805almr line_parts = lsmod[0].split() 580b923fc2a01db6e4548ffd541d86f9be49bc5805almr if len(line_parts) == 4: 581b923fc2a01db6e4548ffd541d86f9be49bc5805almr submodules = line_parts[3].split(",") 582b923fc2a01db6e4548ffd541d86f9be49bc5805almr for submodule in submodules: 583b923fc2a01db6e4548ffd541d86f9be49bc5805almr unload_module(submodule) 584b923fc2a01db6e4548ffd541d86f9be49bc5805almr utils.system("/sbin/modprobe -r %s" % module_name) 585b923fc2a01db6e4548ffd541d86f9be49bc5805almr logging.info("Module %s unloaded" % module_name) 586b923fc2a01db6e4548ffd541d86f9be49bc5805almr else: 587b923fc2a01db6e4548ffd541d86f9be49bc5805almr logging.info("Module %s is already unloaded" % module_name) 5883e9062e854a920eead8e250c4e79838c30a0657cmbligh 5893e9062e854a920eead8e250c4e79838c30a0657cmbligh 5903e9062e854a920eead8e250c4e79838c30a0657cmblighdef module_is_loaded(module_name): 5910afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski module_name = module_name.replace('-', '_') 5922da4d885be334f7db795c3f017daa06ff8ea703dmbligh modules = utils.system_output('/sbin/lsmod').splitlines() 5930afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for module in modules: 5940afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if module.startswith(module_name) and module[len(module_name)] == ' ': 5950afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return True 5960afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return False 5973e9062e854a920eead8e250c4e79838c30a0657cmbligh 5983e9062e854a920eead8e250c4e79838c30a0657cmbligh 5996b34c4cc34bebd20def94c0191ef1e46dd6a1cdemblighdef get_loaded_modules(): 6002da4d885be334f7db795c3f017daa06ff8ea703dmbligh lsmod_output = utils.system_output('/sbin/lsmod').splitlines()[1:] 6010afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return [line.split(None, 1)[0] for line in lsmod_output] 6026b34c4cc34bebd20def94c0191ef1e46dd6a1cdembligh 6036b34c4cc34bebd20def94c0191ef1e46dd6a1cdembligh 6043e9062e854a920eead8e250c4e79838c30a0657cmblighdef get_huge_page_size(): 6052da4d885be334f7db795c3f017daa06ff8ea703dmbligh output = utils.system_output('grep Hugepagesize /proc/meminfo') 6060afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return int(output.split()[1]) # Assumes units always in kB. :( 6073e9062e854a920eead8e250c4e79838c30a0657cmbligh 6083e9062e854a920eead8e250c4e79838c30a0657cmbligh 6093e9062e854a920eead8e250c4e79838c30a0657cmblighdef get_num_huge_pages(): 6102da4d885be334f7db795c3f017daa06ff8ea703dmbligh raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') 6110afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return int(raw_hugepages.split()[2]) 6123e9062e854a920eead8e250c4e79838c30a0657cmbligh 6133e9062e854a920eead8e250c4e79838c30a0657cmbligh 6143e9062e854a920eead8e250c4e79838c30a0657cmblighdef set_num_huge_pages(num): 6152da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) 6163e9062e854a920eead8e250c4e79838c30a0657cmbligh 6173e9062e854a920eead8e250c4e79838c30a0657cmbligh 6186b34c4cc34bebd20def94c0191ef1e46dd6a1cdemblighdef get_cpu_vendor(): 6190afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cpuinfo = open('/proc/cpuinfo').read() 6200afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski vendors = re.findall(r'(?m)^vendor_id\s*:\s*(\S+)\s*$', cpuinfo) 6210afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski for i in xrange(1, len(vendors)): 6220afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if vendors[i] != vendors[0]: 6230afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 6240afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski return vendors[0] 6256b34c4cc34bebd20def94c0191ef1e46dd6a1cdembligh 6266b34c4cc34bebd20def94c0191ef1e46dd6a1cdembligh 6276b34c4cc34bebd20def94c0191ef1e46dd6a1cdemblighdef probe_cpus(): 6280afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 62953da18eddf69243ca175d9a4603cba5b55300726mbligh This routine returns a list of cpu devices found under 63053da18eddf69243ca175d9a4603cba5b55300726mbligh /sys/devices/system/cpu. 6310afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """ 63253da18eddf69243ca175d9a4603cba5b55300726mbligh cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 6332da4d885be334f7db795c3f017daa06ff8ea703dmbligh return utils.system_output(cmd).splitlines() 6346b34c4cc34bebd20def94c0191ef1e46dd6a1cdembligh 6356b34c4cc34bebd20def94c0191ef1e46dd6a1cdembligh 63670c50ad0eec36f4678374040031da84150a75997mblighdef ping_default_gateway(): 6370afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski """Ping the default gateway.""" 63870c50ad0eec36f4678374040031da84150a75997mbligh 6390afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski network = open('/etc/sysconfig/network') 6400afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski m = re.search('GATEWAY=(\S+)', network.read()) 6410afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 6420afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski if m: 6430afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski gw = m.group(1) 6440afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski cmd = 'ping %s -c 5 > /dev/null' % gw 6452da4d885be334f7db795c3f017daa06ff8ea703dmbligh return utils.system(cmd, ignore_status=True) 6460afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski 6470afbb6369aa5aa9a75ea67dd9e95ec4b21c0c181jadmanski raise error.TestError('Unable to find default gateway') 64870c50ad0eec36f4678374040031da84150a75997mbligh 64970c50ad0eec36f4678374040031da84150a75997mbligh 650115feb2fee75e8cb75873905f07763fbfafc8f6ejadmanskidef drop_caches(): 651115feb2fee75e8cb75873905f07763fbfafc8f6ejadmanski """Writes back all dirty pages to disk and clears all the caches.""" 6522da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.system("sync") 6532da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.system("sync") 654115feb2fee75e8cb75873905f07763fbfafc8f6ejadmanski # We ignore failures here as this will fail on 2.6.11 kernels. 6552da4d885be334f7db795c3f017daa06ff8ea703dmbligh utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 656