xref: /external/autotest/client/bin/base_utils.py

"""
DO NOT import this file directly - import client/bin/utils.py,
which will mix this in

Convenience functions for use by tests or whomever.

Note that this file is mixed in by utils.py - note very carefully the
precedence order defined there
"""

import commands
import fnmatch
import glob
import logging
import math
import multiprocessing
import os
import pickle
import re
import shutil

from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import magic
from autotest_lib.client.common_lib import utils


def grep(pattern, file):
    """
    This is mainly to fix the return code inversion from grep
    Also handles compressed files.

    returns 1 if the pattern is present in the file, 0 if not.
    """
    command = 'grep "%s" > /dev/null' % pattern
    ret = cat_file_to_cmd(file, command, ignore_status=True)
    return not ret


def difflist(list1, list2):
    """returns items in list2 that are not in list1"""
    diff = [];
    for x in list2:
        if x not in list1:
            diff.append(x)
    return diff


def cat_file_to_cmd(file, command, ignore_status=0, return_output=False):
    """
    equivalent to 'cat file | command' but knows to use
    zcat or bzcat if appropriate
    """
    if not os.path.isfile(file):
        raise NameError('invalid file %s to cat to command %s'
                % (file, command))

    if return_output:
        run_cmd = utils.system_output
    else:
        run_cmd = utils.system

    if magic.guess_type(file) == 'application/x-bzip2':
        cat = 'bzcat'
    elif magic.guess_type(file) == 'application/x-gzip':
        cat = 'zcat'
    else:
        cat = 'cat'
    return run_cmd('%s %s | %s' % (cat, file, command),
                                                    ignore_status=ignore_status)


def extract_tarball_to_dir(tarball, dir):
    """
    Extract a tarball to a specified directory name instead of whatever
    the top level of a tarball is - useful for versioned directory names, etc
    """
    if os.path.exists(dir):
        if os.path.isdir(dir):
            shutil.rmtree(dir)
        else:
            os.remove(dir)
    pwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.abspath(dir)))
    newdir = extract_tarball(tarball)
    os.rename(newdir, dir)
    os.chdir(pwd)


def extract_tarball(tarball):
    """Returns the directory extracted by the tarball."""
    extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null',
                                    return_output=True).splitlines()

    dir = None

    for line in extracted:
        if line.startswith('./'):
            line = line[2:]
        if not line or line == '.':
            continue
        topdir = line.split('/')[0]
        if os.path.isdir(topdir):
            if dir:
                assert(dir == topdir)
            else:
                dir = topdir
    if dir:
        return dir
    else:
        raise NameError('extracting tarball produced no dir')


def unmap_url_cache(cachedir, url, expected_hash, method="md5"):
    """
    Downloads a file from a URL to a cache directory. If the file is already
    at the expected position and has the expected hash, let's not download it
    again.

    @param cachedir: Directory that might hold a copy of the file we want to
            download.
    @param url: URL for the file we want to download.
    @param expected_hash: Hash string that we expect the file downloaded to
            have.
    @param method: Method used to calculate the hash string (md5, sha1).
    """
    # Let's convert cachedir to a canonical path, if it's not already
    cachedir = os.path.realpath(cachedir)
    if not os.path.isdir(cachedir):
        try:
            os.makedirs(cachedir)
        except:
            raise ValueError('Could not create cache directory %s' % cachedir)
    file_from_url = os.path.basename(url)
    file_local_path = os.path.join(cachedir, file_from_url)

    file_hash = None
    failure_counter = 0
    while not file_hash == expected_hash:
        if os.path.isfile(file_local_path):
            file_hash = hash_file(file_local_path, method)
            if file_hash == expected_hash:
                # File is already at the expected position and ready to go
                src = file_from_url
            else:
                # Let's download the package again, it's corrupted...
                logging.error("Seems that file %s is corrupted, trying to "
                              "download it again", file_from_url)
                src = url
                failure_counter += 1
        else:
            # File is not there, let's download it
            src = url
        if failure_counter > 1:
            raise EnvironmentError("Consistently failed to download the "
                                   "package %s. Aborting further download "
                                   "attempts. This might mean either the "
                                   "network connection has problems or the "
                                   "expected hash string that was determined "
                                   "for this file is wrong", file_from_url)
        file_path = utils.unmap_url(cachedir, src, cachedir)

    return file_path


def force_copy(src, dest):
    """Replace dest with a new copy of src, even if it exists"""
    if os.path.isfile(dest):
        os.remove(dest)
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(src))
    shutil.copyfile(src, dest)
    return dest


def force_link(src, dest):
    """Link src to dest, overwriting it if it exists"""
    return utils.system("ln -sf %s %s" % (src, dest))


def file_contains_pattern(file, pattern):
    """Return true if file contains the specified egrep pattern"""
    if not os.path.isfile(file):
        raise NameError('file %s does not exist' % file)
    return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True)


def list_grep(list, pattern):
    """True if any item in list matches the specified pattern."""
    compiled = re.compile(pattern)
    for line in list:
        match = compiled.search(line)
        if (match):
            return 1
    return 0


def get_os_vendor():
    """Try to guess what's the os vendor
    """
    if os.path.isfile('/etc/SuSE-release'):
        return 'SUSE'

    issue = '/etc/issue'

    if not os.path.isfile(issue):
        return 'Unknown'

    if file_contains_pattern(issue, 'Red Hat'):
        return 'Red Hat'
    elif file_contains_pattern(issue, 'Fedora'):
        return 'Fedora Core'
    elif file_contains_pattern(issue, 'SUSE'):
        return 'SUSE'
    elif file_contains_pattern(issue, 'Ubuntu'):
        return 'Ubuntu'
    elif file_contains_pattern(issue, 'Debian'):
        return 'Debian'
    else:
        return 'Unknown'


def get_cc():
    try:
        return os.environ['CC']
    except KeyError:
        return 'gcc'


def get_vmlinux():
    """Return the full path to vmlinux

    Ahem. This is crap. Pray harder. Bad Martin.
    """
    vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r')
    if os.path.isfile(vmlinux):
        return vmlinux
    vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r')
    if os.path.isfile(vmlinux):
        return vmlinux
    return None


def get_systemmap():
    """Return the full path to System.map

    Ahem. This is crap. Pray harder. Bad Martin.
    """
    map = '/boot/System.map-%s' % utils.system_output('uname -r')
    if os.path.isfile(map):
        return map
    map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r')
    if os.path.isfile(map):
        return map
    return None


def get_modules_dir():
    """Return the modules dir for the running kernel version"""
    kernel_version = utils.system_output('uname -r')
    return '/lib/modules/%s/kernel' % kernel_version


_CPUINFO_RE = re.compile(r'^(?P<key>[^\t]*)\t*: ?(?P<value>.*)$')


def get_cpuinfo():
    """Read /proc/cpuinfo and convert to a list of dicts."""
    cpuinfo = []
    with open('/proc/cpuinfo', 'r') as f:
        cpu = {}
        for line in f:
            line = line.strip()
            if not line:
                cpuinfo.append(cpu)
                cpu = {}
                continue
            match = _CPUINFO_RE.match(line)
            cpu[match.group('key')] = match.group('value')
        if cpu:
            # cpuinfo usually ends in a blank line, so this shouldn't happen.
            cpuinfo.append(cpu)
    return cpuinfo


def get_cpu_arch():
    """Work out which CPU architecture we're running on"""
    f = open('/proc/cpuinfo', 'r')
    cpuinfo = f.readlines()
    f.close()
    if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'):
        return 'power'
    elif list_grep(cpuinfo, '^cpu.*POWER4'):
        return 'power4'
    elif list_grep(cpuinfo, '^cpu.*POWER5'):
        return 'power5'
    elif list_grep(cpuinfo, '^cpu.*POWER6'):
        return 'power6'
    elif list_grep(cpuinfo, '^cpu.*POWER7'):
        return 'power7'
    elif list_grep(cpuinfo, '^cpu.*PPC970'):
        return 'power970'
    elif list_grep(cpuinfo, 'ARM'):
        return 'arm'
    elif list_grep(cpuinfo, '^flags.*:.* lm .*'):
        return 'x86_64'
    elif list_grep(cpuinfo, 'CPU.*implementer.*0x41'):
        return 'arm'
    else:
        return 'i386'


def get_arm_soc_family_from_devicetree():
    """
    Work out which ARM SoC we're running on based on the 'compatible' property
    of the base node of devicetree, if it exists.
    """
    devicetree_compatible = '/sys/firmware/devicetree/base/compatible'
    if not os.path.isfile(devicetree_compatible):
        return None
    f = open(devicetree_compatible, 'r')
    compatible = f.readlines()
    f.close()
    if list_grep(compatible, 'rk3399'):
        return 'rockchip'
    elif list_grep(compatible, 'mt8173'):
        return 'mediatek'
    return None


def get_arm_soc_family():
    """Work out which ARM SoC we're running on"""
    family = get_arm_soc_family_from_devicetree()
    if family is not None:
        return family

    f = open('/proc/cpuinfo', 'r')
    cpuinfo = f.readlines()
    f.close()
    if list_grep(cpuinfo, 'EXYNOS5'):
        return 'exynos5'
    elif list_grep(cpuinfo, 'Tegra'):
        return 'tegra'
    elif list_grep(cpuinfo, 'Rockchip'):
        return 'rockchip'
    return 'arm'


def get_cpu_soc_family():
    """Like get_cpu_arch, but for ARM, returns the SoC family name"""
    f = open('/proc/cpuinfo', 'r')
    cpuinfo = f.readlines()
    f.close()
    family = get_cpu_arch()
    if family == 'arm':
        family = get_arm_soc_family()
    if list_grep(cpuinfo, '^vendor_id.*:.*AMD'):
        family = 'amd'
    return family


INTEL_UARCH_TABLE = {
    '06_1C': 'Atom',
    '06_26': 'Atom',
    '06_36': 'Atom',
    '06_4C': 'Braswell',
    '06_3D': 'Broadwell',
    '06_0D': 'Dothan',
    '06_3A': 'IvyBridge',
    '06_3E': 'IvyBridge',
    '06_3C': 'Haswell',
    '06_3F': 'Haswell',
    '06_45': 'Haswell',
    '06_46': 'Haswell',
    '06_0F': 'Merom',
    '06_16': 'Merom',
    '06_17': 'Nehalem',
    '06_1A': 'Nehalem',
    '06_1D': 'Nehalem',
    '06_1E': 'Nehalem',
    '06_1F': 'Nehalem',
    '06_2E': 'Nehalem',
    '06_2A': 'SandyBridge',
    '06_2D': 'SandyBridge',
    '06_4E': 'Skylake',
    '0F_03': 'Prescott',
    '0F_04': 'Prescott',
    '0F_06': 'Presler',
    '06_25': 'Westmere',
    '06_2C': 'Westmere',
    '06_2F': 'Westmere',
}


def get_intel_cpu_uarch(numeric=False):
    """Return the Intel microarchitecture we're running on, or None.

    Returns None if this is not an Intel CPU. Returns the family and model as
    underscore-separated hex (per Intel manual convention) if the uarch is not
    known, or if numeric is True.
    """
    if not get_current_kernel_arch().startswith('x86'):
        return None
    cpuinfo = get_cpuinfo()[0]
    if cpuinfo['vendor_id'] != 'GenuineIntel':
        return None
    family_model = '%02X_%02X' % (int(cpuinfo['cpu family']),
                                  int(cpuinfo['model']))
    if numeric:
        return family_model
    return INTEL_UARCH_TABLE.get(family_model, family_model)


def get_current_kernel_arch():
    """Get the machine architecture, now just a wrap of 'uname -m'."""
    return os.popen('uname -m').read().rstrip()


def get_file_arch(filename):
    # -L means follow symlinks
    file_data = utils.system_output('file -L ' + filename)
    if file_data.count('80386'):
        return 'i386'
    return None


def count_cpus():
    """number of CPUs in the local machine according to /proc/cpuinfo"""
    try:
       return multiprocessing.cpu_count()
    except Exception as e:
       logging.exception('can not get cpu count from'
                        ' multiprocessing.cpu_count()')
    cpuinfo = get_cpuinfo()
    # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582.
    return len(cpuinfo) or 1


def cpu_online_map():
    """
    Check out the available cpu online map
    """
    cpuinfo = get_cpuinfo()
    cpus = []
    for cpu in cpuinfo:
        cpus.append(cpu['processor'])  # grab cpu number
    return cpus


def get_cpu_family():
    cpuinfo = get_cpuinfo()[0]
    return int(cpuinfo['cpu_family'])


def get_cpu_vendor():
    cpuinfo = get_cpuinfo()
    vendors = [cpu['vendor_id'] for cpu in cpuinfo]
    for v in vendors[1:]:
        if v != vendors[0]:
            raise error.TestError('multiple cpu vendors found: ' + str(vendors))
    return vendors[0]


def probe_cpus():
    """
    This routine returns a list of cpu devices found under
    /sys/devices/system/cpu.
    """
    cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*'
    return utils.system_output(cmd).splitlines()


# Returns total memory in kb
def read_from_meminfo(key):
    meminfo = utils.system_output('grep %s /proc/meminfo' % key)
    return int(re.search(r'\d+', meminfo).group(0))


def memtotal():
    return read_from_meminfo('MemTotal')


def freememtotal():
    return read_from_meminfo('MemFree')

def usable_memtotal():
    # Reserved 5% for OS use
    return int(read_from_meminfo('MemFree') * 0.95)


def rounded_memtotal():
    # Get total of all physical mem, in kbytes
    usable_kbytes = memtotal()
    # usable_kbytes is system's usable DRAM in kbytes,
    #   as reported by memtotal() from device /proc/meminfo memtotal
    #   after Linux deducts 1.5% to 5.1% for system table overhead
    # Undo the unknown actual deduction by rounding up
    #   to next small multiple of a big power-of-two
    #   eg  12GB - 5.1% gets rounded back up to 12GB
    mindeduct = 0.015  # 1.5 percent
    maxdeduct = 0.055  # 5.5 percent
    # deduction range 1.5% .. 5.5% supports physical mem sizes
    #    6GB .. 12GB in steps of .5GB
    #   12GB .. 24GB in steps of 1 GB
    #   24GB .. 48GB in steps of 2 GB ...
    # Finer granularity in physical mem sizes would require
    #   tighter spread between min and max possible deductions

    # increase mem size by at least min deduction, without rounding
    min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
    # increase mem size further by 2**n rounding, by 0..roundKb or more
    round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
    # find least binary roundup 2**n that covers worst-cast roundKb
    mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
    # have round_kbytes <= mod2n < round_kbytes*2
    # round min_kbytes up to next multiple of mod2n
    phys_kbytes = min_kbytes + mod2n - 1
    phys_kbytes = phys_kbytes - (phys_kbytes % mod2n)  # clear low bits
    return phys_kbytes


def sysctl(key, value=None):
    """Generic implementation of sysctl, to read and write.

    @param key: A location under /proc/sys
    @param value: If not None, a value to write into the sysctl.

    @return The single-line sysctl value as a string.
    """
    path = '/proc/sys/%s' % key
    if value is not None:
        utils.write_one_line(path, str(value))
    return utils.read_one_line(path)


def sysctl_kernel(key, value=None):
    """(Very) partial implementation of sysctl, for kernel params"""
    if value is not None:
        # write
        utils.write_one_line('/proc/sys/kernel/%s' % key, str(value))
    else:
        # read
        out = utils.read_one_line('/proc/sys/kernel/%s' % key)
        return int(re.search(r'\d+', out).group(0))


def _convert_exit_status(sts):
    if os.WIFSIGNALED(sts):
        return -os.WTERMSIG(sts)
    elif os.WIFEXITED(sts):
        return os.WEXITSTATUS(sts)
    else:
        # impossible?
        raise RuntimeError("Unknown exit status %d!" % sts)


def where_art_thy_filehandles():
    """Dump the current list of filehandles"""
    os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())


def print_to_tty(string):
    """Output string straight to the tty"""
    open('/dev/tty', 'w').write(string + '\n')


def dump_object(object):
    """Dump an object's attributes and methods

    kind of like dir()
    """
    for item in object.__dict__.iteritems():
        print item
        try:
            (key, value) = item
            dump_object(value)
        except:
            continue


def environ(env_key):
    """return the requested environment variable, or '' if unset"""
    if (os.environ.has_key(env_key)):
        return os.environ[env_key]
    else:
        return ''


def prepend_path(newpath, oldpath):
    """prepend newpath to oldpath"""
    if (oldpath):
        return newpath + ':' + oldpath
    else:
        return newpath


def append_path(oldpath, newpath):
    """append newpath to oldpath"""
    if (oldpath):
        return oldpath + ':' + newpath
    else:
        return newpath


_TIME_OUTPUT_RE = re.compile(
        r'([\d\.]*)user ([\d\.]*)system '
        r'(\d*):([\d\.]*)elapsed (\d*)%CPU')


def avgtime_print(dir):
    """ Calculate some benchmarking statistics.
        Input is a directory containing a file called 'time'.
        File contains one-per-line results of /usr/bin/time.
        Output is average Elapsed, User, and System time in seconds,
          and average CPU percentage.
    """
    user = system = elapsed = cpu = count = 0
    with open(dir + "/time") as f:
        for line in f:
            try:
                m = _TIME_OUTPUT_RE.match(line);
                user += float(m.group(1))
                system += float(m.group(2))
                elapsed += (float(m.group(3)) * 60) + float(m.group(4))
                cpu += float(m.group(5))
                count += 1
            except:
                raise ValueError("badly formatted times")

    return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \
          (elapsed / count, user / count, system / count, cpu / count)


def to_seconds(time_string):
    """Converts a string in M+:SS.SS format to S+.SS"""
    elts = time_string.split(':')
    if len(elts) == 1:
        return time_string
    return str(int(elts[0]) * 60 + float(elts[1]))


_TIME_OUTPUT_RE_2 = re.compile(r'(.*?)user (.*?)system (.*?)elapsed')


def extract_all_time_results(results_string):
    """Extract user, system, and elapsed times into a list of tuples"""
    results = []
    for result in _TIME_OUTPUT_RE_2.findall(results_string):
        results.append(tuple([to_seconds(elt) for elt in result]))
    return results


def running_config():
    """
    Return path of config file of the currently running kernel
    """
    version = utils.system_output('uname -r')
    for config in ('/proc/config.gz', \
                   '/boot/config-%s' % version,
                   '/lib/modules/%s/build/.config' % version):
        if os.path.isfile(config):
            return config
    return None


def check_for_kernel_feature(feature):
    config = running_config()

    if not config:
        raise TypeError("Can't find kernel config file")

    if magic.guess_type(config) == 'application/x-gzip':
        grep = 'zgrep'
    else:
        grep = 'grep'
    grep += ' ^CONFIG_%s= %s' % (feature, config)

    if not utils.system_output(grep, ignore_status=True):
        raise ValueError("Kernel doesn't have a %s feature" % (feature))


def check_glibc_ver(ver):
    glibc_ver = commands.getoutput('ldd --version').splitlines()[0]
    glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group()
    if utils.compare_versions(glibc_ver, ver) == -1:
        raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." %
                              (glibc_ver, ver))

def check_kernel_ver(ver):
    kernel_ver = utils.system_output('uname -r')
    kv_tmp = re.split(r'[-]', kernel_ver)[0:3]
    # In compare_versions, if v1 < v2, return value == -1
    if utils.compare_versions(kv_tmp[0], ver) == -1:
        raise error.TestError("Kernel too old (%s). Kernel > %s is needed." %
                              (kernel_ver, ver))


def human_format(number):
    # Convert number to kilo / mega / giga format.
    if number < 1024:
        return "%d" % number
    kilo = float(number) / 1024.0
    if kilo < 1024:
        return "%.2fk" % kilo
    meg = kilo / 1024.0
    if meg < 1024:
        return "%.2fM" % meg
    gig = meg / 1024.0
    return "%.2fG" % gig


def numa_nodes():
    node_paths = glob.glob('/sys/devices/system/node/node*')
    nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths]
    return (sorted(nodes))


def node_size():
    nodes = max(len(numa_nodes()), 1)
    return ((memtotal() * 1024) / nodes)


def pickle_load(filename):
    return pickle.load(open(filename, 'r'))


# Return the kernel version and build timestamp.
def running_os_release():
    return os.uname()[2:4]


def running_os_ident():
    (version, timestamp) = running_os_release()
    return version + '::' + timestamp


def running_os_full_version():
    (version, timestamp) = running_os_release()
    return version


# much like find . -name 'pattern'
def locate(pattern, root=os.getcwd()):
    for path, dirs, files in os.walk(root):
        for f in files:
            if fnmatch.fnmatch(f, pattern):
                yield os.path.abspath(os.path.join(path, f))


def freespace(path):
    """Return the disk free space, in bytes"""
    s = os.statvfs(path)
    return s.f_bavail * s.f_bsize


def disk_block_size(path):
    """Return the disk block size, in bytes"""
    return os.statvfs(path).f_bsize


_DISK_PARTITION_3_RE = re.compile(r'^(/dev/hd[a-z]+)3', re.M)

def get_disks():
    df_output = utils.system_output('df')
    return _DISK_PARTITION_3_RE.findall(df_output)


def get_disk_size(disk_name):
    """
    Return size of disk in byte. Return 0 in Error Case

    @param disk_name: disk name to find size
    """
    device = os.path.basename(disk_name)
    for line in file('/proc/partitions'):
        try:
            _, _, blocks, name = re.split(r' +', line.strip())
        except ValueError:
            continue
        if name == device:
            return 1024 * int(blocks)
    return 0


def get_disk_size_gb(disk_name):
    """
    Return size of disk in GB (10^9). Return 0 in Error Case

    @param disk_name: disk name to find size
    """
    return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5)


def get_disk_model(disk_name):
    """
    Return model name for internal storage device

    @param disk_name: disk name to find model
    """
    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
    cmd2 = 'grep -E "ID_(NAME|MODEL)="'
    cmd3 = 'cut -f 2 -d"="'
    cmd = ' | '.join([cmd1, cmd2, cmd3])
    return utils.system_output(cmd)


_DISK_DEV_RE = re.compile(r'/dev/sd[a-z]|/dev/mmcblk[0-9]*|/dev/nvme[0-9]*')


def get_disk_from_filename(filename):
    """
    Return the disk device the filename is on.
    If the file is on tmpfs or other special file systems,
    return None.

    @param filename: name of file, full path.
    """

    if not os.path.exists(filename):
        raise error.TestError('file %s missing' % filename)

    if filename[0] != '/':
        raise error.TestError('This code works only with full path')

    m = _DISK_DEV_RE.match(filename)
    while not m:
        if filename[0] != '/':
            return None
        if filename == '/dev/root':
            cmd = 'rootdev -d -s'
        elif filename.startswith('/dev/mapper'):
            cmd = 'dmsetup table "%s"' % os.path.basename(filename)
            dmsetup_output = utils.system_output(cmd).split(' ')
            if dmsetup_output[2] == 'verity':
                maj_min = dmsetup_output[4]
            elif dmsetup_output[2] == 'crypt':
                maj_min = dmsetup_output[6]
            cmd = 'realpath "/dev/block/%s"' % maj_min
        elif filename.startswith('/dev/loop'):
            cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename
        else:
            cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename
        filename = utils.system_output(cmd)
        m = _DISK_DEV_RE.match(filename)
    return m.group(0)


def get_disk_firmware_version(disk_name):
    """
    Return firmware version for internal storage device. (empty string for eMMC)

    @param disk_name: disk name to find model
    """
    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
    cmd2 = 'grep -E "ID_REVISION="'
    cmd3 = 'cut -f 2 -d"="'
    cmd = ' | '.join([cmd1, cmd2, cmd3])
    return utils.system_output(cmd)


def is_disk_scsi(disk_name):
    """
    Return true if disk is a scsi device, return false otherwise

    @param disk_name: disk name check
    """
    return re.match('/dev/sd[a-z]+', disk_name)


def is_disk_harddisk(disk_name):
    """
    Return true if disk is a harddisk, return false otherwise

    @param disk_name: disk name check
    """
    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
    cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="'
    cmd3 = 'cut -f 2 -d"="'
    cmd = ' | '.join([cmd1, cmd2, cmd3])

    rtt = utils.system_output(cmd)

    # eMMC will not have this field; rtt == ''
    # SSD will have zero rotation rate; rtt == '0'
    # For harddisk rtt > 0
    return rtt and int(rtt) > 0


def verify_hdparm_feature(disk_name, feature):
    """
    Check for feature support for SCSI disk using hdparm

    @param disk_name: target disk
    @param feature: hdparm output string of the feature
    """
    cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature)
    ret = utils.system(cmd, ignore_status=True)
    if ret == 0:
        return True
    elif ret == 1:
        return False
    else:
        raise error.TestFail('Error running command %s' % cmd)


def get_storage_error_msg(disk_name, reason):
    """
    Get Error message for storage test which include disk model.
    and also include the firmware version for the SCSI disk

    @param disk_name: target disk
    @param reason: Reason of the error.
    """

    msg = reason

    model = get_disk_model(disk_name)
    msg += ' Disk model: %s' % model

    if is_disk_scsi(disk_name):
        fw = get_disk_firmware_version(disk_name)
        msg += ' firmware: %s' % fw

    return msg


def load_module(module_name, params=None):
    # Checks if a module has already been loaded
    if module_is_loaded(module_name):
        return False

    cmd = '/sbin/modprobe ' + module_name
    if params:
        cmd += ' ' + params
    utils.system(cmd)
    return True


def unload_module(module_name):
    """
    Removes a module. Handles dependencies. If even then it's not possible
    to remove one of the modules, it will trhow an error.CmdError exception.

    @param module_name: Name of the module we want to remove.
    """
    l_raw = utils.system_output("/bin/lsmod").splitlines()
    lsmod = [x for x in l_raw if x.split()[0] == module_name]
    if len(lsmod) > 0:
        line_parts = lsmod[0].split()
        if len(line_parts) == 4:
            submodules = line_parts[3].split(",")
            for submodule in submodules:
                unload_module(submodule)
        utils.system("/sbin/modprobe -r %s" % module_name)
        logging.info("Module %s unloaded", module_name)
    else:
        logging.info("Module %s is already unloaded", module_name)


def module_is_loaded(module_name):
    module_name = module_name.replace('-', '_')
    modules = utils.system_output('/bin/lsmod').splitlines()
    for module in modules:
        if module.startswith(module_name) and module[len(module_name)] == ' ':
            return True
    return False


def get_loaded_modules():
    lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:]
    return [line.split(None, 1)[0] for line in lsmod_output]


def get_huge_page_size():
    output = utils.system_output('grep Hugepagesize /proc/meminfo')
    return int(output.split()[1]) # Assumes units always in kB. :(


def get_num_huge_pages():
    raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages')
    return int(raw_hugepages.split()[2])


def set_num_huge_pages(num):
    utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num)


def ping_default_gateway():
    """Ping the default gateway."""

    network = open('/etc/sysconfig/network')
    m = re.search('GATEWAY=(\S+)', network.read())

    if m:
        gw = m.group(1)
        cmd = 'ping %s -c 5 > /dev/null' % gw
        return utils.system(cmd, ignore_status=True)

    raise error.TestError('Unable to find default gateway')


def drop_caches():
    """Writes back all dirty pages to disk and clears all the caches."""
    utils.system("sync")
    # We ignore failures here as this will fail on 2.6.11 kernels.
    utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True)


def process_is_alive(name_pattern):
    """
    'pgrep name' misses all python processes and also long process names.
    'pgrep -f name' gets all shell commands with name in args.
    So look only for command whose initial pathname ends with name.
    Name itself is an egrep pattern, so it can use | etc for variations.
    """
    return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern,
                        ignore_status=True) == 0


def get_hwclock_seconds(utc=True):
    """
    Return the hardware clock in seconds as a floating point value.
    Use Coordinated Universal Time if utc is True, local time otherwise.
    Raise a ValueError if unable to read the hardware clock.
    """
    cmd = '/sbin/hwclock --debug'
    if utc:
        cmd += ' --utc'
    hwclock_output = utils.system_output(cmd, ignore_status=True)
    match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$',
                      hwclock_output, re.DOTALL)
    if match:
        seconds = int(match.group(1)) + float(match.group(2))
        logging.debug('hwclock seconds = %f', seconds)
        return seconds

    raise ValueError('Unable to read the hardware clock -- ' +
                     hwclock_output)


def set_wake_alarm(alarm_time):
    """
    Set the hardware RTC-based wake alarm to 'alarm_time'.
    """
    utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time))


def set_power_state(state):
    """
    Set the system power state to 'state'.
    """
    utils.write_one_line('/sys/power/state', state)


def standby():
    """
    Power-on suspend (S1)
    """
    set_power_state('standby')


def suspend_to_ram():
    """
    Suspend the system to RAM (S3)
    """
    set_power_state('mem')


def suspend_to_disk():
    """
    Suspend the system to disk (S4)
    """
    set_power_state('disk')