base_utils.py revision 77e0121a0d2a878433b96f4757a44db541530c3c 
1"""
2DO NOT import this file directly - import client/bin/utils.py,
3which will mix this in
4
5Convenience functions for use by tests or whomever.
6
7Note that this file is mixed in by utils.py - note very carefully the
8precedence order defined there
9"""
10import os, shutil, commands, pickle, glob
11import math, re, fnmatch, logging, multiprocessing
12from autotest_lib.client.common_lib import error, utils, magic
13
14
15def grep(pattern, file):
16    """
17    This is mainly to fix the return code inversion from grep
18    Also handles compressed files.
19
20    returns 1 if the pattern is present in the file, 0 if not.
21    """
22    command = 'grep "%s" > /dev/null' % pattern
23    ret = cat_file_to_cmd(file, command, ignore_status=True)
24    return not ret
25
26
27def difflist(list1, list2):
28    """returns items in list2 that are not in list1"""
29    diff = [];
30    for x in list2:
31        if x not in list1:
32            diff.append(x)
33    return diff
34
35
36def cat_file_to_cmd(file, command, ignore_status=0, return_output=False):
37    """
38    equivalent to 'cat file | command' but knows to use
39    zcat or bzcat if appropriate
40    """
41    if not os.path.isfile(file):
42        raise NameError('invalid file %s to cat to command %s'
43                % (file, command))
44
45    if return_output:
46        run_cmd = utils.system_output
47    else:
48        run_cmd = utils.system
49
50    if magic.guess_type(file) == 'application/x-bzip2':
51        cat = 'bzcat'
52    elif magic.guess_type(file) == 'application/x-gzip':
53        cat = 'zcat'
54    else:
55        cat = 'cat'
56    return run_cmd('%s %s | %s' % (cat, file, command),
57                                                    ignore_status=ignore_status)
58
59
60def extract_tarball_to_dir(tarball, dir):
61    """
62    Extract a tarball to a specified directory name instead of whatever
63    the top level of a tarball is - useful for versioned directory names, etc
64    """
65    if os.path.exists(dir):
66        if os.path.isdir(dir):
67            shutil.rmtree(dir)
68        else:
69            os.remove(dir)
70    pwd = os.getcwd()
71    os.chdir(os.path.dirname(os.path.abspath(dir)))
72    newdir = extract_tarball(tarball)
73    os.rename(newdir, dir)
74    os.chdir(pwd)
75
76
77def extract_tarball(tarball):
78    """Returns the directory extracted by the tarball."""
79    extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null',
80                                    return_output=True).splitlines()
81
82    dir = None
83
84    for line in extracted:
85        if line.startswith('./'):
86            line = line[2:]
87        if not line or line == '.':
88            continue
89        topdir = line.split('/')[0]
90        if os.path.isdir(topdir):
91            if dir:
92                assert(dir == topdir)
93            else:
94                dir = topdir
95    if dir:
96        return dir
97    else:
98        raise NameError('extracting tarball produced no dir')
99
100
101def hash_file(filename, size=None, method="md5"):
102    """
103    Calculate the hash of filename.
104    If size is not None, limit to first size bytes.
105    Throw exception if something is wrong with filename.
106    Can be also implemented with bash one-liner (assuming size%1024==0):
107    dd if=filename bs=1024 count=size/1024 | sha1sum -
108
109    @param filename: Path of the file that will have its hash calculated.
110    @param method: Method used to calculate the hash. Supported methods:
111            * md5
112            * sha1
113    @returns: Hash of the file, if something goes wrong, return None.
114    """
115    chunksize = 4096
116    fsize = os.path.getsize(filename)
117
118    if not size or size > fsize:
119        size = fsize
120    f = open(filename, 'rb')
121
122    try:
123        hash = utils.hash(method)
124    except ValueError:
125        logging.error("Unknown hash type %s, returning None", method)
126
127    while size > 0:
128        if chunksize > size:
129            chunksize = size
130        data = f.read(chunksize)
131        if len(data) == 0:
132            logging.debug("Nothing left to read but size=%d", size)
133            break
134        hash.update(data)
135        size -= len(data)
136    f.close()
137    return hash.hexdigest()
138
139
140def unmap_url_cache(cachedir, url, expected_hash, method="md5"):
141    """
142    Downloads a file from a URL to a cache directory. If the file is already
143    at the expected position and has the expected hash, let's not download it
144    again.
145
146    @param cachedir: Directory that might hold a copy of the file we want to
147            download.
148    @param url: URL for the file we want to download.
149    @param expected_hash: Hash string that we expect the file downloaded to
150            have.
151    @param method: Method used to calculate the hash string (md5, sha1).
152    """
153    # Let's convert cachedir to a canonical path, if it's not already
154    cachedir = os.path.realpath(cachedir)
155    if not os.path.isdir(cachedir):
156        try:
157            os.makedirs(cachedir)
158        except:
159            raise ValueError('Could not create cache directory %s' % cachedir)
160    file_from_url = os.path.basename(url)
161    file_local_path = os.path.join(cachedir, file_from_url)
162
163    file_hash = None
164    failure_counter = 0
165    while not file_hash == expected_hash:
166        if os.path.isfile(file_local_path):
167            file_hash = hash_file(file_local_path, method)
168            if file_hash == expected_hash:
169                # File is already at the expected position and ready to go
170                src = file_from_url
171            else:
172                # Let's download the package again, it's corrupted...
173                logging.error("Seems that file %s is corrupted, trying to "
174                              "download it again", file_from_url)
175                src = url
176                failure_counter += 1
177        else:
178            # File is not there, let's download it
179            src = url
180        if failure_counter > 1:
181            raise EnvironmentError("Consistently failed to download the "
182                                   "package %s. Aborting further download "
183                                   "attempts. This might mean either the "
184                                   "network connection has problems or the "
185                                   "expected hash string that was determined "
186                                   "for this file is wrong", file_from_url)
187        file_path = utils.unmap_url(cachedir, src, cachedir)
188
189    return file_path
190
191
192def force_copy(src, dest):
193    """Replace dest with a new copy of src, even if it exists"""
194    if os.path.isfile(dest):
195        os.remove(dest)
196    if os.path.isdir(dest):
197        dest = os.path.join(dest, os.path.basename(src))
198    shutil.copyfile(src, dest)
199    return dest
200
201
202def force_link(src, dest):
203    """Link src to dest, overwriting it if it exists"""
204    return utils.system("ln -sf %s %s" % (src, dest))
205
206
207def file_contains_pattern(file, pattern):
208    """Return true if file contains the specified egrep pattern"""
209    if not os.path.isfile(file):
210        raise NameError('file %s does not exist' % file)
211    return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True)
212
213
214def list_grep(list, pattern):
215    """True if any item in list matches the specified pattern."""
216    compiled = re.compile(pattern)
217    for line in list:
218        match = compiled.search(line)
219        if (match):
220            return 1
221    return 0
222
223
224def get_os_vendor():
225    """Try to guess what's the os vendor
226    """
227    if os.path.isfile('/etc/SuSE-release'):
228        return 'SUSE'
229
230    issue = '/etc/issue'
231
232    if not os.path.isfile(issue):
233        return 'Unknown'
234
235    if file_contains_pattern(issue, 'Red Hat'):
236        return 'Red Hat'
237    elif file_contains_pattern(issue, 'Fedora'):
238        return 'Fedora Core'
239    elif file_contains_pattern(issue, 'SUSE'):
240        return 'SUSE'
241    elif file_contains_pattern(issue, 'Ubuntu'):
242        return 'Ubuntu'
243    elif file_contains_pattern(issue, 'Debian'):
244        return 'Debian'
245    else:
246        return 'Unknown'
247
248
249def get_cc():
250    try:
251        return os.environ['CC']
252    except KeyError:
253        return 'gcc'
254
255
256def get_vmlinux():
257    """Return the full path to vmlinux
258
259    Ahem. This is crap. Pray harder. Bad Martin.
260    """
261    vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r')
262    if os.path.isfile(vmlinux):
263        return vmlinux
264    vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r')
265    if os.path.isfile(vmlinux):
266        return vmlinux
267    return None
268
269
270def get_systemmap():
271    """Return the full path to System.map
272
273    Ahem. This is crap. Pray harder. Bad Martin.
274    """
275    map = '/boot/System.map-%s' % utils.system_output('uname -r')
276    if os.path.isfile(map):
277        return map
278    map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r')
279    if os.path.isfile(map):
280        return map
281    return None
282
283
284def get_modules_dir():
285    """Return the modules dir for the running kernel version"""
286    kernel_version = utils.system_output('uname -r')
287    return '/lib/modules/%s/kernel' % kernel_version
288
289
290_CPUINFO_RE = re.compile(r'^(?P<key>[^\t]*)\t*: ?(?P<value>.*)$')
291
292
293def get_cpuinfo():
294    """Read /proc/cpuinfo and convert to a list of dicts."""
295    cpuinfo = []
296    with open('/proc/cpuinfo', 'r') as f:
297        cpu = {}
298        for line in f:
299            line = line.strip()
300            if not line:
301                cpuinfo.append(cpu)
302                cpu = {}
303                continue
304            match = _CPUINFO_RE.match(line)
305            cpu[match.group('key')] = match.group('value')
306        if cpu:
307            # cpuinfo usually ends in a blank line, so this shouldn't happen.
308            cpuinfo.append(cpu)
309    return cpuinfo
310
311
312def get_cpu_arch():
313    """Work out which CPU architecture we're running on"""
314    f = open('/proc/cpuinfo', 'r')
315    cpuinfo = f.readlines()
316    f.close()
317    if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'):
318        return 'power'
319    elif list_grep(cpuinfo, '^cpu.*POWER4'):
320        return 'power4'
321    elif list_grep(cpuinfo, '^cpu.*POWER5'):
322        return 'power5'
323    elif list_grep(cpuinfo, '^cpu.*POWER6'):
324        return 'power6'
325    elif list_grep(cpuinfo, '^cpu.*POWER7'):
326        return 'power7'
327    elif list_grep(cpuinfo, '^cpu.*PPC970'):
328        return 'power970'
329    elif list_grep(cpuinfo, 'ARM'):
330        return 'arm'
331    elif list_grep(cpuinfo, '^flags.*:.* lm .*'):
332        return 'x86_64'
333    else:
334        return 'i386'
335
336
337def get_arm_soc_family():
338    """Work out which ARM SoC we're running on"""
339    f = open('/proc/cpuinfo', 'r')
340    cpuinfo = f.readlines()
341    f.close()
342    if list_grep(cpuinfo, 'EXYNOS5'):
343        return 'exynos5'
344    elif list_grep(cpuinfo, 'Tegra'):
345        return 'tegra'
346    elif list_grep(cpuinfo, 'Rockchip'):
347        return 'rockchip'
348    return 'arm'
349
350
351def get_cpu_soc_family():
352    """Like get_cpu_arch, but for ARM, returns the SoC family name"""
353    family = get_cpu_arch()
354    if family == 'arm':
355       family = get_arm_soc_family()
356    return family
357
358
359INTEL_UARCH_TABLE = {
360    '06_36': 'Atom',
361    '06_26': 'Atom',
362    '06_1C': 'Atom',
363    '06_3D': 'Broadwell',
364    '06_3F': 'Haswell',
365    '06_3C': 'Haswell',
366    '06_46': 'Haswell',
367    '06_45': 'Haswell',
368    '06_3E': 'IvyBridge',
369    '06_3A': 'IvyBridge',
370    '06_2D': 'SandyBridge',
371    '06_2A': 'SandyBridge',
372    '06_2F': 'Westmere',
373    '06_2C': 'Westmere',
374    '06_25': 'Westmere',
375    '06_2E': 'Nehalem',
376    '06_1F': 'Nehalem',
377    '06_1E': 'Nehalem',
378    '06_1D': 'Nehalem',
379    '06_1A': 'Nehalem',
380    '06_17': 'Nehalem',
381    '06_16': 'Merom',
382    '06_0F': 'Merom',
383    '0F_06': 'Presler',
384    '0F_04': 'Prescott',
385    '0F_03': 'Prescott',
386    '06_0D': 'Dothan',
387}
388
389
390def get_intel_cpu_uarch(numeric=False):
391    """Return the Intel microarchitecture we're running on, or None.
392
393    Returns None if this is not an Intel CPU. Returns the family and model as
394    underscore-separated hex (per Intel manual convention) if the uarch is not
395    known, or if numeric is True.
396    """
397    if not get_current_kernel_arch().startswith('x86'):
398        return None
399    cpuinfo = get_cpuinfo()[0]
400    if cpuinfo['vendor_id'] != 'GenuineIntel':
401        return None
402    family_model = '%02X_%02X' % (int(cpuinfo['cpu family']),
403                                  int(cpuinfo['model']))
404    if numeric:
405        return family_model
406    return INTEL_UARCH_TABLE.get(family_model, family_model)
407
408
409def get_current_kernel_arch():
410    """Get the machine architecture, now just a wrap of 'uname -m'."""
411    return os.popen('uname -m').read().rstrip()
412
413
414def get_file_arch(filename):
415    # -L means follow symlinks
416    file_data = utils.system_output('file -L ' + filename)
417    if file_data.count('80386'):
418        return 'i386'
419    return None
420
421
422def count_cpus():
423    """number of CPUs in the local machine according to /proc/cpuinfo"""
424    try:
425      return multiprocessing.cpu_count()
426    except Exception as e:
427      logging.exception('can not get cpu count from'
428                        ' multiprocessing.cpu_count()')
429    cpuinfo = get_cpuinfo()
430    # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582.
431    return len(cpuinfo) or 1
432
433
434def cpu_online_map():
435    """
436    Check out the available cpu online map
437    """
438    cpuinfo = get_cpuinfo()
439    cpus = []
440    for cpu in cpuinfo:
441      cpus.append(cpu['processor'])  # grab cpu number
442    return cpus
443
444
445def get_cpu_family():
446    cpuinfo = get_cpuinfo()[0]
447    return int(cpuinfo['cpu_family'])
448
449
450def get_cpu_vendor():
451    cpuinfo = get_cpuinfo()
452    vendors = [cpu['vendor_id'] for cpu in cpuinfo]
453    for v in vendors[1:]:
454        if v != vendors[0]:
455            raise error.TestError('multiple cpu vendors found: ' + str(vendors))
456    return vendors[0]
457
458
459def probe_cpus():
460    """
461    This routine returns a list of cpu devices found under
462    /sys/devices/system/cpu.
463    """
464    cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*'
465    return utils.system_output(cmd).splitlines()
466
467
468# Returns total memory in kb
469def read_from_meminfo(key):
470    meminfo = utils.system_output('grep %s /proc/meminfo' % key)
471    return int(re.search(r'\d+', meminfo).group(0))
472
473
474def memtotal():
475    return read_from_meminfo('MemTotal')
476
477
478def freememtotal():
479    return read_from_meminfo('MemFree')
480
481def usable_memtotal():
482    # Reserved 5% for OS use
483    return int(read_from_meminfo('MemFree') * 0.95)
484
485
486def rounded_memtotal():
487    # Get total of all physical mem, in kbytes
488    usable_kbytes = memtotal()
489    # usable_kbytes is system's usable DRAM in kbytes,
490    #   as reported by memtotal() from device /proc/meminfo memtotal
491    #   after Linux deducts 1.5% to 5.1% for system table overhead
492    # Undo the unknown actual deduction by rounding up
493    #   to next small multiple of a big power-of-two
494    #   eg  12GB - 5.1% gets rounded back up to 12GB
495    mindeduct = 0.015  # 1.5 percent
496    maxdeduct = 0.055  # 5.5 percent
497    # deduction range 1.5% .. 5.5% supports physical mem sizes
498    #    6GB .. 12GB in steps of .5GB
499    #   12GB .. 24GB in steps of 1 GB
500    #   24GB .. 48GB in steps of 2 GB ...
501    # Finer granularity in physical mem sizes would require
502    #   tighter spread between min and max possible deductions
503
504    # increase mem size by at least min deduction, without rounding
505    min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
506    # increase mem size further by 2**n rounding, by 0..roundKb or more
507    round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
508    # find least binary roundup 2**n that covers worst-cast roundKb
509    mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
510    # have round_kbytes <= mod2n < round_kbytes*2
511    # round min_kbytes up to next multiple of mod2n
512    phys_kbytes = min_kbytes + mod2n - 1
513    phys_kbytes = phys_kbytes - (phys_kbytes % mod2n)  # clear low bits
514    return phys_kbytes
515
516
517def sysctl(key, value=None):
518    """Generic implementation of sysctl, to read and write.
519
520    @param key: A location under /proc/sys
521    @param value: If not None, a value to write into the sysctl.
522
523    @return The single-line sysctl value as a string.
524    """
525    path = '/proc/sys/%s' % key
526    if value is not None:
527        utils.write_one_line(path, str(value))
528    return utils.read_one_line(path)
529
530
531def sysctl_kernel(key, value=None):
532    """(Very) partial implementation of sysctl, for kernel params"""
533    if value is not None:
534        # write
535        utils.write_one_line('/proc/sys/kernel/%s' % key, str(value))
536    else:
537        # read
538        out = utils.read_one_line('/proc/sys/kernel/%s' % key)
539        return int(re.search(r'\d+', out).group(0))
540
541
542def _convert_exit_status(sts):
543    if os.WIFSIGNALED(sts):
544        return -os.WTERMSIG(sts)
545    elif os.WIFEXITED(sts):
546        return os.WEXITSTATUS(sts)
547    else:
548        # impossible?
549        raise RuntimeError("Unknown exit status %d!" % sts)
550
551
552def where_art_thy_filehandles():
553    """Dump the current list of filehandles"""
554    os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())
555
556
557def print_to_tty(string):
558    """Output string straight to the tty"""
559    open('/dev/tty', 'w').write(string + '\n')
560
561
562def dump_object(object):
563    """Dump an object's attributes and methods
564
565    kind of like dir()
566    """
567    for item in object.__dict__.iteritems():
568        print item
569        try:
570            (key, value) = item
571            dump_object(value)
572        except:
573            continue
574
575
576def environ(env_key):
577    """return the requested environment variable, or '' if unset"""
578    if (os.environ.has_key(env_key)):
579        return os.environ[env_key]
580    else:
581        return ''
582
583
584def prepend_path(newpath, oldpath):
585    """prepend newpath to oldpath"""
586    if (oldpath):
587        return newpath + ':' + oldpath
588    else:
589        return newpath
590
591
592def append_path(oldpath, newpath):
593    """append newpath to oldpath"""
594    if (oldpath):
595        return oldpath + ':' + newpath
596    else:
597        return newpath
598
599
600_TIME_OUTPUT_RE = re.compile(
601        r'([\d\.]*)user ([\d\.]*)system '
602        r'(\d*):([\d\.]*)elapsed (\d*)%CPU')
603
604
605def avgtime_print(dir):
606    """ Calculate some benchmarking statistics.
607        Input is a directory containing a file called 'time'.
608        File contains one-per-line results of /usr/bin/time.
609        Output is average Elapsed, User, and System time in seconds,
610          and average CPU percentage.
611    """
612    user = system = elapsed = cpu = count = 0
613    with open(dir + "/time") as f:
614        for line in f:
615            try:
616                m = _TIME_OUTPUT_RE.match(line);
617                user += float(m.group(1))
618                system += float(m.group(2))
619                elapsed += (float(m.group(3)) * 60) + float(m.group(4))
620                cpu += float(m.group(5))
621                count += 1
622            except:
623                raise ValueError("badly formatted times")
624
625    return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \
626          (elapsed / count, user / count, system / count, cpu / count)
627
628
629def to_seconds(time_string):
630    """Converts a string in M+:SS.SS format to S+.SS"""
631    elts = time_string.split(':')
632    if len(elts) == 1:
633        return time_string
634    return str(int(elts[0]) * 60 + float(elts[1]))
635
636
637_TIME_OUTPUT_RE_2 = re.compile(r'(.*?)user (.*?)system (.*?)elapsed')
638
639
640def extract_all_time_results(results_string):
641    """Extract user, system, and elapsed times into a list of tuples"""
642    results = []
643    for result in _TIME_OUTPUT_RE_2.findall(results_string):
644        results.append(tuple([to_seconds(elt) for elt in result]))
645    return results
646
647
648def running_config():
649    """
650    Return path of config file of the currently running kernel
651    """
652    version = utils.system_output('uname -r')
653    for config in ('/proc/config.gz', \
654                   '/boot/config-%s' % version,
655                   '/lib/modules/%s/build/.config' % version):
656        if os.path.isfile(config):
657            return config
658    return None
659
660
661def check_for_kernel_feature(feature):
662    config = running_config()
663
664    if not config:
665        raise TypeError("Can't find kernel config file")
666
667    if magic.guess_type(config) == 'application/x-gzip':
668        grep = 'zgrep'
669    else:
670        grep = 'grep'
671    grep += ' ^CONFIG_%s= %s' % (feature, config)
672
673    if not utils.system_output(grep, ignore_status=True):
674        raise ValueError("Kernel doesn't have a %s feature" % (feature))
675
676
677def check_glibc_ver(ver):
678    glibc_ver = commands.getoutput('ldd --version').splitlines()[0]
679    glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group()
680    if utils.compare_versions(glibc_ver, ver) == -1:
681        raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." %
682                              (glibc_ver, ver))
683
684def check_kernel_ver(ver):
685    kernel_ver = utils.system_output('uname -r')
686    kv_tmp = re.split(r'[-]', kernel_ver)[0:3]
687    # In compare_versions, if v1 < v2, return value == -1
688    if utils.compare_versions(kv_tmp[0], ver) == -1:
689        raise error.TestError("Kernel too old (%s). Kernel > %s is needed." %
690                              (kernel_ver, ver))
691
692
693def human_format(number):
694    # Convert number to kilo / mega / giga format.
695    if number < 1024:
696        return "%d" % number
697    kilo = float(number) / 1024.0
698    if kilo < 1024:
699        return "%.2fk" % kilo
700    meg = kilo / 1024.0
701    if meg < 1024:
702        return "%.2fM" % meg
703    gig = meg / 1024.0
704    return "%.2fG" % gig
705
706
707def numa_nodes():
708    node_paths = glob.glob('/sys/devices/system/node/node*')
709    nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths]
710    return (sorted(nodes))
711
712
713def node_size():
714    nodes = max(len(numa_nodes()), 1)
715    return ((memtotal() * 1024) / nodes)
716
717
718def pickle_load(filename):
719    return pickle.load(open(filename, 'r'))
720
721
722# Return the kernel version and build timestamp.
723def running_os_release():
724    return os.uname()[2:4]
725
726
727def running_os_ident():
728    (version, timestamp) = running_os_release()
729    return version + '::' + timestamp
730
731
732def running_os_full_version():
733    (version, timestamp) = running_os_release()
734    return version
735
736
737# much like find . -name 'pattern'
738def locate(pattern, root=os.getcwd()):
739    for path, dirs, files in os.walk(root):
740        for f in files:
741            if fnmatch.fnmatch(f, pattern):
742                yield os.path.abspath(os.path.join(path, f))
743
744
745def freespace(path):
746    """Return the disk free space, in bytes"""
747    s = os.statvfs(path)
748    return s.f_bavail * s.f_bsize
749
750
751def disk_block_size(path):
752    """Return the disk block size, in bytes"""
753    return os.statvfs(path).f_bsize
754
755
756_DISK_PARTITION_3_RE = re.compile(r'^(/dev/hd[a-z]+)3', re.M)
757
758def get_disks():
759    df_output = utils.system_output('df')
760    return _DISK_PARTITION_3_RE.findall(df_output)
761
762
763def get_disk_size(disk_name):
764    """
765    Return size of disk in byte. Return 0 in Error Case
766
767    @param disk_name: disk name to find size
768    """
769    device = os.path.basename(disk_name)
770    for line in file('/proc/partitions'):
771        try:
772            _, _, blocks, name = re.split(r' +', line.strip())
773        except ValueError:
774            continue
775        if name == device:
776            return 1024 * int(blocks)
777    return 0
778
779
780def get_disk_size_gb(disk_name):
781    """
782    Return size of disk in GB (10^9). Return 0 in Error Case
783
784    @param disk_name: disk name to find size
785    """
786    return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5)
787
788
789def get_disk_model(disk_name):
790    """
791    Return model name for internal storage device
792
793    @param disk_name: disk name to find model
794    """
795    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
796    cmd2 = 'grep -E "ID_(NAME|MODEL)="'
797    cmd3 = 'cut -f 2 -d"="'
798    cmd = ' | '.join([cmd1, cmd2, cmd3])
799    return utils.system_output(cmd)
800
801
802_DISK_DEV_RE = re.compile(r'/dev/sd[a-z]|/dev/mmcblk[0-9]*')
803
804
805def get_disk_from_filename(filename):
806    """
807    Return the disk device the filename is on.
808    If the file is on tmpfs or other special file systems,
809    return None.
810
811    @param filename: name of file, full path.
812    """
813
814    if not os.path.exists(filename):
815        raise error.TestError('file %s missing' % filename)
816
817    if filename[0] != '/':
818        raise error.TestError('This code works only with full path')
819
820    m = _DISK_DEV_RE.match(filename)
821    while not m:
822        if filename[0] != '/':
823            return None
824        if filename == '/dev/root':
825            cmd = 'rootdev -d -s'
826        elif filename.startswith('/dev/mapper'):
827            cmd = 'dmsetup table "%s"' % os.path.basename(filename)
828            dmsetup_output = utils.system_output(cmd).split(' ')
829            if dmsetup_output[2] == 'verity':
830                maj_min = dmsetup_output[4]
831            elif dmsetup_output[2] == 'crypt':
832                maj_min = dmsetup_output[6]
833            cmd = 'realpath "/dev/block/%s"' % maj_min
834        elif filename.startswith('/dev/loop'):
835            cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename
836        else:
837            cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename
838        filename = utils.system_output(cmd)
839        m = _DISK_DEV_RE.match(filename)
840    return m.group(0)
841
842
843def get_disk_firmware_version(disk_name):
844    """
845    Return firmware version for internal storage device. (empty string for eMMC)
846
847    @param disk_name: disk name to find model
848    """
849    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
850    cmd2 = 'grep -E "ID_REVISION="'
851    cmd3 = 'cut -f 2 -d"="'
852    cmd = ' | '.join([cmd1, cmd2, cmd3])
853    return utils.system_output(cmd)
854
855
856def is_disk_scsi(disk_name):
857    """
858    Return true if disk is a scsi device, return false otherwise
859
860    @param disk_name: disk name check
861    """
862    return re.match('/dev/sd[a-z]+', disk_name)
863
864
865def is_disk_harddisk(disk_name):
866    """
867    Return true if disk is a harddisk, return false otherwise
868
869    @param disk_name: disk name check
870    """
871    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
872    cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="'
873    cmd3 = 'cut -f 2 -d"="'
874    cmd = ' | '.join([cmd1, cmd2, cmd3])
875
876    rtt = utils.system_output(cmd)
877
878    # eMMC will not have this field; rtt == ''
879    # SSD will have zero rotation rate; rtt == '0'
880    # For harddisk rtt > 0
881    return rtt and int(rtt) > 0
882
883
884def verify_hdparm_feature(disk_name, feature):
885    """
886    Check for feature support for SCSI disk using hdparm
887
888    @param disk_name: target disk
889    @param feature: hdparm output string of the feature
890    """
891    cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature)
892    ret = utils.system(cmd, ignore_status=True)
893    if ret == 0:
894        return True
895    elif ret == 1:
896        return False
897    else:
898        raise error.TestFail('Error running command %s' % cmd)
899
900
901def get_storage_error_msg(disk_name, reason):
902    """
903    Get Error message for storage test which include disk model.
904    and also include the firmware version for the SCSI disk
905
906    @param disk_name: target disk
907    @param reason: Reason of the error.
908    """
909
910    msg = reason
911
912    model = get_disk_model(disk_name)
913    msg += ' Disk model: %s' % model
914
915    if is_disk_scsi(disk_name):
916        fw = get_disk_firmware_version(disk_name)
917        msg += ' firmware: %s' % fw
918
919    return msg
920
921
922def load_module(module_name):
923    # Checks if a module has already been loaded
924    if module_is_loaded(module_name):
925        return False
926
927    utils.system('/sbin/modprobe ' + module_name)
928    return True
929
930
931def unload_module(module_name):
932    """
933    Removes a module. Handles dependencies. If even then it's not possible
934    to remove one of the modules, it will trhow an error.CmdError exception.
935
936    @param module_name: Name of the module we want to remove.
937    """
938    l_raw = utils.system_output("/bin/lsmod").splitlines()
939    lsmod = [x for x in l_raw if x.split()[0] == module_name]
940    if len(lsmod) > 0:
941        line_parts = lsmod[0].split()
942        if len(line_parts) == 4:
943            submodules = line_parts[3].split(",")
944            for submodule in submodules:
945                unload_module(submodule)
946        utils.system("/sbin/modprobe -r %s" % module_name)
947        logging.info("Module %s unloaded", module_name)
948    else:
949        logging.info("Module %s is already unloaded", module_name)
950
951
952def module_is_loaded(module_name):
953    module_name = module_name.replace('-', '_')
954    modules = utils.system_output('/bin/lsmod').splitlines()
955    for module in modules:
956        if module.startswith(module_name) and module[len(module_name)] == ' ':
957            return True
958    return False
959
960
961def get_loaded_modules():
962    lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:]
963    return [line.split(None, 1)[0] for line in lsmod_output]
964
965
966def get_huge_page_size():
967    output = utils.system_output('grep Hugepagesize /proc/meminfo')
968    return int(output.split()[1]) # Assumes units always in kB. :(
969
970
971def get_num_huge_pages():
972    raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages')
973    return int(raw_hugepages.split()[2])
974
975
976def set_num_huge_pages(num):
977    utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num)
978
979
980def ping_default_gateway():
981    """Ping the default gateway."""
982
983    network = open('/etc/sysconfig/network')
984    m = re.search('GATEWAY=(\S+)', network.read())
985
986    if m:
987        gw = m.group(1)
988        cmd = 'ping %s -c 5 > /dev/null' % gw
989        return utils.system(cmd, ignore_status=True)
990
991    raise error.TestError('Unable to find default gateway')
992
993
994def drop_caches():
995    """Writes back all dirty pages to disk and clears all the caches."""
996    utils.system("sync")
997    # We ignore failures here as this will fail on 2.6.11 kernels.
998    utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True)
999
1000
1001def process_is_alive(name_pattern):
1002    """
1003    'pgrep name' misses all python processes and also long process names.
1004    'pgrep -f name' gets all shell commands with name in args.
1005    So look only for command whose initial pathname ends with name.
1006    Name itself is an egrep pattern, so it can use | etc for variations.
1007    """
1008    return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern,
1009                        ignore_status=True) == 0
1010
1011
1012def get_hwclock_seconds(utc=True):
1013    """
1014    Return the hardware clock in seconds as a floating point value.
1015    Use Coordinated Universal Time if utc is True, local time otherwise.
1016    Raise a ValueError if unable to read the hardware clock.
1017    """
1018    cmd = '/sbin/hwclock --debug'
1019    if utc:
1020        cmd += ' --utc'
1021    hwclock_output = utils.system_output(cmd, ignore_status=True)
1022    match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$',
1023                      hwclock_output, re.DOTALL)
1024    if match:
1025        seconds = int(match.group(1)) + float(match.group(2))
1026        logging.debug('hwclock seconds = %f', seconds)
1027        return seconds
1028
1029    raise ValueError('Unable to read the hardware clock -- ' +
1030                     hwclock_output)
1031
1032
1033def set_wake_alarm(alarm_time):
1034    """
1035    Set the hardware RTC-based wake alarm to 'alarm_time'.
1036    """
1037    utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time))
1038
1039
1040def set_power_state(state):
1041    """
1042    Set the system power state to 'state'.
1043    """
1044    utils.write_one_line('/sys/power/state', state)
1045
1046
1047def standby():
1048    """
1049    Power-on suspend (S1)
1050    """
1051    set_power_state('standby')
1052
1053
1054def suspend_to_ram():
1055    """
1056    Suspend the system to RAM (S3)
1057    """
1058    set_power_state('mem')
1059
1060
1061def suspend_to_disk():
1062    """
1063    Suspend the system to disk (S4)
1064    """
1065    set_power_state('disk')
1066