1"""
2DO NOT import this file directly - import client/bin/utils.py,
3which will mix this in
4
5Convenience functions for use by tests or whomever.
6
7Note that this file is mixed in by utils.py - note very carefully the
8precedence order defined there
9"""
10
11import commands
12import fnmatch
13import glob
14import logging
15import math
16import multiprocessing
17import os
18import pickle
19import re
20import shutil
21
22from autotest_lib.client.common_lib import error
23from autotest_lib.client.common_lib import magic
24from autotest_lib.client.common_lib import utils
25
26
27def grep(pattern, file):
28    """
29    This is mainly to fix the return code inversion from grep
30    Also handles compressed files.
31
32    returns 1 if the pattern is present in the file, 0 if not.
33    """
34    command = 'grep "%s" > /dev/null' % pattern
35    ret = cat_file_to_cmd(file, command, ignore_status=True)
36    return not ret
37
38
39def difflist(list1, list2):
40    """returns items in list2 that are not in list1"""
41    diff = [];
42    for x in list2:
43        if x not in list1:
44            diff.append(x)
45    return diff
46
47
48def cat_file_to_cmd(file, command, ignore_status=0, return_output=False):
49    """
50    equivalent to 'cat file | command' but knows to use
51    zcat or bzcat if appropriate
52    """
53    if not os.path.isfile(file):
54        raise NameError('invalid file %s to cat to command %s'
55                % (file, command))
56
57    if return_output:
58        run_cmd = utils.system_output
59    else:
60        run_cmd = utils.system
61
62    if magic.guess_type(file) == 'application/x-bzip2':
63        cat = 'bzcat'
64    elif magic.guess_type(file) == 'application/x-gzip':
65        cat = 'zcat'
66    else:
67        cat = 'cat'
68    return run_cmd('%s %s | %s' % (cat, file, command),
69                                                    ignore_status=ignore_status)
70
71
72def extract_tarball_to_dir(tarball, dir):
73    """
74    Extract a tarball to a specified directory name instead of whatever
75    the top level of a tarball is - useful for versioned directory names, etc
76    """
77    if os.path.exists(dir):
78        if os.path.isdir(dir):
79            shutil.rmtree(dir)
80        else:
81            os.remove(dir)
82    pwd = os.getcwd()
83    os.chdir(os.path.dirname(os.path.abspath(dir)))
84    newdir = extract_tarball(tarball)
85    os.rename(newdir, dir)
86    os.chdir(pwd)
87
88
89def extract_tarball(tarball):
90    """Returns the directory extracted by the tarball."""
91    extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null',
92                                    return_output=True).splitlines()
93
94    dir = None
95
96    for line in extracted:
97        if line.startswith('./'):
98            line = line[2:]
99        if not line or line == '.':
100            continue
101        topdir = line.split('/')[0]
102        if os.path.isdir(topdir):
103            if dir:
104                assert(dir == topdir)
105            else:
106                dir = topdir
107    if dir:
108        return dir
109    else:
110        raise NameError('extracting tarball produced no dir')
111
112
113def unmap_url_cache(cachedir, url, expected_hash, method="md5"):
114    """
115    Downloads a file from a URL to a cache directory. If the file is already
116    at the expected position and has the expected hash, let's not download it
117    again.
118
119    @param cachedir: Directory that might hold a copy of the file we want to
120            download.
121    @param url: URL for the file we want to download.
122    @param expected_hash: Hash string that we expect the file downloaded to
123            have.
124    @param method: Method used to calculate the hash string (md5, sha1).
125    """
126    # Let's convert cachedir to a canonical path, if it's not already
127    cachedir = os.path.realpath(cachedir)
128    if not os.path.isdir(cachedir):
129        try:
130            os.makedirs(cachedir)
131        except:
132            raise ValueError('Could not create cache directory %s' % cachedir)
133    file_from_url = os.path.basename(url)
134    file_local_path = os.path.join(cachedir, file_from_url)
135
136    file_hash = None
137    failure_counter = 0
138    while not file_hash == expected_hash:
139        if os.path.isfile(file_local_path):
140            file_hash = hash_file(file_local_path, method)
141            if file_hash == expected_hash:
142                # File is already at the expected position and ready to go
143                src = file_from_url
144            else:
145                # Let's download the package again, it's corrupted...
146                logging.error("Seems that file %s is corrupted, trying to "
147                              "download it again", file_from_url)
148                src = url
149                failure_counter += 1
150        else:
151            # File is not there, let's download it
152            src = url
153        if failure_counter > 1:
154            raise EnvironmentError("Consistently failed to download the "
155                                   "package %s. Aborting further download "
156                                   "attempts. This might mean either the "
157                                   "network connection has problems or the "
158                                   "expected hash string that was determined "
159                                   "for this file is wrong", file_from_url)
160        file_path = utils.unmap_url(cachedir, src, cachedir)
161
162    return file_path
163
164
165def force_copy(src, dest):
166    """Replace dest with a new copy of src, even if it exists"""
167    if os.path.isfile(dest):
168        os.remove(dest)
169    if os.path.isdir(dest):
170        dest = os.path.join(dest, os.path.basename(src))
171    shutil.copyfile(src, dest)
172    return dest
173
174
175def force_link(src, dest):
176    """Link src to dest, overwriting it if it exists"""
177    return utils.system("ln -sf %s %s" % (src, dest))
178
179
180def file_contains_pattern(file, pattern):
181    """Return true if file contains the specified egrep pattern"""
182    if not os.path.isfile(file):
183        raise NameError('file %s does not exist' % file)
184    return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True)
185
186
187def list_grep(list, pattern):
188    """True if any item in list matches the specified pattern."""
189    compiled = re.compile(pattern)
190    for line in list:
191        match = compiled.search(line)
192        if (match):
193            return 1
194    return 0
195
196
197def get_os_vendor():
198    """Try to guess what's the os vendor
199    """
200    if os.path.isfile('/etc/SuSE-release'):
201        return 'SUSE'
202
203    issue = '/etc/issue'
204
205    if not os.path.isfile(issue):
206        return 'Unknown'
207
208    if file_contains_pattern(issue, 'Red Hat'):
209        return 'Red Hat'
210    elif file_contains_pattern(issue, 'Fedora'):
211        return 'Fedora Core'
212    elif file_contains_pattern(issue, 'SUSE'):
213        return 'SUSE'
214    elif file_contains_pattern(issue, 'Ubuntu'):
215        return 'Ubuntu'
216    elif file_contains_pattern(issue, 'Debian'):
217        return 'Debian'
218    else:
219        return 'Unknown'
220
221
222def get_cc():
223    try:
224        return os.environ['CC']
225    except KeyError:
226        return 'gcc'
227
228
229def get_vmlinux():
230    """Return the full path to vmlinux
231
232    Ahem. This is crap. Pray harder. Bad Martin.
233    """
234    vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r')
235    if os.path.isfile(vmlinux):
236        return vmlinux
237    vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r')
238    if os.path.isfile(vmlinux):
239        return vmlinux
240    return None
241
242
243def get_systemmap():
244    """Return the full path to System.map
245
246    Ahem. This is crap. Pray harder. Bad Martin.
247    """
248    map = '/boot/System.map-%s' % utils.system_output('uname -r')
249    if os.path.isfile(map):
250        return map
251    map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r')
252    if os.path.isfile(map):
253        return map
254    return None
255
256
257def get_modules_dir():
258    """Return the modules dir for the running kernel version"""
259    kernel_version = utils.system_output('uname -r')
260    return '/lib/modules/%s/kernel' % kernel_version
261
262
263_CPUINFO_RE = re.compile(r'^(?P<key>[^\t]*)\t*: ?(?P<value>.*)$')
264
265
266def get_cpuinfo():
267    """Read /proc/cpuinfo and convert to a list of dicts."""
268    cpuinfo = []
269    with open('/proc/cpuinfo', 'r') as f:
270        cpu = {}
271        for line in f:
272            line = line.strip()
273            if not line:
274                cpuinfo.append(cpu)
275                cpu = {}
276                continue
277            match = _CPUINFO_RE.match(line)
278            cpu[match.group('key')] = match.group('value')
279        if cpu:
280            # cpuinfo usually ends in a blank line, so this shouldn't happen.
281            cpuinfo.append(cpu)
282    return cpuinfo
283
284
285def get_cpu_arch():
286    """Work out which CPU architecture we're running on"""
287    f = open('/proc/cpuinfo', 'r')
288    cpuinfo = f.readlines()
289    f.close()
290    if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'):
291        return 'power'
292    elif list_grep(cpuinfo, '^cpu.*POWER4'):
293        return 'power4'
294    elif list_grep(cpuinfo, '^cpu.*POWER5'):
295        return 'power5'
296    elif list_grep(cpuinfo, '^cpu.*POWER6'):
297        return 'power6'
298    elif list_grep(cpuinfo, '^cpu.*POWER7'):
299        return 'power7'
300    elif list_grep(cpuinfo, '^cpu.*PPC970'):
301        return 'power970'
302    elif list_grep(cpuinfo, 'ARM'):
303        return 'arm'
304    elif list_grep(cpuinfo, '^flags.*:.* lm .*'):
305        return 'x86_64'
306    elif list_grep(cpuinfo, 'CPU.*implementer.*0x41'):
307        return 'arm'
308    else:
309        return 'i386'
310
311
312def get_arm_soc_family_from_devicetree():
313    """
314    Work out which ARM SoC we're running on based on the 'compatible' property
315    of the base node of devicetree, if it exists.
316    """
317    devicetree_compatible = '/sys/firmware/devicetree/base/compatible'
318    if not os.path.isfile(devicetree_compatible):
319        return None
320    f = open(devicetree_compatible, 'r')
321    compatible = f.readlines()
322    f.close()
323    if list_grep(compatible, 'rk3399'):
324        return 'rockchip'
325    elif list_grep(compatible, 'mt8173'):
326        return 'mediatek'
327    return None
328
329
330def get_arm_soc_family():
331    """Work out which ARM SoC we're running on"""
332    family = get_arm_soc_family_from_devicetree()
333    if family is not None:
334        return family
335
336    f = open('/proc/cpuinfo', 'r')
337    cpuinfo = f.readlines()
338    f.close()
339    if list_grep(cpuinfo, 'EXYNOS5'):
340        return 'exynos5'
341    elif list_grep(cpuinfo, 'Tegra'):
342        return 'tegra'
343    elif list_grep(cpuinfo, 'Rockchip'):
344        return 'rockchip'
345    return 'arm'
346
347
348def get_cpu_soc_family():
349    """Like get_cpu_arch, but for ARM, returns the SoC family name"""
350    f = open('/proc/cpuinfo', 'r')
351    cpuinfo = f.readlines()
352    f.close()
353    family = get_cpu_arch()
354    if family == 'arm':
355        family = get_arm_soc_family()
356    if list_grep(cpuinfo, '^vendor_id.*:.*AMD'):
357        family = 'amd'
358    return family
359
360
361INTEL_UARCH_TABLE = {
362    '06_1C': 'Atom',
363    '06_26': 'Atom',
364    '06_36': 'Atom',
365    '06_4C': 'Braswell',
366    '06_3D': 'Broadwell',
367    '06_0D': 'Dothan',
368    '06_3A': 'IvyBridge',
369    '06_3E': 'IvyBridge',
370    '06_3C': 'Haswell',
371    '06_3F': 'Haswell',
372    '06_45': 'Haswell',
373    '06_46': 'Haswell',
374    '06_0F': 'Merom',
375    '06_16': 'Merom',
376    '06_17': 'Nehalem',
377    '06_1A': 'Nehalem',
378    '06_1D': 'Nehalem',
379    '06_1E': 'Nehalem',
380    '06_1F': 'Nehalem',
381    '06_2E': 'Nehalem',
382    '06_2A': 'SandyBridge',
383    '06_2D': 'SandyBridge',
384    '06_4E': 'Skylake',
385    '0F_03': 'Prescott',
386    '0F_04': 'Prescott',
387    '0F_06': 'Presler',
388    '06_25': 'Westmere',
389    '06_2C': 'Westmere',
390    '06_2F': 'Westmere',
391}
392
393
394def get_intel_cpu_uarch(numeric=False):
395    """Return the Intel microarchitecture we're running on, or None.
396
397    Returns None if this is not an Intel CPU. Returns the family and model as
398    underscore-separated hex (per Intel manual convention) if the uarch is not
399    known, or if numeric is True.
400    """
401    if not get_current_kernel_arch().startswith('x86'):
402        return None
403    cpuinfo = get_cpuinfo()[0]
404    if cpuinfo['vendor_id'] != 'GenuineIntel':
405        return None
406    family_model = '%02X_%02X' % (int(cpuinfo['cpu family']),
407                                  int(cpuinfo['model']))
408    if numeric:
409        return family_model
410    return INTEL_UARCH_TABLE.get(family_model, family_model)
411
412
413def get_current_kernel_arch():
414    """Get the machine architecture, now just a wrap of 'uname -m'."""
415    return os.popen('uname -m').read().rstrip()
416
417
418def get_file_arch(filename):
419    # -L means follow symlinks
420    file_data = utils.system_output('file -L ' + filename)
421    if file_data.count('80386'):
422        return 'i386'
423    return None
424
425
426def count_cpus():
427    """number of CPUs in the local machine according to /proc/cpuinfo"""
428    try:
429       return multiprocessing.cpu_count()
430    except Exception as e:
431       logging.exception('can not get cpu count from'
432                        ' multiprocessing.cpu_count()')
433    cpuinfo = get_cpuinfo()
434    # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582.
435    return len(cpuinfo) or 1
436
437
438def cpu_online_map():
439    """
440    Check out the available cpu online map
441    """
442    cpuinfo = get_cpuinfo()
443    cpus = []
444    for cpu in cpuinfo:
445        cpus.append(cpu['processor'])  # grab cpu number
446    return cpus
447
448
449def get_cpu_family():
450    cpuinfo = get_cpuinfo()[0]
451    return int(cpuinfo['cpu_family'])
452
453
454def get_cpu_vendor():
455    cpuinfo = get_cpuinfo()
456    vendors = [cpu['vendor_id'] for cpu in cpuinfo]
457    for v in vendors[1:]:
458        if v != vendors[0]:
459            raise error.TestError('multiple cpu vendors found: ' + str(vendors))
460    return vendors[0]
461
462
463def probe_cpus():
464    """
465    This routine returns a list of cpu devices found under
466    /sys/devices/system/cpu.
467    """
468    cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*'
469    return utils.system_output(cmd).splitlines()
470
471
472# Returns total memory in kb
473def read_from_meminfo(key):
474    meminfo = utils.system_output('grep %s /proc/meminfo' % key)
475    return int(re.search(r'\d+', meminfo).group(0))
476
477
478def memtotal():
479    return read_from_meminfo('MemTotal')
480
481
482def freememtotal():
483    return read_from_meminfo('MemFree')
484
485def usable_memtotal():
486    # Reserved 5% for OS use
487    return int(read_from_meminfo('MemFree') * 0.95)
488
489
490def rounded_memtotal():
491    # Get total of all physical mem, in kbytes
492    usable_kbytes = memtotal()
493    # usable_kbytes is system's usable DRAM in kbytes,
494    #   as reported by memtotal() from device /proc/meminfo memtotal
495    #   after Linux deducts 1.5% to 5.1% for system table overhead
496    # Undo the unknown actual deduction by rounding up
497    #   to next small multiple of a big power-of-two
498    #   eg  12GB - 5.1% gets rounded back up to 12GB
499    mindeduct = 0.015  # 1.5 percent
500    maxdeduct = 0.055  # 5.5 percent
501    # deduction range 1.5% .. 5.5% supports physical mem sizes
502    #    6GB .. 12GB in steps of .5GB
503    #   12GB .. 24GB in steps of 1 GB
504    #   24GB .. 48GB in steps of 2 GB ...
505    # Finer granularity in physical mem sizes would require
506    #   tighter spread between min and max possible deductions
507
508    # increase mem size by at least min deduction, without rounding
509    min_kbytes = int(usable_kbytes / (1.0 - mindeduct))
510    # increase mem size further by 2**n rounding, by 0..roundKb or more
511    round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes
512    # find least binary roundup 2**n that covers worst-cast roundKb
513    mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2)))
514    # have round_kbytes <= mod2n < round_kbytes*2
515    # round min_kbytes up to next multiple of mod2n
516    phys_kbytes = min_kbytes + mod2n - 1
517    phys_kbytes = phys_kbytes - (phys_kbytes % mod2n)  # clear low bits
518    return phys_kbytes
519
520
521def sysctl(key, value=None):
522    """Generic implementation of sysctl, to read and write.
523
524    @param key: A location under /proc/sys
525    @param value: If not None, a value to write into the sysctl.
526
527    @return The single-line sysctl value as a string.
528    """
529    path = '/proc/sys/%s' % key
530    if value is not None:
531        utils.write_one_line(path, str(value))
532    return utils.read_one_line(path)
533
534
535def sysctl_kernel(key, value=None):
536    """(Very) partial implementation of sysctl, for kernel params"""
537    if value is not None:
538        # write
539        utils.write_one_line('/proc/sys/kernel/%s' % key, str(value))
540    else:
541        # read
542        out = utils.read_one_line('/proc/sys/kernel/%s' % key)
543        return int(re.search(r'\d+', out).group(0))
544
545
546def _convert_exit_status(sts):
547    if os.WIFSIGNALED(sts):
548        return -os.WTERMSIG(sts)
549    elif os.WIFEXITED(sts):
550        return os.WEXITSTATUS(sts)
551    else:
552        # impossible?
553        raise RuntimeError("Unknown exit status %d!" % sts)
554
555
556def where_art_thy_filehandles():
557    """Dump the current list of filehandles"""
558    os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid())
559
560
561def print_to_tty(string):
562    """Output string straight to the tty"""
563    open('/dev/tty', 'w').write(string + '\n')
564
565
566def dump_object(object):
567    """Dump an object's attributes and methods
568
569    kind of like dir()
570    """
571    for item in object.__dict__.iteritems():
572        print item
573        try:
574            (key, value) = item
575            dump_object(value)
576        except:
577            continue
578
579
580def environ(env_key):
581    """return the requested environment variable, or '' if unset"""
582    if (os.environ.has_key(env_key)):
583        return os.environ[env_key]
584    else:
585        return ''
586
587
588def prepend_path(newpath, oldpath):
589    """prepend newpath to oldpath"""
590    if (oldpath):
591        return newpath + ':' + oldpath
592    else:
593        return newpath
594
595
596def append_path(oldpath, newpath):
597    """append newpath to oldpath"""
598    if (oldpath):
599        return oldpath + ':' + newpath
600    else:
601        return newpath
602
603
604_TIME_OUTPUT_RE = re.compile(
605        r'([\d\.]*)user ([\d\.]*)system '
606        r'(\d*):([\d\.]*)elapsed (\d*)%CPU')
607
608
609def avgtime_print(dir):
610    """ Calculate some benchmarking statistics.
611        Input is a directory containing a file called 'time'.
612        File contains one-per-line results of /usr/bin/time.
613        Output is average Elapsed, User, and System time in seconds,
614          and average CPU percentage.
615    """
616    user = system = elapsed = cpu = count = 0
617    with open(dir + "/time") as f:
618        for line in f:
619            try:
620                m = _TIME_OUTPUT_RE.match(line);
621                user += float(m.group(1))
622                system += float(m.group(2))
623                elapsed += (float(m.group(3)) * 60) + float(m.group(4))
624                cpu += float(m.group(5))
625                count += 1
626            except:
627                raise ValueError("badly formatted times")
628
629    return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \
630          (elapsed / count, user / count, system / count, cpu / count)
631
632
633def to_seconds(time_string):
634    """Converts a string in M+:SS.SS format to S+.SS"""
635    elts = time_string.split(':')
636    if len(elts) == 1:
637        return time_string
638    return str(int(elts[0]) * 60 + float(elts[1]))
639
640
641_TIME_OUTPUT_RE_2 = re.compile(r'(.*?)user (.*?)system (.*?)elapsed')
642
643
644def extract_all_time_results(results_string):
645    """Extract user, system, and elapsed times into a list of tuples"""
646    results = []
647    for result in _TIME_OUTPUT_RE_2.findall(results_string):
648        results.append(tuple([to_seconds(elt) for elt in result]))
649    return results
650
651
652def running_config():
653    """
654    Return path of config file of the currently running kernel
655    """
656    version = utils.system_output('uname -r')
657    for config in ('/proc/config.gz', \
658                   '/boot/config-%s' % version,
659                   '/lib/modules/%s/build/.config' % version):
660        if os.path.isfile(config):
661            return config
662    return None
663
664
665def check_for_kernel_feature(feature):
666    config = running_config()
667
668    if not config:
669        raise TypeError("Can't find kernel config file")
670
671    if magic.guess_type(config) == 'application/x-gzip':
672        grep = 'zgrep'
673    else:
674        grep = 'grep'
675    grep += ' ^CONFIG_%s= %s' % (feature, config)
676
677    if not utils.system_output(grep, ignore_status=True):
678        raise ValueError("Kernel doesn't have a %s feature" % (feature))
679
680
681def check_glibc_ver(ver):
682    glibc_ver = commands.getoutput('ldd --version').splitlines()[0]
683    glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group()
684    if utils.compare_versions(glibc_ver, ver) == -1:
685        raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." %
686                              (glibc_ver, ver))
687
688def check_kernel_ver(ver):
689    kernel_ver = utils.system_output('uname -r')
690    kv_tmp = re.split(r'[-]', kernel_ver)[0:3]
691    # In compare_versions, if v1 < v2, return value == -1
692    if utils.compare_versions(kv_tmp[0], ver) == -1:
693        raise error.TestError("Kernel too old (%s). Kernel > %s is needed." %
694                              (kernel_ver, ver))
695
696
697def human_format(number):
698    # Convert number to kilo / mega / giga format.
699    if number < 1024:
700        return "%d" % number
701    kilo = float(number) / 1024.0
702    if kilo < 1024:
703        return "%.2fk" % kilo
704    meg = kilo / 1024.0
705    if meg < 1024:
706        return "%.2fM" % meg
707    gig = meg / 1024.0
708    return "%.2fG" % gig
709
710
711def numa_nodes():
712    node_paths = glob.glob('/sys/devices/system/node/node*')
713    nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths]
714    return (sorted(nodes))
715
716
717def node_size():
718    nodes = max(len(numa_nodes()), 1)
719    return ((memtotal() * 1024) / nodes)
720
721
722def pickle_load(filename):
723    return pickle.load(open(filename, 'r'))
724
725
726# Return the kernel version and build timestamp.
727def running_os_release():
728    return os.uname()[2:4]
729
730
731def running_os_ident():
732    (version, timestamp) = running_os_release()
733    return version + '::' + timestamp
734
735
736def running_os_full_version():
737    (version, timestamp) = running_os_release()
738    return version
739
740
741# much like find . -name 'pattern'
742def locate(pattern, root=os.getcwd()):
743    for path, dirs, files in os.walk(root):
744        for f in files:
745            if fnmatch.fnmatch(f, pattern):
746                yield os.path.abspath(os.path.join(path, f))
747
748
749def freespace(path):
750    """Return the disk free space, in bytes"""
751    s = os.statvfs(path)
752    return s.f_bavail * s.f_bsize
753
754
755def disk_block_size(path):
756    """Return the disk block size, in bytes"""
757    return os.statvfs(path).f_bsize
758
759
760_DISK_PARTITION_3_RE = re.compile(r'^(/dev/hd[a-z]+)3', re.M)
761
762def get_disks():
763    df_output = utils.system_output('df')
764    return _DISK_PARTITION_3_RE.findall(df_output)
765
766
767def get_disk_size(disk_name):
768    """
769    Return size of disk in byte. Return 0 in Error Case
770
771    @param disk_name: disk name to find size
772    """
773    device = os.path.basename(disk_name)
774    for line in file('/proc/partitions'):
775        try:
776            _, _, blocks, name = re.split(r' +', line.strip())
777        except ValueError:
778            continue
779        if name == device:
780            return 1024 * int(blocks)
781    return 0
782
783
784def get_disk_size_gb(disk_name):
785    """
786    Return size of disk in GB (10^9). Return 0 in Error Case
787
788    @param disk_name: disk name to find size
789    """
790    return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5)
791
792
793def get_disk_model(disk_name):
794    """
795    Return model name for internal storage device
796
797    @param disk_name: disk name to find model
798    """
799    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
800    cmd2 = 'grep -E "ID_(NAME|MODEL)="'
801    cmd3 = 'cut -f 2 -d"="'
802    cmd = ' | '.join([cmd1, cmd2, cmd3])
803    return utils.system_output(cmd)
804
805
806_DISK_DEV_RE = re.compile(r'/dev/sd[a-z]|/dev/mmcblk[0-9]*|/dev/nvme[0-9]*')
807
808
809def get_disk_from_filename(filename):
810    """
811    Return the disk device the filename is on.
812    If the file is on tmpfs or other special file systems,
813    return None.
814
815    @param filename: name of file, full path.
816    """
817
818    if not os.path.exists(filename):
819        raise error.TestError('file %s missing' % filename)
820
821    if filename[0] != '/':
822        raise error.TestError('This code works only with full path')
823
824    m = _DISK_DEV_RE.match(filename)
825    while not m:
826        if filename[0] != '/':
827            return None
828        if filename == '/dev/root':
829            cmd = 'rootdev -d -s'
830        elif filename.startswith('/dev/mapper'):
831            cmd = 'dmsetup table "%s"' % os.path.basename(filename)
832            dmsetup_output = utils.system_output(cmd).split(' ')
833            if dmsetup_output[2] == 'verity':
834                maj_min = dmsetup_output[4]
835            elif dmsetup_output[2] == 'crypt':
836                maj_min = dmsetup_output[6]
837            cmd = 'realpath "/dev/block/%s"' % maj_min
838        elif filename.startswith('/dev/loop'):
839            cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename
840        else:
841            cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename
842        filename = utils.system_output(cmd)
843        m = _DISK_DEV_RE.match(filename)
844    return m.group(0)
845
846
847def get_disk_firmware_version(disk_name):
848    """
849    Return firmware version for internal storage device. (empty string for eMMC)
850
851    @param disk_name: disk name to find model
852    """
853    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
854    cmd2 = 'grep -E "ID_REVISION="'
855    cmd3 = 'cut -f 2 -d"="'
856    cmd = ' | '.join([cmd1, cmd2, cmd3])
857    return utils.system_output(cmd)
858
859
860def is_disk_scsi(disk_name):
861    """
862    Return true if disk is a scsi device, return false otherwise
863
864    @param disk_name: disk name check
865    """
866    return re.match('/dev/sd[a-z]+', disk_name)
867
868
869def is_disk_harddisk(disk_name):
870    """
871    Return true if disk is a harddisk, return false otherwise
872
873    @param disk_name: disk name check
874    """
875    cmd1 = 'udevadm info --query=property --name=%s' % disk_name
876    cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="'
877    cmd3 = 'cut -f 2 -d"="'
878    cmd = ' | '.join([cmd1, cmd2, cmd3])
879
880    rtt = utils.system_output(cmd)
881
882    # eMMC will not have this field; rtt == ''
883    # SSD will have zero rotation rate; rtt == '0'
884    # For harddisk rtt > 0
885    return rtt and int(rtt) > 0
886
887
888def verify_hdparm_feature(disk_name, feature):
889    """
890    Check for feature support for SCSI disk using hdparm
891
892    @param disk_name: target disk
893    @param feature: hdparm output string of the feature
894    """
895    cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature)
896    ret = utils.system(cmd, ignore_status=True)
897    if ret == 0:
898        return True
899    elif ret == 1:
900        return False
901    else:
902        raise error.TestFail('Error running command %s' % cmd)
903
904
905def get_storage_error_msg(disk_name, reason):
906    """
907    Get Error message for storage test which include disk model.
908    and also include the firmware version for the SCSI disk
909
910    @param disk_name: target disk
911    @param reason: Reason of the error.
912    """
913
914    msg = reason
915
916    model = get_disk_model(disk_name)
917    msg += ' Disk model: %s' % model
918
919    if is_disk_scsi(disk_name):
920        fw = get_disk_firmware_version(disk_name)
921        msg += ' firmware: %s' % fw
922
923    return msg
924
925
926def load_module(module_name, params=None):
927    # Checks if a module has already been loaded
928    if module_is_loaded(module_name):
929        return False
930
931    cmd = '/sbin/modprobe ' + module_name
932    if params:
933        cmd += ' ' + params
934    utils.system(cmd)
935    return True
936
937
938def unload_module(module_name):
939    """
940    Removes a module. Handles dependencies. If even then it's not possible
941    to remove one of the modules, it will trhow an error.CmdError exception.
942
943    @param module_name: Name of the module we want to remove.
944    """
945    l_raw = utils.system_output("/bin/lsmod").splitlines()
946    lsmod = [x for x in l_raw if x.split()[0] == module_name]
947    if len(lsmod) > 0:
948        line_parts = lsmod[0].split()
949        if len(line_parts) == 4:
950            submodules = line_parts[3].split(",")
951            for submodule in submodules:
952                unload_module(submodule)
953        utils.system("/sbin/modprobe -r %s" % module_name)
954        logging.info("Module %s unloaded", module_name)
955    else:
956        logging.info("Module %s is already unloaded", module_name)
957
958
959def module_is_loaded(module_name):
960    module_name = module_name.replace('-', '_')
961    modules = utils.system_output('/bin/lsmod').splitlines()
962    for module in modules:
963        if module.startswith(module_name) and module[len(module_name)] == ' ':
964            return True
965    return False
966
967
968def get_loaded_modules():
969    lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:]
970    return [line.split(None, 1)[0] for line in lsmod_output]
971
972
973def get_huge_page_size():
974    output = utils.system_output('grep Hugepagesize /proc/meminfo')
975    return int(output.split()[1]) # Assumes units always in kB. :(
976
977
978def get_num_huge_pages():
979    raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages')
980    return int(raw_hugepages.split()[2])
981
982
983def set_num_huge_pages(num):
984    utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num)
985
986
987def ping_default_gateway():
988    """Ping the default gateway."""
989
990    network = open('/etc/sysconfig/network')
991    m = re.search('GATEWAY=(\S+)', network.read())
992
993    if m:
994        gw = m.group(1)
995        cmd = 'ping %s -c 5 > /dev/null' % gw
996        return utils.system(cmd, ignore_status=True)
997
998    raise error.TestError('Unable to find default gateway')
999
1000
1001def drop_caches():
1002    """Writes back all dirty pages to disk and clears all the caches."""
1003    utils.system("sync")
1004    # We ignore failures here as this will fail on 2.6.11 kernels.
1005    utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True)
1006
1007
1008def process_is_alive(name_pattern):
1009    """
1010    'pgrep name' misses all python processes and also long process names.
1011    'pgrep -f name' gets all shell commands with name in args.
1012    So look only for command whose initial pathname ends with name.
1013    Name itself is an egrep pattern, so it can use | etc for variations.
1014    """
1015    return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern,
1016                        ignore_status=True) == 0
1017
1018
1019def get_hwclock_seconds(utc=True):
1020    """
1021    Return the hardware clock in seconds as a floating point value.
1022    Use Coordinated Universal Time if utc is True, local time otherwise.
1023    Raise a ValueError if unable to read the hardware clock.
1024    """
1025    cmd = '/sbin/hwclock --debug'
1026    if utc:
1027        cmd += ' --utc'
1028    hwclock_output = utils.system_output(cmd, ignore_status=True)
1029    match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$',
1030                      hwclock_output, re.DOTALL)
1031    if match:
1032        seconds = int(match.group(1)) + float(match.group(2))
1033        logging.debug('hwclock seconds = %f', seconds)
1034        return seconds
1035
1036    raise ValueError('Unable to read the hardware clock -- ' +
1037                     hwclock_output)
1038
1039
1040def set_wake_alarm(alarm_time):
1041    """
1042    Set the hardware RTC-based wake alarm to 'alarm_time'.
1043    """
1044    utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time))
1045
1046
1047def set_power_state(state):
1048    """
1049    Set the system power state to 'state'.
1050    """
1051    utils.write_one_line('/sys/power/state', state)
1052
1053
1054def standby():
1055    """
1056    Power-on suspend (S1)
1057    """
1058    set_power_state('standby')
1059
1060
1061def suspend_to_ram():
1062    """
1063    Suspend the system to RAM (S3)
1064    """
1065    set_power_state('mem')
1066
1067
1068def suspend_to_disk():
1069    """
1070    Suspend the system to disk (S4)
1071    """
1072    set_power_state('disk')
1073