1"""This module gives the mkfs creation options for an existing filesystem.
2
3tune2fs or xfs_growfs is called according to the filesystem. The results,
4filesystem tunables, are parsed and mapped to corresponding mkfs options.
5"""
6import os, re, tempfile
7import common
8from autotest_lib.client.common_lib import error, utils
9
10
11def opt_string2dict(opt_string):
12    """Breaks the mkfs.ext* option string into dictionary."""
13    # Example string: '-j -q -i 8192 -b 4096'. There may be extra whitespaces.
14    opt_dict = {}
15
16    for item in opt_string.split('-'):
17        item = item.strip()
18        if ' ' in item:
19            (opt, value) = item.split(' ', 1)
20            opt_dict['-%s' % opt] = value
21        elif item != '':
22            opt_dict['-%s' % item] = None
23    # Convert all the digit strings to int.
24    for key, value in opt_dict.iteritems():
25        if value and value.isdigit():
26            opt_dict[key] = int(value)
27
28    return opt_dict
29
30
31def parse_mke2fs_conf(fs_type, conf_file='/etc/mke2fs.conf'):
32    """Parses mke2fs config file for default settings."""
33    # Please see /ect/mke2fs.conf for an example.
34    default_opt = {}
35    fs_opt = {}
36    current_fs_type = ''
37    current_section = ''
38    f = open(conf_file, 'r')
39    for line in f:
40        if '[defaults]' == line.strip():
41            current_section = '[defaults]'
42        elif '[fs_types]' == line.strip():
43            current_section = '[fs_types]'
44        elif current_section == '[defaults]':
45            components = line.split('=', 1)
46            if len(components) == 2:
47                default_opt[components[0].strip()] = components[1].strip()
48        elif current_section == '[fs_types]':
49            m = re.search('(\w+) = {', line)
50            if m:
51                current_fs_type = m.group(1)
52            else:
53                components = line.split('=', 1)
54                if len(components) == 2 and current_fs_type == fs_type:
55                    default_opt[components[0].strip()] = components[1].strip()
56    f.close()
57
58    # fs_types options override the defaults options
59    for key, value in fs_opt.iteritems():
60        default_opt[key] = value
61
62    # Convert all the digit strings to int.
63    for key, value in default_opt.iteritems():
64        if value and value.isdigit():
65            default_opt[key] = int(value)
66
67    return default_opt
68
69
70def convert_conf_opt(default_opt):
71    conf_opt_mapping = {'blocksize': '-b',
72                        'inode_ratio': '-i',
73                        'inode_size': '-I'}
74    mkfs_opt = {}
75
76    # Here we simply concatenate the feature string while we really need
77    # to do the better and/or operations.
78    if 'base_features' in default_opt:
79        mkfs_opt['-O'] = default_opt['base_features']
80    if 'default_features' in default_opt:
81        mkfs_opt['-O'] += ',%s' % default_opt['default_features']
82    if 'features' in default_opt:
83        mkfs_opt['-O'] += ',%s' % default_opt['features']
84
85    for key, value in conf_opt_mapping.iteritems():
86        if key in default_opt:
87            mkfs_opt[value] = default_opt[key]
88
89    if '-O' in mkfs_opt:
90        mkfs_opt['-O'] = mkfs_opt['-O'].split(',')
91
92    return mkfs_opt
93
94
95def merge_ext_features(conf_feature, user_feature):
96    user_feature_list = user_feature.split(',')
97
98    merged_feature = []
99    # Removes duplicate entries in conf_list.
100    for item in conf_feature:
101        if item not in merged_feature:
102            merged_feature.append(item)
103
104    # User options override config options.
105    for item in user_feature_list:
106        if item[0] == '^':
107            if item[1:] in merged_feature:
108                merged_feature.remove(item[1:])
109            else:
110                merged_feature.append(item)
111        elif item not in merged_feature:
112            merged_feature.append(item)
113    return merged_feature
114
115
116def ext_tunables(dev):
117    """Call tune2fs -l and parse the result."""
118    cmd = 'tune2fs -l %s' % dev
119    try:
120        out = utils.system_output(cmd)
121    except error.CmdError:
122        tools_dir = os.path.join(os.environ['AUTODIR'], 'tools')
123        cmd = '%s/tune2fs.ext4dev -l %s' % (tools_dir, dev)
124        out = utils.system_output(cmd)
125    # Load option mappings
126    tune2fs_dict = {}
127    for line in out.splitlines():
128        components = line.split(':', 1)
129        if len(components) == 2:
130            value = components[1].strip()
131            option = components[0]
132            if value.isdigit():
133                tune2fs_dict[option] = int(value)
134            else:
135                tune2fs_dict[option] = value
136
137    return tune2fs_dict
138
139
140def ext_mkfs_options(tune2fs_dict, mkfs_option):
141    """Map the tune2fs options to mkfs options."""
142
143    def __inode_count(tune_dict, k):
144        return (tune_dict['Block count']/tune_dict[k] + 1) * (
145            tune_dict['Block size'])
146
147    def __block_count(tune_dict, k):
148        return int(100*tune_dict[k]/tune_dict['Block count'] + 1)
149
150    def __volume_name(tune_dict, k):
151        if tune_dict[k] != '<none>':
152            return tune_dict[k]
153        else:
154            return ''
155
156    # mappings between fs features and mkfs options
157    ext_mapping = {'Blocks per group': '-g',
158                   'Block size': '-b',
159                   'Filesystem features': '-O',
160                   'Filesystem OS type': '-o',
161                   'Filesystem revision #': '-r',
162                   'Filesystem volume name': '-L',
163                   'Flex block group size': '-G',
164                   'Fragment size': '-f',
165                   'Inode count': '-i',
166                   'Inode size': '-I',
167                   'Journal inode': '-j',
168                   'Reserved block count': '-m'}
169
170    conversions = {
171        'Journal inode': lambda d, k: None,
172        'Filesystem volume name': __volume_name,
173        'Reserved block count': __block_count,
174        'Inode count': __inode_count,
175        'Filesystem features': lambda d, k: re.sub(' ', ',', d[k]),
176        'Filesystem revision #': lambda d, k: d[k][0]}
177
178    for key, value in ext_mapping.iteritems():
179        if key not in tune2fs_dict:
180            continue
181        if key in conversions:
182            mkfs_option[value] = conversions[key](tune2fs_dict, key)
183        else:
184            mkfs_option[value] = tune2fs_dict[key]
185
186
187def xfs_tunables(dev):
188    """Call xfs_grow -n to get filesystem tunables."""
189    # Have to mount the filesystem to call xfs_grow.
190    tmp_mount_dir = tempfile.mkdtemp()
191    cmd = 'mount %s %s' % (dev, tmp_mount_dir)
192    utils.system_output(cmd)
193    xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
194    cmd = '%s -n %s' % (xfs_growfs, dev)
195    try:
196        out = utils.system_output(cmd)
197    finally:
198        # Clean.
199        cmd = 'umount %s' % dev
200        utils.system_output(cmd, ignore_status=True)
201        os.rmdir(tmp_mount_dir)
202
203    ## The output format is given in report_info (xfs_growfs.c)
204    ## "meta-data=%-22s isize=%-6u agcount=%u, agsize=%u blks\n"
205    ## "                 =%-22s sectsz=%-5u attr=%u\n"
206    ## "data         =%-22s bsize=%-6u blocks=%llu, imaxpct=%u\n"
207    ## "                 =%-22s sunit=%-6u swidth=%u blks\n"
208    ## "naming     =version %-14u bsize=%-6u\n"
209    ## "log            =%-22s bsize=%-6u blocks=%u, version=%u\n"
210    ## "                 =%-22s sectsz=%-5u sunit=%u blks, lazy-count=%u\n"
211    ## "realtime =%-22s extsz=%-6u blocks=%llu, rtextents=%llu\n"
212
213    tune2fs_dict = {}
214    # Flag for extracting naming version number
215    keep_version = False
216    for line in out.splitlines():
217        m = re.search('^([-\w]+)', line)
218        if m:
219            main_tag = m.group(1)
220        pairs = line.split()
221        for pair in pairs:
222            # naming: version needs special treatment
223            if pair == '=version':
224                # 1 means the next pair is the version number we want
225                keep_version = True
226                continue
227            if keep_version:
228                tune2fs_dict['naming: version'] = pair
229                # Resets the flag since we have logged the version
230                keep_version = False
231                continue
232            # Ignores the strings without '=', such as 'blks'
233            if '=' not in pair:
234                continue
235            key, value = pair.split('=')
236            tagged_key = '%s: %s' % (main_tag, key)
237            if re.match('[0-9]+', value):
238                tune2fs_dict[tagged_key] = int(value.rstrip(','))
239            else:
240                tune2fs_dict[tagged_key] = value.rstrip(',')
241
242    return tune2fs_dict
243
244
245def xfs_mkfs_options(tune2fs_dict, mkfs_option):
246    """Maps filesystem tunables to their corresponding mkfs options."""
247
248    # Mappings
249    xfs_mapping = {'meta-data: isize': '-i size',
250                   'meta-data: agcount': '-d agcount',
251                   'meta-data: sectsz': '-s size',
252                   'meta-data: attr': '-i attr',
253                   'data: bsize': '-b size',
254                   'data: imaxpct': '-i maxpct',
255                   'data: sunit': '-d sunit',
256                   'data: swidth': '-d swidth',
257                   'data: unwritten': '-d unwritten',
258                   'naming: version': '-n version',
259                   'naming: bsize': '-n size',
260                   'log: version': '-l version',
261                   'log: sectsz': '-l sectsize',
262                   'log: sunit': '-l sunit',
263                   'log: lazy-count': '-l lazy-count',
264                   'realtime: extsz': '-r extsize',
265                   'realtime: blocks': '-r size',
266                   'realtime: rtextents': '-r rtdev'}
267
268    mkfs_option['-l size'] = tune2fs_dict['log: bsize'] * (
269        tune2fs_dict['log: blocks'])
270
271    for key, value in xfs_mapping.iteritems():
272        mkfs_option[value] = tune2fs_dict[key]
273
274
275def compare_features(needed_feature, current_feature):
276    """Compare two ext* feature lists."""
277    if len(needed_feature) != len(current_feature):
278        return False
279    for feature in current_feature:
280        if feature not in needed_feature:
281            return False
282    return True
283
284
285def match_ext_options(fs_type, dev, needed_options):
286    """Compare the current ext* filesystem tunables with needed ones."""
287    # mkfs.ext* will load default options from /etc/mke2fs.conf
288    conf_opt = parse_mke2fs_conf(fs_type)
289    # We need to convert the conf options to mkfs options.
290    conf_mkfs_opt = convert_conf_opt(conf_opt)
291    # Breaks user mkfs option string to dictionary.
292    needed_opt_dict = opt_string2dict(needed_options)
293    # Removes ignored options.
294    ignored_option = ['-c', '-q', '-E', '-F']
295    for opt in ignored_option:
296        if opt in needed_opt_dict:
297            del needed_opt_dict[opt]
298
299   # User options override config options.
300    needed_opt = conf_mkfs_opt
301    for key, value in needed_opt_dict.iteritems():
302        if key == '-N' or key == '-T':
303            raise Exception('-N/T is not allowed.')
304        elif key == '-O':
305            needed_opt[key] = merge_ext_features(needed_opt[key], value)
306        else:
307            needed_opt[key] = value
308
309    # '-j' option will add 'has_journal' feature.
310    if '-j' in needed_opt and 'has_journal' not in needed_opt['-O']:
311        needed_opt['-O'].append('has_journal')
312    # 'extents' will be shown as 'extent' in the outcome of tune2fs
313    if 'extents' in needed_opt['-O']:
314        needed_opt['-O'].append('extent')
315        needed_opt['-O'].remove('extents')
316    # large_file is a byproduct of resize_inode.
317    if 'large_file' not in needed_opt['-O'] and (
318        'resize_inode' in needed_opt['-O']):
319        needed_opt['-O'].append('large_file')
320
321    current_opt = {}
322    tune2fs_dict = ext_tunables(dev)
323    ext_mkfs_options(tune2fs_dict, current_opt)
324
325    # Does the match
326    for key, value in needed_opt.iteritems():
327        if key == '-O':
328            if not compare_features(value, current_opt[key].split(',')):
329                return False
330        elif key not in current_opt or value != current_opt[key]:
331            return False
332    return True
333
334
335def match_xfs_options(dev, needed_options):
336    """Compare the current ext* filesystem tunables with needed ones."""
337    tmp_mount_dir = tempfile.mkdtemp()
338    cmd = 'mount %s %s' % (dev, tmp_mount_dir)
339    utils.system_output(cmd)
340    xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
341    cmd = '%s -n %s' % (xfs_growfs, dev)
342    try:
343        current_option = utils.system_output(cmd)
344    finally:
345        # Clean.
346        cmd = 'umount %s' % dev
347        utils.system_output(cmd, ignore_status=True)
348        os.rmdir(tmp_mount_dir)
349
350    # '-N' has the same effect as '-n' in mkfs.ext*. Man mkfs.xfs for details.
351    cmd = 'mkfs.xfs %s -N -f %s' % (needed_options, dev)
352    needed_out = utils.system_output(cmd)
353    # 'mkfs.xfs -N' produces slightly different result than 'xfs_growfs -n'
354    needed_out = re.sub('internal log', 'internal    ', needed_out)
355    if current_option == needed_out:
356        return True
357    else:
358        return False
359
360
361def match_mkfs_option(fs_type, dev, needed_options):
362    """Compare the current filesystem tunables with needed ones."""
363    if fs_type.startswith('ext'):
364        ret = match_ext_options(fs_type, dev, needed_options)
365    elif fs_type == 'xfs':
366        ret = match_xfs_options(dev, needed_options)
367    else:
368        ret = False
369
370    return ret
371