1# Lint as: python2, python3
2"""This module gives the mkfs creation options for an existing filesystem.
3
4tune2fs or xfs_growfs is called according to the filesystem. The results,
5filesystem tunables, are parsed and mapped to corresponding mkfs options.
6"""
7from __future__ import absolute_import
8from __future__ import division
9from __future__ import print_function
10
11import os, re, tempfile
12
13import six
14
15import common
16from autotest_lib.client.common_lib import error, utils
17
18
19def opt_string2dict(opt_string):
20    """Breaks the mkfs.ext* option string into dictionary."""
21    # Example string: '-j -q -i 8192 -b 4096'. There may be extra whitespaces.
22    opt_dict = {}
23
24    for item in opt_string.split('-'):
25        item = item.strip()
26        if ' ' in item:
27            (opt, value) = item.split(' ', 1)
28            opt_dict['-%s' % opt] = value
29        elif item != '':
30            opt_dict['-%s' % item] = None
31    # Convert all the digit strings to int.
32    for key, value in six.iteritems(opt_dict):
33        if value and value.isdigit():
34            opt_dict[key] = int(value)
35
36    return opt_dict
37
38
39def parse_mke2fs_conf(fs_type, conf_file='/etc/mke2fs.conf'):
40    """Parses mke2fs config file for default settings."""
41    # Please see /ect/mke2fs.conf for an example.
42    default_opt = {}
43    fs_opt = {}
44    current_fs_type = ''
45    current_section = ''
46    f = open(conf_file, 'r')
47    for line in f:
48        if '[defaults]' == line.strip():
49            current_section = '[defaults]'
50        elif '[fs_types]' == line.strip():
51            current_section = '[fs_types]'
52        elif current_section == '[defaults]':
53            components = line.split('=', 1)
54            if len(components) == 2:
55                default_opt[components[0].strip()] = components[1].strip()
56        elif current_section == '[fs_types]':
57            m = re.search('(\w+) = {', line)
58            if m:
59                current_fs_type = m.group(1)
60            else:
61                components = line.split('=', 1)
62                if len(components) == 2 and current_fs_type == fs_type:
63                    default_opt[components[0].strip()] = components[1].strip()
64    f.close()
65
66    # fs_types options override the defaults options
67    for key, value in six.iteritems(fs_opt):
68        default_opt[key] = value
69
70    # Convert all the digit strings to int.
71    for key, value in six.iteritems(default_opt):
72        if value and value.isdigit():
73            default_opt[key] = int(value)
74
75    return default_opt
76
77
78def convert_conf_opt(default_opt):
79    conf_opt_mapping = {'blocksize': '-b',
80                        'inode_ratio': '-i',
81                        'inode_size': '-I'}
82    mkfs_opt = {}
83
84    # Here we simply concatenate the feature string while we really need
85    # to do the better and/or operations.
86    if 'base_features' in default_opt:
87        mkfs_opt['-O'] = default_opt['base_features']
88    if 'default_features' in default_opt:
89        mkfs_opt['-O'] += ',%s' % default_opt['default_features']
90    if 'features' in default_opt:
91        mkfs_opt['-O'] += ',%s' % default_opt['features']
92
93    for key, value in six.iteritems(conf_opt_mapping):
94        if key in default_opt:
95            mkfs_opt[value] = default_opt[key]
96
97    if '-O' in mkfs_opt:
98        mkfs_opt['-O'] = mkfs_opt['-O'].split(',')
99
100    return mkfs_opt
101
102
103def merge_ext_features(conf_feature, user_feature):
104    user_feature_list = user_feature.split(',')
105
106    merged_feature = []
107    # Removes duplicate entries in conf_list.
108    for item in conf_feature:
109        if item not in merged_feature:
110            merged_feature.append(item)
111
112    # User options override config options.
113    for item in user_feature_list:
114        if item[0] == '^':
115            if item[1:] in merged_feature:
116                merged_feature.remove(item[1:])
117            else:
118                merged_feature.append(item)
119        elif item not in merged_feature:
120            merged_feature.append(item)
121    return merged_feature
122
123
124def ext_tunables(dev):
125    """Call tune2fs -l and parse the result."""
126    cmd = 'tune2fs -l %s' % dev
127    try:
128        out = utils.system_output(cmd)
129    except error.CmdError:
130        tools_dir = os.path.join(os.environ['AUTODIR'], 'tools')
131        cmd = '%s/tune2fs.ext4dev -l %s' % (tools_dir, dev)
132        out = utils.system_output(cmd)
133    # Load option mappings
134    tune2fs_dict = {}
135    for line in out.splitlines():
136        components = line.split(':', 1)
137        if len(components) == 2:
138            value = components[1].strip()
139            option = components[0]
140            if value.isdigit():
141                tune2fs_dict[option] = int(value)
142            else:
143                tune2fs_dict[option] = value
144
145    return tune2fs_dict
146
147
148def ext_mkfs_options(tune2fs_dict, mkfs_option):
149    """Map the tune2fs options to mkfs options."""
150
151    def __inode_count(tune_dict, k):
152        return (tune_dict['Block count']/tune_dict[k] + 1) * (
153            tune_dict['Block size'])
154
155    def __block_count(tune_dict, k):
156        return int(100*tune_dict[k]/tune_dict['Block count'] + 1)
157
158    def __volume_name(tune_dict, k):
159        if tune_dict[k] != '<none>':
160            return tune_dict[k]
161        else:
162            return ''
163
164    # mappings between fs features and mkfs options
165    ext_mapping = {'Blocks per group': '-g',
166                   'Block size': '-b',
167                   'Filesystem features': '-O',
168                   'Filesystem OS type': '-o',
169                   'Filesystem revision #': '-r',
170                   'Filesystem volume name': '-L',
171                   'Flex block group size': '-G',
172                   'Fragment size': '-f',
173                   'Inode count': '-i',
174                   'Inode size': '-I',
175                   'Journal inode': '-j',
176                   'Reserved block count': '-m'}
177
178    conversions = {
179        'Journal inode': lambda d, k: None,
180        'Filesystem volume name': __volume_name,
181        'Reserved block count': __block_count,
182        'Inode count': __inode_count,
183        'Filesystem features': lambda d, k: re.sub(' ', ',', d[k]),
184        'Filesystem revision #': lambda d, k: d[k][0]}
185
186    for key, value in six.iteritems(ext_mapping):
187        if key not in tune2fs_dict:
188            continue
189        if key in conversions:
190            mkfs_option[value] = conversions[key](tune2fs_dict, key)
191        else:
192            mkfs_option[value] = tune2fs_dict[key]
193
194
195def xfs_tunables(dev):
196    """Call xfs_grow -n to get filesystem tunables."""
197    # Have to mount the filesystem to call xfs_grow.
198    tmp_mount_dir = tempfile.mkdtemp()
199    cmd = 'mount %s %s' % (dev, tmp_mount_dir)
200    utils.system_output(cmd)
201    xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
202    cmd = '%s -n %s' % (xfs_growfs, dev)
203    try:
204        out = utils.system_output(cmd)
205    finally:
206        # Clean.
207        cmd = 'umount %s' % dev
208        utils.system_output(cmd, ignore_status=True)
209        os.rmdir(tmp_mount_dir)
210
211    ## The output format is given in report_info (xfs_growfs.c)
212    ## "meta-data=%-22s isize=%-6u agcount=%u, agsize=%u blks\n"
213    ## "                 =%-22s sectsz=%-5u attr=%u\n"
214    ## "data         =%-22s bsize=%-6u blocks=%llu, imaxpct=%u\n"
215    ## "                 =%-22s sunit=%-6u swidth=%u blks\n"
216    ## "naming     =version %-14u bsize=%-6u\n"
217    ## "log            =%-22s bsize=%-6u blocks=%u, version=%u\n"
218    ## "                 =%-22s sectsz=%-5u sunit=%u blks, lazy-count=%u\n"
219    ## "realtime =%-22s extsz=%-6u blocks=%llu, rtextents=%llu\n"
220
221    tune2fs_dict = {}
222    # Flag for extracting naming version number
223    keep_version = False
224    for line in out.splitlines():
225        m = re.search('^([-\w]+)', line)
226        if m:
227            main_tag = m.group(1)
228        pairs = line.split()
229        for pair in pairs:
230            # naming: version needs special treatment
231            if pair == '=version':
232                # 1 means the next pair is the version number we want
233                keep_version = True
234                continue
235            if keep_version:
236                tune2fs_dict['naming: version'] = pair
237                # Resets the flag since we have logged the version
238                keep_version = False
239                continue
240            # Ignores the strings without '=', such as 'blks'
241            if '=' not in pair:
242                continue
243            key, value = pair.split('=')
244            tagged_key = '%s: %s' % (main_tag, key)
245            if re.match('[0-9]+', value):
246                tune2fs_dict[tagged_key] = int(value.rstrip(','))
247            else:
248                tune2fs_dict[tagged_key] = value.rstrip(',')
249
250    return tune2fs_dict
251
252
253def xfs_mkfs_options(tune2fs_dict, mkfs_option):
254    """Maps filesystem tunables to their corresponding mkfs options."""
255
256    # Mappings
257    xfs_mapping = {'meta-data: isize': '-i size',
258                   'meta-data: agcount': '-d agcount',
259                   'meta-data: sectsz': '-s size',
260                   'meta-data: attr': '-i attr',
261                   'data: bsize': '-b size',
262                   'data: imaxpct': '-i maxpct',
263                   'data: sunit': '-d sunit',
264                   'data: swidth': '-d swidth',
265                   'data: unwritten': '-d unwritten',
266                   'naming: version': '-n version',
267                   'naming: bsize': '-n size',
268                   'log: version': '-l version',
269                   'log: sectsz': '-l sectsize',
270                   'log: sunit': '-l sunit',
271                   'log: lazy-count': '-l lazy-count',
272                   'realtime: extsz': '-r extsize',
273                   'realtime: blocks': '-r size',
274                   'realtime: rtextents': '-r rtdev'}
275
276    mkfs_option['-l size'] = tune2fs_dict['log: bsize'] * (
277        tune2fs_dict['log: blocks'])
278
279    for key, value in six.iteritems(xfs_mapping):
280        mkfs_option[value] = tune2fs_dict[key]
281
282
283def compare_features(needed_feature, current_feature):
284    """Compare two ext* feature lists."""
285    if len(needed_feature) != len(current_feature):
286        return False
287    for feature in current_feature:
288        if feature not in needed_feature:
289            return False
290    return True
291
292
293def match_ext_options(fs_type, dev, needed_options):
294    """Compare the current ext* filesystem tunables with needed ones."""
295    # mkfs.ext* will load default options from /etc/mke2fs.conf
296    conf_opt = parse_mke2fs_conf(fs_type)
297    # We need to convert the conf options to mkfs options.
298    conf_mkfs_opt = convert_conf_opt(conf_opt)
299    # Breaks user mkfs option string to dictionary.
300    needed_opt_dict = opt_string2dict(needed_options)
301    # Removes ignored options.
302    ignored_option = ['-c', '-q', '-E', '-F']
303    for opt in ignored_option:
304        if opt in needed_opt_dict:
305            del needed_opt_dict[opt]
306
307   # User options override config options.
308    needed_opt = conf_mkfs_opt
309    for key, value in six.iteritems(needed_opt_dict):
310        if key == '-N' or key == '-T':
311            raise Exception('-N/T is not allowed.')
312        elif key == '-O':
313            needed_opt[key] = merge_ext_features(needed_opt[key], value)
314        else:
315            needed_opt[key] = value
316
317    # '-j' option will add 'has_journal' feature.
318    if '-j' in needed_opt and 'has_journal' not in needed_opt['-O']:
319        needed_opt['-O'].append('has_journal')
320    # 'extents' will be shown as 'extent' in the outcome of tune2fs
321    if 'extents' in needed_opt['-O']:
322        needed_opt['-O'].append('extent')
323        needed_opt['-O'].remove('extents')
324    # large_file is a byproduct of resize_inode.
325    if 'large_file' not in needed_opt['-O'] and (
326        'resize_inode' in needed_opt['-O']):
327        needed_opt['-O'].append('large_file')
328
329    current_opt = {}
330    tune2fs_dict = ext_tunables(dev)
331    ext_mkfs_options(tune2fs_dict, current_opt)
332
333    # Does the match
334    for key, value in six.iteritems(needed_opt):
335        if key == '-O':
336            if not compare_features(value, current_opt[key].split(',')):
337                return False
338        elif key not in current_opt or value != current_opt[key]:
339            return False
340    return True
341
342
343def match_xfs_options(dev, needed_options):
344    """Compare the current ext* filesystem tunables with needed ones."""
345    tmp_mount_dir = tempfile.mkdtemp()
346    cmd = 'mount %s %s' % (dev, tmp_mount_dir)
347    utils.system_output(cmd)
348    xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
349    cmd = '%s -n %s' % (xfs_growfs, dev)
350    try:
351        current_option = utils.system_output(cmd)
352    finally:
353        # Clean.
354        cmd = 'umount %s' % dev
355        utils.system_output(cmd, ignore_status=True)
356        os.rmdir(tmp_mount_dir)
357
358    # '-N' has the same effect as '-n' in mkfs.ext*. Man mkfs.xfs for details.
359    cmd = 'mkfs.xfs %s -N -f %s' % (needed_options, dev)
360    needed_out = utils.system_output(cmd)
361    # 'mkfs.xfs -N' produces slightly different result than 'xfs_growfs -n'
362    needed_out = re.sub('internal log', 'internal    ', needed_out)
363    if current_option == needed_out:
364        return True
365    else:
366        return False
367
368
369def match_mkfs_option(fs_type, dev, needed_options):
370    """Compare the current filesystem tunables with needed ones."""
371    if fs_type.startswith('ext'):
372        ret = match_ext_options(fs_type, dev, needed_options)
373    elif fs_type == 'xfs':
374        ret = match_xfs_options(dev, needed_options)
375    else:
376        ret = False
377
378    return ret
379