Skip to content

Commit

Permalink
feat: New spec "/etc/lvm/devices/system.devices" and parser (#3457)
Browse files Browse the repository at this point in the history
* feat: New spec "/etc/lvm/devices/system.devices" and parser

* Add the parser in lvm.py
* Update doc examples to make it pass

Signed-off-by: Huanhuan Li <[email protected]>
  • Loading branch information
huali027 authored Jul 7, 2022
1 parent aa0c992 commit 18a9721
Show file tree
Hide file tree
Showing 4 changed files with 197 additions and 22 deletions.
83 changes: 61 additions & 22 deletions insights/parsers/lvm.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
Parsers for lvm data based on output of various commands and file contents.
This module contains the classes that parse the output of the commands `lvs`,
`pvs`, and `vgs`, and the contents of the file `/etc/lvm/lvm.conf`.
`pvs`, and `vgs`, and the content of the files `/etc/lvm/lvm.conf`,
`/etc/lvm/devices/system.devices`.
Pvs - command ``/sbin/pvs --nameprefixes --noheadings --separator='|' -a -o pv_all``
------------------------------------------------------------------------------------
Expand All @@ -28,13 +29,16 @@
LvmConf - file ``/etc/lvm/lvm.conf``
------------------------------------
LvmSystemDevices - file ``/etc/lvm/devices/system.devices``
-----------------------------------------------------------
"""
from __future__ import print_function

import json
from collections import defaultdict

from insights.parsers import ParseException
from insights.parsers import ParseException, optlist_to_dict, SkipException
from insights.specs import Specs

from .. import (
Expand Down Expand Up @@ -291,13 +295,10 @@ class PvsHeadings(LvmHeadings):
warning strings.
Examples:
>>> pvs_data = shared[PvsHeadings]
>>> pvs_data[0]
{'PV': '/dev/fedora/home', 'VG': '', 'Fmt': '', 'Attr': '---', 'PSize': '0',
'PFree': '0', 'DevSize': '418.75g', 'PV_UUID': '', 'PMdaFree': '0',
'PMdaSize': '0', '#PMda': '0', '#PMdaUse': '0', 'PE': '0', 'PV_KEY': '/dev/fedora/home+no_uuid'}
>>> pvs_data[0]['PV']
'/dev/fedora/home'
>>> pvs_data[0]['PMdaSize']
'0'
"""

Expand Down Expand Up @@ -428,11 +429,10 @@ class VgsHeadings(LvmHeadings):
warning strings.
Examples:
>>> vgs_info = shared[VgsHeadings]
>>> vgs_info.data[0]
{}
>>> vgs_info.data[2]['LSize']
'2.00g'
>>> vgs_info.data[0]['VG']
'DATA_OTM_VG'
>>> vgs_info.data[0]['VG_UUID']
'xK6HXk-xl2O-cqW5-2izb-LI9M-4fV0-dAzfcc'
"""

PRIMARY_KEY = Vgs.PRIMARY_KEY
Expand Down Expand Up @@ -621,12 +621,9 @@ class LvsHeadings(LvmHeadings):
warning strings.
Examples:
>>> lvs_info = shared[LvsHeadings]
>>> lvs_info.data[0]
{'LV': 'lv_app', 'VG': 'vg_root', 'Attr': '-wi-ao----', 'LSize': '71.63',
'Pool': '', 'Origin': '', 'Data%': '', 'Meta%': '', 'Move': '', 'Log': '',
'Cpy%Sync': '', 'Convert': '', 'LV_Tags': '', 'Devices': '/dev/sda2(7136)'}
>>> lvs_info.data[2]['LSize']
>>> lvs_info.data[0]['Devices']
'/dev/sda2(7136)'
>>> lvs_info.data[1]['LSize']
'2.00g'
"""

Expand Down Expand Up @@ -674,10 +671,8 @@ class LvmConf(LegacyItemAccess, Parser):
}
Examples:
>>> lvm_conf_data = shared[LvmConf]
>>> lvm_conf_data.data
{"locking_type": 1, "volume_list": ["vg1", "vg2/lvol1", "@tag1", "@*"],
"filter": ["a/sda[0-9]*$/", "r/sd.*/"], "history_size": 100}
>>> 'vg2' in lvm_conf_data.data.get('volume_list')
True
>>> lvm_conf_data.get("locking_type")
1
"""
Expand Down Expand Up @@ -729,6 +724,50 @@ def parse_content(self, content):
self.data = json.loads(_lvm_render(dict(dd)))


@parser(Specs.lvm_system_devices)
class LvmSystemDevices(Parser, dict):
"""
Parse the content of the ``/etc/lvm/devices/system.devices`` file.
It returns a dict. The key is the device id, the value is a dict of
other info.
Sample input::
VERSION=1.1.2
IDTYPE=devname IDNAME=/dev/vda2 DEVNAME=/dev/vda2 PVID=phl0clFbAokp9UXqbIgI5YYQxuTIJVkD PART=2
Sample output::
{
'/dev/vda2': {
'IDTYPE': 'devname',
'DEVNAME': '/dev/vda2',
'PVID': 'phl0clFbAokp9UXqbIgI5YYQxuTIJVkD',
'PART': '2'
}
}
Example:
>>> type(devices)
<class 'insights.parsers.lvm.LvmSystemDevices'>
>>> devices['/dev/vda2']['IDTYPE']
'devname'
>>> devices['/dev/vda2']['PVID']
'phl0clFbAokp9UXqbIgI5YYQxuTIJVkD'
Raises:
SkipException: when there is no device info.
"""

def parse_content(self, content):
for line in content:
if 'IDNAME' in line:
dict_info = optlist_to_dict(line, opt_sep=None)
self[dict_info.pop('IDNAME')] = dict_info
if not self:
raise SkipException("No valid content.")


if __name__ == "__main__":
# This is a quick script to generate the key mappings in each subclass.
# Run each lvm command with --separator="|", --nameprefixes and *not* --noheadings
Expand Down
1 change: 1 addition & 0 deletions insights/specs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,7 @@ class Specs(SpecSet):
lsvmbus = RegistryPoint()
lvdisplay = RegistryPoint()
lvm_conf = RegistryPoint(filterable=True)
lvm_system_devices = RegistryPoint()
lvmconfig = RegistryPoint()
lvs_noheadings = RegistryPoint()
lvs_noheadings_all = RegistryPoint()
Expand Down
1 change: 1 addition & 0 deletions insights/specs/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,7 @@ class DefaultSpecs(Specs):
lsscsi = simple_command("/usr/bin/lsscsi")
lsvmbus = simple_command("/usr/sbin/lsvmbus -vv")
lvm_conf = simple_file("/etc/lvm/lvm.conf")
lvm_system_devices = simple_file("/etc/lvm/devices/system.devices")
lvs_noheadings = simple_command("/sbin/lvs --nameprefixes --noheadings --separator='|' -a -o lv_name,lv_size,lv_attr,mirror_log,vg_name,devices,region_size,data_percent,metadata_percent,segtype,seg_monitor,lv_kernel_major,lv_kernel_minor --config=\"global{locking_type=0}\"")
mac_addresses = glob_file("/sys/class/net/*/address")
machine_id = first_file(["etc/insights-client/machine-id", "etc/redhat-access-insights/machine-id", "etc/redhat_access_proactive/machine-id"])
Expand Down
134 changes: 134 additions & 0 deletions insights/tests/parsers/test_lvm.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
from __future__ import print_function
import pytest
import doctest

from insights.parsers import SkipException
from insights.parsers import lvm
from insights.tests import context_wrap
from .lvm_test_data import LVMCONFIG
Expand Down Expand Up @@ -44,13 +48,117 @@
Configuration setting "activation/thin_check_options" unknown.
""".strip()

VGSHEADING_CONTENT_DOC = """
WARNING: Locking disabled. Be careful! This could corrupt your metadata.
Using volume group(s) on command line.
VG Attr Ext #PV #LV #SN VSize VFree VG UUID VProfile #VMda VMdaFree VMdaSize #VMdaUse VG Tags
DATA_OTM_VG wz--n- 4.00m 6 1 0 2.05t 1020.00m xK6HXk-xl2O-cqW5-2izb-LI9M-4fV0-dAzfcc 6 507.00k 1020.00k 6
ITM_VG wz--n- 4.00m 1 1 0 16.00g 4.00m nws5dd-INe6-1db6-9U1N-F0G3-S1z2-5XTdO4 1 508.00k 1020.00k 1
ORABIN_OTM_VG wz--n- 4.00m 2 3 0 190.00g 0 hfJwg8-hset-YgUY-X6NJ-gkWE-EunZ-KuCXGP 2 507.50k 1020.00k 2
REDO_OTM_VG wz--n- 4.00m 1 3 0 50.00g 0 Q2YtGy-CWKU-sEYj-mqHk-rbdP-Hzup-wi8jsf 1 507.50k 1020.00k 1
SWAP_OTM_VG wz--n- 4.00m 1 1 0 24.00g 8.00g hAerzZ-U8QU-ICkc-xxCj-N2Ny-rWzq-pmTpWJ 1 508.00k 1020.00k 1
rootvg wz--n- 4.00m 1 6 0 19.51g 1.95g p4tLLb-ikeo-Ankk-2xJ6-iHYf-D4E6-KFCFvr 1 506.50k 1020.00k 1
Reloading config files
Wiping internal VG cache
""".strip()

LVS_HAEADING_OUTUPT = """
WARNING: Locking disabled. Be careful! This could corrupt your metadata.
LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert LV Tags Devices
lv_app vg_root -wi-ao---- 71.63g /dev/sda2(7136)
lv_home vg_root -wi-ao---- 2.00g /dev/sda2(2272)
lv_opt vg_root -wi-ao---- 5.00g /dev/sda2(2784)
lv_root vg_root -wi-ao---- 5.00g /dev/sda2(0)
lv_tmp vg_root -wi-ao---- 1.00g /dev/sda2(4064)
lv_usr vg_root -wi-ao---- 5.00g /dev/sda2(4320)
lv_usrlocal vg_root -wi-ao---- 1.00g /dev/sda2(5600)
lv_var vg_root -wi-ao---- 5.00g /dev/sda2(5856)
swap vg_root -wi-ao---- 3.88g /dev/sda2(1280)
""".strip()

CONTENT_WITH_EXTRA_LOCK_TIPS = """
WARNING: locking_type (0) is deprecated, using --nolocking.
WARNING: File locking is disabled.
Reading VG shared_vg1 without a lock.
LVM2_VG_FMT='lvm2'|LVM2_VG_UUID='V1AMDw-e9x6-AdCq-p3Hr-6cYn-otra-ducLAc'|LVM2_VG_NAME='rhel'|LVM2_VG_ATTR='wz--n-'|LVM2_VG_PERMISSIONS='writeable'|LVM2_VG_EXTENDABLE='extendable'|LVM2_VG_EXPORTED=''|LVM2_VG_AUTOACTIVATION='enabled'|LVM2_VG_PARTIAL=''|LVM2_VG_ALLOCATION_POLICY='normal'|LVM2_VG_CLUSTERED=''|LVM2_VG_SHARED=''|LVM2_VG_SIZE='<29.00g'|LVM2_VG_FREE='0 '|LVM2_VG_SYSID=''|LVM2_VG_SYSTEMID=''|LVM2_VG_LOCK_TYPE=''|LVM2_VG_LOCK_ARGS=''|LVM2_VG_EXTENT_SIZE='4.00m'|LVM2_VG_EXTENT_COUNT='7423'|LVM2_VG_FREE_COUNT='0'|LVM2_MAX_LV='0'|LVM2_MAX_PV='0'|LVM2_PV_COUNT='1'|LVM2_VG_MISSING_PV_COUNT='0'|LVM2_LV_COUNT='2'|LVM2_SNAP_COUNT='0'|LVM2_VG_SEQNO='3'|LVM2_VG_TAGS=''|LVM2_VG_PROFILE=''|LVM2_VG_MDA_COUNT='1'|LVM2_VG_MDA_USED_COUNT='1'|LVM2_VG_MDA_FREE='507.50k'|LVM2_VG_MDA_SIZE='1020.00k'|LVM2_VG_MDA_COPIES='unmanaged'
"""

LVM_CONF = """
locking_type = 1
#locking_type = 2
# volume_list = [ "vg1", "vg2/lvol1", "@tag1", "@*" ]
volume_list = [ "vg2", "vg3/lvol3", "@tag2", "@*" ]
# filter = [ "a|loop|", "r|/dev/hdc|", "a|/dev/ide|", "r|.*|" ]
filter = [ "r/sda[0-9]*$/", "a/sd.*/" ]
filter = [ "a/sda[0-9]*$/", "r/sd.*/" ]
shell {
history_size = 100
}
""".strip()

PVS_HEADINGS_OUTPUT = """
WARNING: Locking disabled. Be careful! This could corrupt your metadata.
Scanning all devices to update lvmetad.
No PV label found on /dev/loop0.
No PV label found on /dev/loop1.
No PV label found on /dev/sda1.
No PV label found on /dev/fedora/root.
No PV label found on /dev/sda2.
No PV label found on /dev/fedora/swap.
No PV label found on /dev/fedora/home.
No PV label found on /dev/mapper/docker-253:1-2361272-pool.
Wiping internal VG cache
Wiping cache of LVM-capable devices
PV VG Fmt Attr PSize PFree DevSize PV UUID PMdaFree PMdaSize #PMda #PMdaUse PE
/dev/fedora/home --- 0 0 418.75g 0 0 0 0 0
/dev/fedora/root --- 0 0 50.00g 0 0 0 0 0
/dev/fedora/swap --- 0 0 7.69g 0 0 0 0 0
/dev/loop0 --- 0 0 100.00g 0 0 0 0 0
/dev/loop1 --- 0 0 2.00g 0 0 0 0 0
/dev/mapper/docker-253:1-2361272-pool --- 0 0 100.00g 0 0 0 0 0
/dev/mapper/luks-7430952e-7101-4716-9b46-786ce4684f8d fedora lvm2 a-- 476.45g 4.00m 476.45g FPLCRf-d918-LVL7-6e3d-n3ED-aiZv-EesuzY 0 1020.00k 1 1 121970
/dev/sda1 --- 0 0 500.00m 0 0 0 0 0
/dev/sda2 --- 0 0 476.45g 0 0 0 0 0
Reloading config files
Wiping internal VG cache
""".strip()

VGS_OUTPUT = """
WARNING: Locking disabled. Be careful! This could corrupt your metadata.
Using volume group(s) on command line.
VG Attr Ext #PV #LV #SN VSize VFree VG UUID VProfile #VMda VMdaFree VMdaSize #VMdaUse VG Tags
DATA_OTM_VG wz--n- 4.00m 6 1 0 2.05t 1020.00m xK6HXk-xl2O-cqW5-2izb-LI9M-4fV0-dAzfcc 6 507.00k 1020.00k 6
ITM_VG wz--n- 4.00m 1 1 0 16.00g 4.00m nws5dd-INe6-1db6-9U1N-F0G3-S1z2-5XTdO4 1 508.00k 1020.00k 1
ORABIN_OTM_VG wz--n- 4.00m 2 3 0 190.00g 0 hfJwg8-hset-YgUY-X6NJ-gkWE-EunZ-KuCXGP 2 507.50k 1020.00k 2
REDO_OTM_VG wz--n- 4.00m 1 3 0 50.00g 0 Q2YtGy-CWKU-sEYj-mqHk-rbdP-Hzup-wi8jsf 1 507.50k 1020.00k 1
SWAP_OTM_VG wz--n- 4.00m 1 1 0 24.00g 8.00g hAerzZ-U8QU-ICkc-xxCj-N2Ny-rWzq-pmTpWJ 1 508.00k 1020.00k 1
rootvg wz--n- 4.00m 1 6 0 19.51g 1.95g p4tLLb-ikeo-Ankk-2xJ6-iHYf-D4E6-KFCFvr 1 506.50k 1020.00k 1
Reloading config files
Wiping internal VG cache
""".strip()

SYSTEM_DEVICES1 = """
# LVM uses devices listed in this file.
# Created by LVM command lvmdevices pid 2631 at Fri May 27 07:37:11 2022
VERSION=1.1.2
IDTYPE=devname IDNAME=/dev/vda2 DEVNAME=/dev/vda2 PVID=phl0clFbAokp9UXqbIgI5YYQxuTIJVkD PART=2
""".strip()

SYSTEM_DEVICES2 = """
# LVM uses devices listed in this file.
# Created by LVM command lvmdevices pid 2631 at Fri May 27 07:37:11 2022
VERSION=1.1.2
IDTYPE=sys_wwid IDNAME=/dev/vda2 DEVNAME=/dev/vda2 PVID=phl0clFbAokp9UXqbIgI5YYQxuTIJVkD PART=2
IDTYPE=sys_serial IDNAME=/dev/vda1 DEVNAME=/dev/vda1 PVID=phl0clFbAokp9UXqbIgI5YYQdeTIJVkD PART=1
""".strip()

SYSTEM_DEVICES3 = """
# LVM uses devices listed in this file.
# Created by LVM command lvmdevices pid 2631 at Fri May 27 07:37:11 2022
VERSION=1.1.2
""".strip()


def test_find_warnings():
data = [l for l in lvm.find_warnings(WARNINGS_CONTENT.splitlines())]
Expand Down Expand Up @@ -94,3 +202,29 @@ def test_vgs_with_extra_tips():
assert len(result.data.get('warnings')) == 3
assert 'Reading VG shared_vg1 without a lock.' in result.data.get('warnings')
assert len(result.data.get('content')) == 1


def test_system_devices():
devices = lvm.LvmSystemDevices(context_wrap(SYSTEM_DEVICES2))
assert len(devices) == 2
assert '/dev/vda1' in devices
assert devices['/dev/vda1']['IDTYPE'] == 'sys_serial'
assert '/dev/vda2' in devices
assert devices['/dev/vda2']['IDTYPE'] == 'sys_wwid'


def test_system_devices_exception():
with pytest.raises(SkipException):
lvm.LvmSystemDevices(context_wrap(SYSTEM_DEVICES3))


def test_docs():
env = {
'devices': lvm.LvmSystemDevices(context_wrap(SYSTEM_DEVICES1)),
'lvm_conf_data': lvm.LvmConf(context_wrap(LVM_CONF)),
'pvs_data': lvm.PvsHeadings(context_wrap(PVS_HEADINGS_OUTPUT)),
'vgs_info': lvm.VgsHeadings(context_wrap(VGSHEADING_CONTENT_DOC)),
'lvs_info': lvm.LvsHeadings(context_wrap(LVS_HAEADING_OUTUPT))
}
failed, total = doctest.testmod(lvm, globs=env)
assert failed == 0

0 comments on commit 18a9721

Please sign in to comment.