Skip to content

Commit

Permalink
Change UDEV rules for switchdev mode
Browse files Browse the repository at this point in the history 
The operator will create two rules specific
for PFs in swithcdev mode:
- rule to persist PF name after switching to switchdev mode
- rule to rename VF representors

First rule can be applied before PF is switched to switchdev
mode. Second rule can be applied only when the PF is already
in switchdev mode(phys_port_name and phys_switch_id) may
not be available when PF is in legacy mode.

reload of UDEV rules is required to apply VF representor rule
for already created VFs.

Signed-off-by: Yury Kulazhenkov <[email protected]>
  • Loading branch information
@ykulazhenkov
ykulazhenkov committed Mar 21, 2024
1 parent 68ef448 commit b40c9ba
Show file tree
Hide file tree
Showing 9 changed files with 298 additions and 62 deletions.
8 changes: 7 additions & 1 deletion api/v1/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,13 @@ func GetVfDeviceID(deviceID string) string {
}

func IsSwitchdevModeSpec(spec SriovNetworkNodeStateSpec) bool {
for _, iface := range spec.Interfaces {
return ContainsSwitchdevInterface(spec.Interfaces)
}

// ContainsSwitchdevInterface returns true if provided interface list contains interface
// with switchdev configuration
func ContainsSwitchdevInterface(interfaces []Interface) bool {
for _, iface := range interfaces {
if iface.EswitchMode == ESwithModeSwitchDev {
return true
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/consts/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ const (
UdevDisableNM = "/bindata/scripts/udev-find-sriov-pf.sh"
UdevRepName = "/bindata/scripts/switchdev-vf-link-name.sh"
// nolint:goconst
PFNameUdevRule = `SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", KERNELS=="%s", NAME="%s"`
// nolint:goconst
NMUdevRule = `SUBSYSTEM=="net", ` +
`ACTION=="add|change|move", ` +
`ATTRS{device}=="%s", ` +
Expand Down
66 changes: 54 additions & 12 deletions pkg/helper/mock/mock_helper.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

59 changes: 45 additions & 14 deletions pkg/host/internal/sriov/sriov.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,18 +315,25 @@ func (s *sriov) configSriovPFDevice(iface *sriovnetworkv1.Interface) error {
if err := s.configureHWOptionsForSwitchdev(iface); err != nil {
return err
}
// remove all UDEV rules for the PF before adding new rules to
// make sure that rules are always in a consistent state, e.g. there is no
// switchdev-related rules for PF in legacy mode
if err := s.removeUdevRules(iface.PciAddress); err != nil {
log.Log.Error(err, "configSriovPFDevice(): fail to remove udev rules", "device", iface.PciAddress)
return err
}
err := s.addUdevRules(iface)
if err != nil {
log.Log.Error(err, "configSriovPFDevice(): fail to set add udev rules", "device", iface.PciAddress)
log.Log.Error(err, "configSriovPFDevice(): fail to add udev rules", "device", iface.PciAddress)
return err
}
err = s.createVFs(iface)
if err != nil {
log.Log.Error(err, "configSriovPFDevice(): fail to set NumVfs for device", "device", iface.PciAddress)
errRemove := s.removeUdevRules(iface.PciAddress)
if errRemove != nil {
log.Log.Error(errRemove, "configSriovPFDevice(): fail to remove udev rule", "device", iface.PciAddress)
}
return err
}
if err := s.addVfRepresentorUdevRule(iface); err != nil {
log.Log.Error(err, "configSriovPFDevice(): fail to add VR representor udev rule", "device", iface.PciAddress)
return err
}
// set PF mtu
Expand Down Expand Up @@ -596,6 +603,14 @@ func (s *sriov) ConfigSriovInterfaces(storeManager store.ManagerInterface,
log.Log.Error(err, "cannot configure sriov interfaces")
return fmt.Errorf("cannot configure sriov interfaces")
}
if sriovnetworkv1.ContainsSwitchdevInterface(interfaces) && len(toBeConfigured) > 0 {
// for switchdev devices we create udev rule that renames VF representors
// after VFs are created. Reload rules to update interfaces
if err := s.udevHelper.LoadUdevRules(); err != nil {
log.Log.Error(err, "cannot reload udev rules")
return fmt.Errorf("failed to reload udev rules: %v", err)
}
}

if vars.ParallelNicConfig {
err = s.resetSriovInterfacesInParallel(storeManager, toBeResetted)
Expand Down Expand Up @@ -890,25 +905,38 @@ func (s *sriov) encapTypeToLinkType(encapType string) string {

// create required udev rules for PF:
// * rule to disable NetworkManager for VFs - for all modes
// * rule to rename VF representors - only for switchdev mode
// * rule to keep PF name after switching to switchdev mode - only for switchdev mode
func (s *sriov) addUdevRules(iface *sriovnetworkv1.Interface) error {
log.Log.V(2).Info("addUdevRules(): add udev rules for device",
"device", iface.PciAddress)
if err := s.udevHelper.AddUdevRule(iface.PciAddress); err != nil {
if err := s.udevHelper.AddDisableNMUdevRule(iface.PciAddress); err != nil {
return err
}
if sriovnetworkv1.GetEswitchModeFromSpec(iface) == sriovnetworkv1.ESwithModeSwitchDev {
if err := s.udevHelper.AddPersistPFNameUdevRule(iface.PciAddress, iface.Name); err != nil {
return err
}
}
return nil
}

// add switchdev-specific udev rule that renames representors.
// this rule relies on phys_port_name and phys_switch_id parameter which
// on old kernels can be read only after switching PF to switchdev mode.
// if PF doesn't expose phys_port_name and phys_switch_id, then rule creation will be skipped
func (s *sriov) addVfRepresentorUdevRule(iface *sriovnetworkv1.Interface) error {
if sriovnetworkv1.GetEswitchModeFromSpec(iface) == sriovnetworkv1.ESwithModeSwitchDev {
portName, err := s.networkHelper.GetPhysPortName(iface.Name)
if err != nil {
return err
log.Log.Error(err, "addVfRepresentorUdevRule(): WARNING: can't read phys_port_name for device, skip creation of UDEV rule")
return nil
}
switchID, err := s.networkHelper.GetPhysSwitchID(iface.Name)
if err != nil {
return err
}
if err := s.udevHelper.AddVfRepresentorUdevRule(iface.PciAddress, iface.Name, switchID, portName); err != nil {
return err
log.Log.Error(err, "addVfRepresentorUdevRule(): WARNING: can't read phys_switch_id for device, skip creation of UDEV rule")
return nil
}
return s.udevHelper.AddVfRepresentorUdevRule(iface.PciAddress, iface.Name, switchID, portName)
}
return nil
}
Expand All @@ -917,10 +945,13 @@ func (s *sriov) addUdevRules(iface *sriovnetworkv1.Interface) error {
func (s *sriov) removeUdevRules(pciAddress string) error {
log.Log.V(2).Info("removeUdevRules(): remove udev rules for device",
"device", pciAddress)
if err := s.udevHelper.RemoveUdevRule(pciAddress); err != nil {
if err := s.udevHelper.RemoveDisableNMUdevRule(pciAddress); err != nil {
return err
}
if err := s.udevHelper.RemoveVfRepresentorUdevRule(pciAddress); err != nil {
return err
}
return s.udevHelper.RemoveVfRepresentorUdevRule(pciAddress)
return s.udevHelper.RemovePersistPFNameUdevRule(pciAddress)
}

// create VFs on the PF
Expand Down
13 changes: 8 additions & 5 deletions pkg/host/internal/sriov/sriov_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ var _ = Describe("SRIOV", func() {
dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0)
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{
Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil)
hostMock.EXPECT().AddUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil)
dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2", "0000:d8:00.3"}, nil)
pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl)
netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(3)
Expand Down Expand Up @@ -179,7 +179,7 @@ var _ = Describe("SRIOV", func() {
dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0)
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{
Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil)
hostMock.EXPECT().AddUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil)
dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil)
pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl)
netlinkLibMock.EXPECT().LinkByName("enp216s0f0np0").Return(pfLinkMock, nil).Times(2)
Expand Down Expand Up @@ -227,7 +227,8 @@ var _ = Describe("SRIOV", func() {
hostMock.EXPECT().IsKernelLockdownMode().Return(false)
dputilsLibMock.EXPECT().GetSriovVFcapacity("0000:d8:00.0").Return(1)
dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0)
hostMock.EXPECT().AddUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().AddPersistPFNameUdevRule("0000:d8:00.0", "enp216s0f0np0").Return(nil)
hostMock.EXPECT().EnableHwTcOffload("enp216s0f0np0").Return(nil)
hostMock.EXPECT().GetDevlinkDeviceParam("0000:d8:00.0", "flow_steering_mode").Return("", syscall.EINVAL)
dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2"}, nil).Times(2)
Expand Down Expand Up @@ -257,6 +258,7 @@ var _ = Describe("SRIOV", func() {
hostMock.EXPECT().GetPhysSwitchID("enp216s0f0np0").Return("7cfe90ff2cc0", nil)
hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil)
hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa")
hostMock.EXPECT().LoadUdevRules().Return(nil)

storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil)

Expand Down Expand Up @@ -345,7 +347,8 @@ var _ = Describe("SRIOV", func() {
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
nil)
hostMock.EXPECT().RemoveUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().SetNetdevMTU("0000:d8:00.0", 1500).Return(nil)

Expand Down Expand Up @@ -391,7 +394,7 @@ var _ = Describe("SRIOV", func() {
netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
nil)
hostMock.EXPECT().AddUdevRule("0000:d8:00.0").Return(nil)
hostMock.EXPECT().AddDisableNMUdevRule("0000:d8:00.0").Return(nil)
dputilsLibMock.EXPECT().GetVFList("0000:d8:00.0").Return([]string{"0000:d8:00.2", "0000:d8:00.3"}, nil)
hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil)
hostMock.EXPECT().Unbind("0000:d8:00.3").Return(nil)
Expand Down
42 changes: 36 additions & 6 deletions pkg/host/internal/udev/udev.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,19 +69,32 @@ func (u *udev) PrepareVFRepUdevRule() error {
return nil
}

// AddUdevRule adds a udev rule that disables network-manager for VFs on the concrete PF
func (u *udev) AddUdevRule(pfPciAddress string) error {
log.Log.V(2).Info("AddUdevRule()", "device", pfPciAddress)
// AddDisableNMUdevRule adds udev rule that disables NetworkManager for VFs on the concrete PF:
func (u *udev) AddDisableNMUdevRule(pfPciAddress string) error {
log.Log.V(2).Info("AddDisableNMUdevRule()", "device", pfPciAddress)
udevRuleContent := fmt.Sprintf(consts.NMUdevRule, strings.Join(vars.SupportedVfIds, "|"), pfPciAddress)
return u.addUdevRule(pfPciAddress, "10-nm-disable", udevRuleContent)
}

// RemoveUdevRule removes a udev rule that disables network-manager for VFs on the concrete PF
func (u *udev) RemoveUdevRule(pfPciAddress string) error {
log.Log.V(2).Info("RemoveUdevRule()", "device", pfPciAddress)
// RemoveDisableNMUdevRule removes udev rule that disables NetworkManager for VFs on the concrete PF
func (u *udev) RemoveDisableNMUdevRule(pfPciAddress string) error {
log.Log.V(2).Info("RemoveDisableNMUdevRule()", "device", pfPciAddress)
return u.removeUdevRule(pfPciAddress, "10-nm-disable")
}

// AddPersistPFNameUdevRule add udev rule that preserves PF name after switching to switchdev mode
func (u *udev) AddPersistPFNameUdevRule(pfPciAddress, pfName string) error {
log.Log.V(2).Info("AddPersistPFNameUdevRule()", "device", pfPciAddress)
udevRuleContent := fmt.Sprintf(consts.PFNameUdevRule, pfPciAddress, pfName)
return u.addUdevRule(pfPciAddress, "10-pf-name", udevRuleContent)
}

// RemovePersistPFNameUdevRule removes udev rule that preserves PF name after switching to switchdev mode
func (u *udev) RemovePersistPFNameUdevRule(pfPciAddress string) error {
log.Log.V(2).Info("RemovePersistPFNameUdevRule()", "device", pfPciAddress)
return u.removeUdevRule(pfPciAddress, "10-pf-name")
}

// AddVfRepresentorUdevRule adds udev rule that renames VF representors on the concrete PF
func (u *udev) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort string) error {
log.Log.V(2).Info("AddVfRepresentorUdevRule()",
Expand All @@ -96,6 +109,23 @@ func (u *udev) RemoveVfRepresentorUdevRule(pfPciAddress string) error {
return u.removeUdevRule(pfPciAddress, "20-switchdev")
}

// LoadUdevRules triggers udev rules for network subsystem
func (u *udev) LoadUdevRules() error {
log.Log.V(2).Info("LoadUdevRules()")
udevAdmTool := "udevadm"
_, stderr, err := u.utilsHelper.RunCommand(udevAdmTool, "control", "--reload-rules")
if err != nil {
log.Log.Error(err, "LoadUdevRules(): failed to reload rules", "error", stderr)
return err
}
_, stderr, err = u.utilsHelper.RunCommand(udevAdmTool, "trigger", "--action", "add", "--attr-match", "subsystem=net")
if err != nil {
log.Log.Error(err, "LoadUdevRules(): failed to trigger rules", "error", stderr)
return err
}
return nil
}

func (u *udev) addUdevRule(pfPciAddress, ruleName, ruleContent string) error {
log.Log.V(2).Info("addUdevRule()", "device", pfPciAddress, "rule", ruleName)
rulePath := u.getRuleFolderPath()
Expand Down
Loading

0 comments on commit b40c9ba

Please sign in to comment.