Reload VF driver only when rdma is enabled

k8snetworkplumbingwg · Dec 1, 2021 · 4949504 · 4949504
1 parent 11faae7
commit 4949504
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 57 deletions.
diff --git a/api/v1/helper.go b/api/v1/helper.go
@@ -348,6 +348,7 @@ func (p *SriovNetworkNodePolicy) generateVfGroup(iface *InterfaceExt) (*VfGroup,
 		VfRange:      rng,
 		PolicyName:   p.GetName(),
 		Mtu:          p.Spec.Mtu,
+		IsRdma:       p.Spec.IsRdma,
 	}, nil
 }
 

diff --git a/api/v1/sriovnetworknodestate_types.go b/api/v1/sriovnetworknodestate_types.go
@@ -45,6 +45,7 @@ type VfGroup struct {
 	VfRange      string `json:"vfRange,omitempty"`
 	PolicyName   string `json:"policyName,omitempty"`
 	Mtu          int    `json:"mtu,omitempty"`
+	IsRdma       bool   `json:"isRdma,omitempty"`
 }
 
 type Interfaces []Interface

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml
@@ -59,6 +59,8 @@ spec:
                         properties:
                           deviceType:
                             type: string
+                          isRdma:
+                            type: boolean
                           mtu:
                             type: integer
                           policyName:

diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
@@ -233,13 +233,6 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor
 			glog.Errorf("configSriovDevice(): fail to set NumVfs for device %s", iface.PciAddress)
 			return err
 		}
-		if strings.EqualFold(iface.LinkType, "IB") {
-			if err = setVfsGuid(iface); err != nil {
-				return err
-			}
-		} else if err = setVfsAdminMac(ifaceStatus); err != nil {
-			return err
-		}
 	}
 	// set PF mtu
 	if iface.Mtu > 0 && iface.Mtu != ifaceStatus.Mtu {
@@ -255,22 +248,40 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor
 		if err != nil {
 			glog.Warningf("configSriovDevice(): unable to parse VFs for device %+v %q", iface.PciAddress, err)
 		}
+		pfLink, err := netlink.LinkByName(iface.Name)
+		if err != nil {
+			glog.Errorf("setVfGuid(): unable to get PF link for device %+v %q", iface, err)
+			return err
+		}
+
 		for _, addr := range vfAddrs {
 			var group sriovnetworkv1.VfGroup
 			i := 0
-			driver := ""
+			var driver string
+			var isRdma bool
 			vfID, err := dputils.GetVFID(addr)
 			for i, group = range iface.VfGroups {
 				if err != nil {
 					glog.Warningf("configSriovDevice(): unable to get VF id %+v %q", iface.PciAddress, err)
 				}
 				if sriovnetworkv1.IndexInRange(vfID, group.VfRange) {
+					isRdma = group.IsRdma
 					if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) {
 						driver = group.DeviceType
 					}
 					break
 				}
 			}
+			if strings.EqualFold(iface.LinkType, "IB") {
+				if err = setVfGuid(addr, pfLink); err != nil {
+					return err
+				}
+			} else if err = setVfAdminMac(addr, pfLink); err != nil {
+				return err
+			}
+			if err = unbindDriverIfNeeded(addr, isRdma); err != nil {
+				return err
+			}
 			if driver == "" {
 				if err := BindDefaultDriver(addr); err != nil {
 					glog.Warningf("configSriovDevice(): fail to bind default driver for device %s", addr)
@@ -529,39 +540,33 @@ func vfIsReady(pciAddr string) (netlink.Link, error) {
 	return vfLink, nil
 }
 
-func setVfsAdminMac(iface *sriovnetworkv1.InterfaceExt) error {
-	glog.Infof("setVfsAdminMac(): device %s", iface.PciAddress)
-	pfLink, err := netlink.LinkByName(iface.Name)
+func setVfAdminMac(vfAddr string, pfLink netlink.Link) error {
+	glog.Infof("setVfAdminMac(): VF %s", vfAddr)
+
+	vfID, err := dputils.GetVFID(vfAddr)
 	if err != nil {
-		glog.Errorf("setVfsAdminMac(): unable to get PF link for device %+v %q", iface, err)
+		glog.Errorf("setVfAdminMac(): unable to get VF id %+v %q", vfAddr, err)
 		return err
 	}
-	vfs, err := dputils.GetVFList(iface.PciAddress)
+	vfLink, err := vfIsReady(vfAddr)
 	if err != nil {
+		glog.Errorf("setVfAdminMac(): VF link is not ready for device %+v %q", vfAddr, err)
 		return err
 	}
-	for _, addr := range vfs {
-		vfID, err := dputils.GetVFID(addr)
-		if err != nil {
-			glog.Errorf("setVfsAdminMac(): unable to get VF id %+v %q", iface.PciAddress, err)
-			return err
-		}
-		vfLink, err := vfIsReady(addr)
-		if err != nil {
-			glog.Errorf("setVfsAdminMac(): VF link is not ready for device %+v %q", addr, err)
-			return err
-		}
-		if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil {
-			return err
-		}
-		if err = Unbind(addr); err != nil {
-			return err
-		}
-		if err = BindDefaultDriver(addr); err != nil {
+	if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func unbindDriverIfNeeded(vfAddr string, isRdma bool) error {
+	if isRdma {
+		glog.Infof("unbindDriverIfNeeded(): unbind driver for %s", vfAddr)
+		if err := Unbind(vfAddr); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }
 
@@ -584,36 +589,22 @@ func getLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string {
 	return ""
 }
 
-func setVfsGuid(iface *sriovnetworkv1.Interface) error {
-	glog.Infof("setVfsGuid(): device %s", iface.PciAddress)
-	pfLink, err := netlink.LinkByName(iface.Name)
+func setVfGuid(vfAddr string, pfLink netlink.Link) error {
+	glog.Infof("setVfGuid(): VF %s", vfAddr)
+	vfID, err := dputils.GetVFID(vfAddr)
 	if err != nil {
-		glog.Errorf("setVfsGuid(): unable to get PF link for device %+v %q", iface, err)
+		glog.Errorf("setVfGuid(): unable to get VF id %+v %q", vfAddr, err)
 		return err
 	}
-	vfs, err := dputils.GetVFList(iface.PciAddress)
-	if err != nil {
+	guid := generateRandomGuid()
+	if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil {
 		return err
 	}
-	for _, addr := range vfs {
-		vfID, err := dputils.GetVFID(addr)
-		if err != nil {
-			glog.Errorf("setVfsGuid(): unable to get VF id %+v %q", iface.PciAddress, err)
-			return err
-		}
-		guid := generateRandomGuid()
-		if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil {
-			return err
-		}
-		if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil {
-			return err
-		}
-		if err = Unbind(addr); err != nil {
-			return err
-		}
-		if err = BindDefaultDriver(addr); err != nil {
-			return err
-		}
+	if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil {
+		return err
+	}
+	if err = Unbind(vfAddr); err != nil {
+		return err
 	}
 
 	return nil