diff --git a/api/v1/helper.go b/api/v1/helper.go index c17a85718..3dee6681d 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -348,6 +348,7 @@ func (p *SriovNetworkNodePolicy) generateVfGroup(iface *InterfaceExt) (*VfGroup, VfRange: rng, PolicyName: p.GetName(), Mtu: p.Spec.Mtu, + IsRdma: p.Spec.IsRdma, }, nil } diff --git a/api/v1/sriovnetworknodestate_types.go b/api/v1/sriovnetworknodestate_types.go index abcc121ad..b3147ce9e 100644 --- a/api/v1/sriovnetworknodestate_types.go +++ b/api/v1/sriovnetworknodestate_types.go @@ -45,6 +45,7 @@ type VfGroup struct { VfRange string `json:"vfRange,omitempty"` PolicyName string `json:"policyName,omitempty"` Mtu int `json:"mtu,omitempty"` + IsRdma bool `json:"isRdma,omitempty"` } type Interfaces []Interface diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index d2595edcb..8ccd4ef29 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -59,6 +59,8 @@ spec: properties: deviceType: type: string + isRdma: + type: boolean mtu: type: integer policyName: diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 181216dfa..b02793410 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -233,13 +233,6 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor glog.Errorf("configSriovDevice(): fail to set NumVfs for device %s", iface.PciAddress) return err } - if strings.EqualFold(iface.LinkType, "IB") { - if err = setVfsGuid(iface); err != nil { - return err - } - } else if err = setVfsAdminMac(ifaceStatus); err != nil { - return err - } } // set PF mtu if iface.Mtu > 0 && iface.Mtu != ifaceStatus.Mtu { @@ -255,22 +248,40 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor if err != nil { glog.Warningf("configSriovDevice(): unable to parse VFs for device %+v %q", iface.PciAddress, err) } + pfLink, err := netlink.LinkByName(iface.Name) + if err != nil { + glog.Errorf("setVfGuid(): unable to get PF link for device %+v %q", iface, err) + return err + } + for _, addr := range vfAddrs { var group sriovnetworkv1.VfGroup i := 0 - driver := "" + var driver string + var isRdma bool vfID, err := dputils.GetVFID(addr) for i, group = range iface.VfGroups { if err != nil { glog.Warningf("configSriovDevice(): unable to get VF id %+v %q", iface.PciAddress, err) } if sriovnetworkv1.IndexInRange(vfID, group.VfRange) { + isRdma = group.IsRdma if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) { driver = group.DeviceType } break } } + if strings.EqualFold(iface.LinkType, "IB") { + if err = setVfGuid(addr, pfLink); err != nil { + return err + } + } else if err = setVfAdminMac(addr, pfLink); err != nil { + return err + } + if err = unbindDriverIfNeeded(addr, isRdma); err != nil { + return err + } if driver == "" { if err := BindDefaultDriver(addr); err != nil { glog.Warningf("configSriovDevice(): fail to bind default driver for device %s", addr) @@ -529,39 +540,33 @@ func vfIsReady(pciAddr string) (netlink.Link, error) { return vfLink, nil } -func setVfsAdminMac(iface *sriovnetworkv1.InterfaceExt) error { - glog.Infof("setVfsAdminMac(): device %s", iface.PciAddress) - pfLink, err := netlink.LinkByName(iface.Name) +func setVfAdminMac(vfAddr string, pfLink netlink.Link) error { + glog.Infof("setVfAdminMac(): VF %s", vfAddr) + + vfID, err := dputils.GetVFID(vfAddr) if err != nil { - glog.Errorf("setVfsAdminMac(): unable to get PF link for device %+v %q", iface, err) + glog.Errorf("setVfAdminMac(): unable to get VF id %+v %q", vfAddr, err) return err } - vfs, err := dputils.GetVFList(iface.PciAddress) + vfLink, err := vfIsReady(vfAddr) if err != nil { + glog.Errorf("setVfAdminMac(): VF link is not ready for device %+v %q", vfAddr, err) return err } - for _, addr := range vfs { - vfID, err := dputils.GetVFID(addr) - if err != nil { - glog.Errorf("setVfsAdminMac(): unable to get VF id %+v %q", iface.PciAddress, err) - return err - } - vfLink, err := vfIsReady(addr) - if err != nil { - glog.Errorf("setVfsAdminMac(): VF link is not ready for device %+v %q", addr, err) - return err - } - if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { - return err - } - if err = Unbind(addr); err != nil { - return err - } - if err = BindDefaultDriver(addr); err != nil { + if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { + return err + } + + return nil +} + +func unbindDriverIfNeeded(vfAddr string, isRdma bool) error { + if isRdma { + glog.Infof("unbindDriverIfNeeded(): unbind driver for %s", vfAddr) + if err := Unbind(vfAddr); err != nil { return err } } - return nil } @@ -584,36 +589,22 @@ func getLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string { return "" } -func setVfsGuid(iface *sriovnetworkv1.Interface) error { - glog.Infof("setVfsGuid(): device %s", iface.PciAddress) - pfLink, err := netlink.LinkByName(iface.Name) +func setVfGuid(vfAddr string, pfLink netlink.Link) error { + glog.Infof("setVfGuid(): VF %s", vfAddr) + vfID, err := dputils.GetVFID(vfAddr) if err != nil { - glog.Errorf("setVfsGuid(): unable to get PF link for device %+v %q", iface, err) + glog.Errorf("setVfGuid(): unable to get VF id %+v %q", vfAddr, err) return err } - vfs, err := dputils.GetVFList(iface.PciAddress) - if err != nil { + guid := generateRandomGuid() + if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { return err } - for _, addr := range vfs { - vfID, err := dputils.GetVFID(addr) - if err != nil { - glog.Errorf("setVfsGuid(): unable to get VF id %+v %q", iface.PciAddress, err) - return err - } - guid := generateRandomGuid() - if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { - return err - } - if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { - return err - } - if err = Unbind(addr); err != nil { - return err - } - if err = BindDefaultDriver(addr); err != nil { - return err - } + if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { + return err + } + if err = Unbind(vfAddr); err != nil { + return err } return nil